From f918a237da712e725c449a1ac376ebd1afb036ea Mon Sep 17 00:00:00 2001 From: Lchangliang <915311741@qq.com> Date: Mon, 18 Jul 2022 16:12:06 +0800 Subject: [PATCH] [improvement](light-schema-change) Support tablet schema cache --- be/src/exec/olap_scanner.cpp | 4 +- be/src/olap/base_tablet.cpp | 11 +- be/src/olap/base_tablet.h | 7 +- be/src/olap/compaction.cpp | 15 +- be/src/olap/compaction.h | 2 +- be/src/olap/data_dir.cpp | 9 +- be/src/olap/delta_writer.cpp | 7 +- be/src/olap/delta_writer.h | 2 +- be/src/olap/push_handler.cpp | 24 +-- be/src/olap/push_handler.h | 13 +- be/src/olap/rowset/beta_rowset.cpp | 3 +- be/src/olap/rowset/beta_rowset.h | 2 +- be/src/olap/rowset/rowset.cpp | 6 +- be/src/olap/rowset/rowset.h | 8 +- be/src/olap/rowset/rowset_factory.cpp | 2 +- be/src/olap/rowset/rowset_factory.h | 2 +- be/src/olap/rowset/rowset_meta.h | 54 ++++-- be/src/olap/rowset/rowset_writer_context.h | 4 +- be/src/olap/rowset/segment_v2/segment.cpp | 18 +- be/src/olap/rowset/segment_v2/segment.h | 6 +- .../rowset/segment_v2/segment_iterator.cpp | 10 +- .../olap/rowset/segment_v2/segment_writer.cpp | 6 +- .../olap/rowset/segment_v2/segment_writer.h | 5 +- be/src/olap/schema_change.cpp | 85 +++++---- be/src/olap/schema_change.h | 2 +- be/src/olap/snapshot_manager.cpp | 14 +- be/src/olap/snapshot_manager.h | 2 +- be/src/olap/tablet.cpp | 46 ++--- be/src/olap/tablet.h | 34 ++-- be/src/olap/tablet_manager.cpp | 2 +- be/src/olap/tablet_schema.cpp | 14 ++ be/src/olap/tablet_schema.h | 4 + be/src/olap/tablet_schema_cache.h | 61 +++++++ be/src/olap/task/engine_checksum_task.cpp | 6 +- be/src/olap/task/engine_clone_task.cpp | 5 +- be/src/service/doris_main.cpp | 1 + be/src/vec/exec/volap_scanner.cpp | 2 +- be/test/olap/delete_handler_test.cpp | 106 ++++++------ .../engine_storage_migration_task_test.cpp | 1 - be/test/olap/rowset/beta_rowset_test.cpp | 16 +- be/test/olap/rowset/rowset_tree_test.cpp | 8 +- .../olap/rowset/segment_v2/segment_test.cpp | 163 +++++++++--------- be/test/olap/tablet_meta_test.cpp | 6 +- be/test/olap/tablet_test.cpp | 7 +- be/test/olap/txn_manager_test.cpp | 12 +- be/test/testutil/mock_rowset.h | 5 +- be/test/testutil/run_all_tests.cpp | 1 + 47 files changed, 481 insertions(+), 342 deletions(-) create mode 100644 be/src/olap/tablet_schema_cache.h diff --git a/be/src/exec/olap_scanner.cpp b/be/src/exec/olap_scanner.cpp index 97d289c3f445bc..31a25c4c75d0fa 100644 --- a/be/src/exec/olap_scanner.cpp +++ b/be/src/exec/olap_scanner.cpp @@ -83,7 +83,7 @@ Status OlapScanner::prepare( LOG(WARNING) << ss.str(); return Status::InternalError(ss.str()); } - _tablet_schema = _tablet->tablet_schema(); + _tablet_schema.copy_from(*_tablet->tablet_schema()); if (!_parent->_olap_scan_node.columns_desc.empty() && _parent->_olap_scan_node.columns_desc[0].col_unique_id >= 0) { _tablet_schema.clear_columns(); @@ -289,7 +289,7 @@ Status OlapScanner::_init_return_columns(bool need_seq_col) { } // expand the sequence column - if (_tablet->tablet_schema().has_sequence_col() && need_seq_col) { + if (_tablet_schema.has_sequence_col() && need_seq_col) { bool has_replace_col = false; for (auto col : _return_columns) { if (_tablet_schema.column(col).aggregation() == diff --git a/be/src/olap/base_tablet.cpp b/be/src/olap/base_tablet.cpp index 041707077146a1..443923e28ff960 100644 --- a/be/src/olap/base_tablet.cpp +++ b/be/src/olap/base_tablet.cpp @@ -19,6 +19,7 @@ #include "gutil/strings/substitute.h" #include "olap/data_dir.h" +#include "olap/tablet_schema_cache.h" #include "util/doris_metrics.h" #include "util/path_util.h" @@ -29,10 +30,8 @@ extern MetricPrototype METRIC_query_scan_rows; extern MetricPrototype METRIC_query_scan_count; BaseTablet::BaseTablet(TabletMetaSharedPtr tablet_meta, DataDir* data_dir) - : _state(tablet_meta->tablet_state()), - _tablet_meta(tablet_meta), - _schema(tablet_meta->tablet_schema()), - _data_dir(data_dir) { + : _state(tablet_meta->tablet_state()), _tablet_meta(tablet_meta), _data_dir(data_dir) { + _schema = TabletSchemaCache::instance()->insert(_tablet_meta->tablet_schema().to_key()); _gen_tablet_path(); std::stringstream ss; @@ -72,8 +71,8 @@ void BaseTablet::_gen_tablet_path() { bool BaseTablet::set_tablet_schema_into_rowset_meta() { bool flag = false; for (RowsetMetaSharedPtr rowset_meta : _tablet_meta->all_mutable_rs_metas()) { - if (!rowset_meta->get_rowset_pb().has_tablet_schema()) { - rowset_meta->set_tablet_schema(&_schema); + if (!rowset_meta->tablet_schema()) { + rowset_meta->set_tablet_schema(_schema); flag = true; } } diff --git a/be/src/olap/base_tablet.h b/be/src/olap/base_tablet.h index 26b44f96651079..43e0e822adb191 100644 --- a/be/src/olap/base_tablet.h +++ b/be/src/olap/base_tablet.h @@ -22,6 +22,7 @@ #include "olap/olap_define.h" #include "olap/tablet_meta.h" +#include "olap/tablet_schema.h" #include "olap/utils.h" #include "util/metrics.h" @@ -64,7 +65,7 @@ class BaseTablet : public std::enable_shared_from_this { void set_storage_policy(const std::string& policy) { _tablet_meta->set_storage_policy(policy); } // properties encapsulated in TabletSchema - virtual const TabletSchema& tablet_schema() const; + virtual TabletSchemaSPtr tablet_schema() const; bool set_tablet_schema_into_rowset_meta(); @@ -74,7 +75,7 @@ class BaseTablet : public std::enable_shared_from_this { protected: TabletState _state; TabletMetaSharedPtr _tablet_meta; - const TabletSchema& _schema; + TabletSchemaSPtr _schema; DataDir* _data_dir; std::string _tablet_path; @@ -145,7 +146,7 @@ inline bool BaseTablet::equal(int64_t id, int32_t hash) { return (tablet_id() == id) && (schema_hash() == hash); } -inline const TabletSchema& BaseTablet::tablet_schema() const { +inline TabletSchemaSPtr BaseTablet::tablet_schema() const { return _schema; } diff --git a/be/src/olap/compaction.cpp b/be/src/olap/compaction.cpp index 9a0e50d591b2f0..d81290c9b1c573 100644 --- a/be/src/olap/compaction.cpp +++ b/be/src/olap/compaction.cpp @@ -19,6 +19,7 @@ #include "common/status.h" #include "gutil/strings/substitute.h" +#include "olap/rowset/rowset.h" #include "olap/rowset/rowset_meta.h" #include "olap/tablet.h" #include "util/time.h" @@ -150,9 +151,13 @@ Status Compaction::do_compaction_impl(int64_t permits) { LOG(INFO) << "start " << merge_type << compaction_name() << ". tablet=" << _tablet->full_name() << ", output_version=" << _output_version << ", permits: " << permits; // get cur schema if rowset schema exist, rowset schema must be newer than tablet schema - const TabletSchema cur_tablet_schema = _tablet->tablet_schema(); + std::vector rowset_metas(_input_rowsets.size()); + std::transform(_input_rowsets.begin(), _input_rowsets.end(), rowset_metas.begin(), + [](const RowsetSharedPtr& rowset) { return rowset->rowset_meta(); }); + TabletSchemaSPtr cur_tablet_schema = + _tablet->rowset_meta_with_max_schema_version(rowset_metas)->tablet_schema(); - RETURN_NOT_OK(construct_output_rowset_writer(&cur_tablet_schema)); + RETURN_NOT_OK(construct_output_rowset_writer(cur_tablet_schema)); RETURN_NOT_OK(construct_input_rowset_readers()); TRACE("prepare finished"); @@ -166,10 +171,10 @@ Status Compaction::do_compaction_impl(int64_t permits) { } if (use_vectorized_compaction) { - res = Merger::vmerge_rowsets(_tablet, compaction_type(), &cur_tablet_schema, + res = Merger::vmerge_rowsets(_tablet, compaction_type(), cur_tablet_schema.get(), _input_rs_readers, _output_rs_writer.get(), &stats); } else { - res = Merger::merge_rowsets(_tablet, compaction_type(), &cur_tablet_schema, + res = Merger::merge_rowsets(_tablet, compaction_type(), cur_tablet_schema.get(), _input_rs_readers, _output_rs_writer.get(), &stats); } @@ -233,7 +238,7 @@ Status Compaction::do_compaction_impl(int64_t permits) { return Status::OK(); } -Status Compaction::construct_output_rowset_writer(const TabletSchema* schema) { +Status Compaction::construct_output_rowset_writer(TabletSchemaSPtr schema) { return _tablet->create_rowset_writer(_output_version, VISIBLE, NONOVERLAPPING, schema, _oldest_write_timestamp, _newest_write_timestamp, &_output_rs_writer); diff --git a/be/src/olap/compaction.h b/be/src/olap/compaction.h index 7d7a43bfd37988..a4debd9bb2e561 100644 --- a/be/src/olap/compaction.h +++ b/be/src/olap/compaction.h @@ -65,7 +65,7 @@ class Compaction { Status modify_rowsets(); void gc_output_rowset(); - Status construct_output_rowset_writer(const TabletSchema* schema); + Status construct_output_rowset_writer(TabletSchemaSPtr schema); Status construct_input_rowset_readers(); Status check_version_continuity(const std::vector& rowsets); diff --git a/be/src/olap/data_dir.cpp b/be/src/olap/data_dir.cpp index baf51034e6a266..6d67cab4f43895 100644 --- a/be/src/olap/data_dir.cpp +++ b/be/src/olap/data_dir.cpp @@ -18,6 +18,7 @@ #include "olap/data_dir.h" #include +#include #include #include #include @@ -475,8 +476,8 @@ Status DataDir::load() { } if (rowset_meta->rowset_state() == RowsetStatePB::COMMITTED && rowset_meta->tablet_uid() == tablet->tablet_uid()) { - if (!rowset_meta->get_rowset_pb().has_tablet_schema()) { - rowset_meta->set_tablet_schema(&tablet->tablet_schema()); + if (!rowset_meta->tablet_schema()) { + rowset_meta->set_tablet_schema(tablet->tablet_schema()); RowsetMetaManager::save(_meta, rowset_meta->tablet_uid(), rowset_meta->rowset_id(), rowset_meta->get_rowset_pb()); } @@ -498,8 +499,8 @@ Status DataDir::load() { } } else if (rowset_meta->rowset_state() == RowsetStatePB::VISIBLE && rowset_meta->tablet_uid() == tablet->tablet_uid()) { - if (!rowset_meta->get_rowset_pb().has_tablet_schema()) { - rowset_meta->set_tablet_schema(&tablet->tablet_schema()); + if (!rowset_meta->tablet_schema()) { + rowset_meta->set_tablet_schema(tablet->tablet_schema()); RowsetMetaManager::save(_meta, rowset_meta->tablet_uid(), rowset_meta->rowset_id(), rowset_meta->get_rowset_pb()); } diff --git a/be/src/olap/delta_writer.cpp b/be/src/olap/delta_writer.cpp index e50a8567b4f256..0596a5d52fd80b 100644 --- a/be/src/olap/delta_writer.cpp +++ b/be/src/olap/delta_writer.cpp @@ -123,10 +123,11 @@ Status DeltaWriter::init() { _req.txn_id, _req.load_id)); } // build tablet schema in request level - _build_current_tablet_schema(_req.index_id, _req.ptable_schema_param, _tablet->tablet_schema()); + _build_current_tablet_schema(_req.index_id, _req.ptable_schema_param, + *_tablet->tablet_schema()); RETURN_NOT_OK(_tablet->create_rowset_writer(_req.txn_id, _req.load_id, PREPARED, OVERLAPPING, - _tablet_schema.get(), &_rowset_writer)); + _tablet_schema, &_rowset_writer)); _schema.reset(new Schema(*_tablet_schema)); _reset_mem_table(); @@ -379,7 +380,7 @@ int64_t DeltaWriter::partition_id() const { void DeltaWriter::_build_current_tablet_schema(int64_t index_id, const POlapTableSchemaParam& ptable_schema_param, const TabletSchema& ori_tablet_schema) { - *_tablet_schema = ori_tablet_schema; + _tablet_schema->copy_from(ori_tablet_schema); //new tablet schame if new table if (ptable_schema_param.indexes_size() > 0 && ptable_schema_param.indexes(0).columns_desc_size() != 0 && diff --git a/be/src/olap/delta_writer.h b/be/src/olap/delta_writer.h index 1ce62de3385cfd..6e808c5142c003 100644 --- a/be/src/olap/delta_writer.h +++ b/be/src/olap/delta_writer.h @@ -128,7 +128,7 @@ class DeltaWriter { // tablet schema owned by delta writer, all write will use this tablet schema // it's build from tablet_schema（stored when create tablet） and OlapTableSchema // every request will have it's own tablet schema so simple schema change can work - std::unique_ptr _tablet_schema; + TabletSchemaSPtr _tablet_schema; bool _delta_written_success; StorageEngine* _storage_engine; diff --git a/be/src/olap/push_handler.cpp b/be/src/olap/push_handler.cpp index 0fc20b370c3d8f..3a934776f3f4b6 100644 --- a/be/src/olap/push_handler.cpp +++ b/be/src/olap/push_handler.cpp @@ -31,6 +31,7 @@ #include "olap/schema_change.h" #include "olap/storage_engine.h" #include "olap/tablet.h" +#include "olap/tablet_schema.h" #include "runtime/exec_env.h" namespace doris { @@ -115,7 +116,8 @@ Status PushHandler::_do_streaming_ingestion(TabletSharedPtr tablet, const TPushR } DeletePredicatePB del_pred; - auto tablet_schema = tablet_var.tablet->tablet_schema(); + TabletSchema tablet_schema; + tablet_schema.copy_from(*tablet_var.tablet->tablet_schema()); if (!request.columns_desc.empty() && request.columns_desc[0].col_unique_id >= 0) { tablet_schema.clear_columns(); for (const auto& column_desc : request.columns_desc) { @@ -141,12 +143,12 @@ Status PushHandler::_do_streaming_ingestion(TabletSharedPtr tablet, const TPushR << ". tablet: " << tablet_vars->at(0).tablet->full_name(); return Status::OLAPInternalError(OLAP_ERR_TOO_MANY_VERSION); } - - auto tablet_schema = tablet_vars->at(0).tablet->tablet_schema(); + auto tablet_schema = std::make_shared(); + tablet_schema->copy_from(*tablet_vars->at(0).tablet->tablet_schema()); if (!request.columns_desc.empty() && request.columns_desc[0].col_unique_id >= 0) { - tablet_schema.clear_columns(); + tablet_schema->clear_columns(); for (const auto& column_desc : request.columns_desc) { - tablet_schema.append_column(TabletColumn(column_desc)); + tablet_schema->append_column(TabletColumn(column_desc)); } } @@ -154,12 +156,12 @@ Status PushHandler::_do_streaming_ingestion(TabletSharedPtr tablet, const TPushR if (push_type == PUSH_NORMAL_V2) { res = _convert_v2(tablet_vars->at(0).tablet, tablet_vars->at(1).tablet, &(tablet_vars->at(0).rowset_to_add), &(tablet_vars->at(1).rowset_to_add), - &tablet_schema); + tablet_schema); } else { res = _convert(tablet_vars->at(0).tablet, tablet_vars->at(1).tablet, &(tablet_vars->at(0).rowset_to_add), &(tablet_vars->at(1).rowset_to_add), - &tablet_schema); + tablet_schema); } if (!res.ok()) { LOG(WARNING) << "fail to convert tmp file when realtime push. res=" << res @@ -219,7 +221,7 @@ void PushHandler::_get_tablet_infos(const std::vector& tablet_vars, Status PushHandler::_convert_v2(TabletSharedPtr cur_tablet, TabletSharedPtr new_tablet, RowsetSharedPtr* cur_rowset, RowsetSharedPtr* new_rowset, - const TabletSchema* tablet_schema) { + TabletSchemaSPtr tablet_schema) { Status res = Status::OK(); uint32_t num_rows = 0; PUniqueId load_id; @@ -344,7 +346,7 @@ Status PushHandler::_convert_v2(TabletSharedPtr cur_tablet, TabletSharedPtr new_ Status PushHandler::_convert(TabletSharedPtr cur_tablet, TabletSharedPtr new_tablet, RowsetSharedPtr* cur_rowset, RowsetSharedPtr* new_rowset, - const TabletSchema* tablet_schema) { + TabletSchemaSPtr tablet_schema) { Status res = Status::OK(); RowCursor row; BinaryFile raw_file; @@ -515,7 +517,7 @@ IBinaryReader* IBinaryReader::create(bool need_decompress) { BinaryReader::BinaryReader() : _row_buf(nullptr), _row_buf_size(0) {} -Status BinaryReader::init(const TabletSchema* tablet_schema, BinaryFile* file) { +Status BinaryReader::init(TabletSchemaSPtr tablet_schema, BinaryFile* file) { Status res = Status::OK(); do { @@ -657,7 +659,7 @@ LzoBinaryReader::LzoBinaryReader() _row_num(0), _next_row_start(0) {} -Status LzoBinaryReader::init(const TabletSchema* tablet_schema, BinaryFile* file) { +Status LzoBinaryReader::init(TabletSchemaSPtr tablet_schema, BinaryFile* file) { Status res = Status::OK(); do { diff --git a/be/src/olap/push_handler.h b/be/src/olap/push_handler.h index 70c134c9839ce4..b22d319845b663 100644 --- a/be/src/olap/push_handler.h +++ b/be/src/olap/push_handler.h @@ -30,6 +30,7 @@ #include "olap/olap_common.h" #include "olap/row_cursor.h" #include "olap/rowset/rowset.h" +#include "olap/tablet_schema.h" namespace doris { @@ -61,12 +62,12 @@ class PushHandler { private: Status _convert_v2(TabletSharedPtr cur_tablet, TabletSharedPtr new_tablet_vec, RowsetSharedPtr* cur_rowset, RowsetSharedPtr* new_rowset, - const TabletSchema* tablet_schema); + TabletSchemaSPtr tablet_schema); // Convert local data file to internal formatted delta, // return new delta's SegmentGroup Status _convert(TabletSharedPtr cur_tablet, TabletSharedPtr new_tablet_vec, RowsetSharedPtr* cur_rowset, RowsetSharedPtr* new_rowset, - const TabletSchema* tablet_schema); + TabletSchemaSPtr tablet_schema); // Only for debug std::string _debug_version_list(const Versions& versions) const; @@ -114,7 +115,7 @@ class IBinaryReader { static IBinaryReader* create(bool need_decompress); virtual ~IBinaryReader() = default; - virtual Status init(const TabletSchema* tablet_schema, BinaryFile* file) = 0; + virtual Status init(TabletSchemaSPtr tablet_schema, BinaryFile* file) = 0; virtual Status finalize() = 0; virtual Status next(RowCursor* row) = 0; @@ -133,7 +134,7 @@ class IBinaryReader { _ready(false) {} BinaryFile* _file; - const TabletSchema* _tablet_schema; + TabletSchemaSPtr _tablet_schema; size_t _content_len; size_t _curr; uint32_t _adler_checksum; @@ -146,7 +147,7 @@ class BinaryReader : public IBinaryReader { explicit BinaryReader(); ~BinaryReader() override { finalize(); } - Status init(const TabletSchema* tablet_schema, BinaryFile* file) override; + Status init(TabletSchemaSPtr tablet_schema, BinaryFile* file) override; Status finalize() override; Status next(RowCursor* row) override; @@ -163,7 +164,7 @@ class LzoBinaryReader : public IBinaryReader { explicit LzoBinaryReader(); ~LzoBinaryReader() override { finalize(); } - Status init(const TabletSchema* tablet_schema, BinaryFile* file) override; + Status init(TabletSchemaSPtr tablet_schema, BinaryFile* file) override; Status finalize() override; Status next(RowCursor* row) override; diff --git a/be/src/olap/rowset/beta_rowset.cpp b/be/src/olap/rowset/beta_rowset.cpp index 1e7cbde9f4708c..1a7c26ec0e2d5f 100644 --- a/be/src/olap/rowset/beta_rowset.cpp +++ b/be/src/olap/rowset/beta_rowset.cpp @@ -28,6 +28,7 @@ #include "io/fs/s3_file_system.h" #include "olap/olap_define.h" #include "olap/rowset/beta_rowset_reader.h" +#include "olap/tablet_schema.h" #include "olap/utils.h" #include "util/doris_metrics.h" @@ -59,7 +60,7 @@ std::string BetaRowset::remote_segment_path(int64_t tablet_id, const RowsetId& r segment_id); } -BetaRowset::BetaRowset(const TabletSchema* schema, const std::string& tablet_path, +BetaRowset::BetaRowset(TabletSchemaSPtr schema, const std::string& tablet_path, RowsetMetaSharedPtr rowset_meta) : Rowset(schema, tablet_path, std::move(rowset_meta)) {} diff --git a/be/src/olap/rowset/beta_rowset.h b/be/src/olap/rowset/beta_rowset.h index 8d96bfe090cf72..95aa52b3170943 100644 --- a/be/src/olap/rowset/beta_rowset.h +++ b/be/src/olap/rowset/beta_rowset.h @@ -77,7 +77,7 @@ class BetaRowset : public Rowset { Status load_segments(std::vector* segments); protected: - BetaRowset(const TabletSchema* schema, const std::string& tablet_path, + BetaRowset(TabletSchemaSPtr schema, const std::string& tablet_path, RowsetMetaSharedPtr rowset_meta); // init segment groups diff --git a/be/src/olap/rowset/rowset.cpp b/be/src/olap/rowset/rowset.cpp index 009ce832bf4909..42fdb077a57727 100644 --- a/be/src/olap/rowset/rowset.cpp +++ b/be/src/olap/rowset/rowset.cpp @@ -17,11 +17,13 @@ #include "olap/rowset/rowset.h" +#include "olap/tablet_schema.h" +#include "olap/tablet_schema_cache.h" #include "util/time.h" namespace doris { -Rowset::Rowset(const TabletSchema* schema, const std::string& tablet_path, +Rowset::Rowset(TabletSchemaSPtr schema, const std::string& tablet_path, RowsetMetaSharedPtr rowset_meta) : _tablet_path(tablet_path), _rowset_meta(std::move(rowset_meta)), _refs_by_reader(0) { _is_pending = !_rowset_meta->has_version(); @@ -32,7 +34,7 @@ Rowset::Rowset(const TabletSchema* schema, const std::string& tablet_path, _is_cumulative = version.first != version.second; } // build schema from RowsetMeta.tablet_schema or Tablet.tablet_schema - _schema = _rowset_meta->tablet_schema() != nullptr ? _rowset_meta->tablet_schema() : schema; + _schema = _rowset_meta->tablet_schema() ? _rowset_meta->tablet_schema() : schema; } Status Rowset::load(bool use_cache) { diff --git a/be/src/olap/rowset/rowset.h b/be/src/olap/rowset/rowset.h index 3488ed2b54d8ce..68a4b21a80ff28 100644 --- a/be/src/olap/rowset/rowset.h +++ b/be/src/olap/rowset/rowset.h @@ -138,7 +138,7 @@ class Rowset : public std::enable_shared_from_this { // publish rowset to make it visible to read void make_visible(Version version); - const TabletSchema* tablet_schema() { return _schema; } + TabletSchemaSPtr tablet_schema() { return _schema; } // helper class to access RowsetMeta int64_t start_version() const { return rowset_meta()->version().first; } @@ -158,7 +158,7 @@ class Rowset : public std::enable_shared_from_this { bool delete_flag() const { return rowset_meta()->delete_flag(); } int64_t num_segments() const { return rowset_meta()->num_segments(); } void to_rowset_pb(RowsetMetaPB* rs_meta) const { return rowset_meta()->to_rowset_pb(rs_meta); } - const RowsetMetaPB& get_rowset_pb() const { return rowset_meta()->get_rowset_pb(); } + RowsetMetaPB get_rowset_pb() const { return rowset_meta()->get_rowset_pb(); } int64_t oldest_write_timestamp() const { return rowset_meta()->oldest_write_timestamp(); } int64_t newest_write_timestamp() const { return rowset_meta()->newest_write_timestamp(); } KeysType keys_type() { return _schema->keys_type(); } @@ -269,7 +269,7 @@ class Rowset : public std::enable_shared_from_this { DISALLOW_COPY_AND_ASSIGN(Rowset); // this is non-public because all clients should use RowsetFactory to obtain pointer to initialized Rowset - Rowset(const TabletSchema* schema, const std::string& tablet_path, + Rowset(TabletSchemaSPtr schema, const std::string& tablet_path, RowsetMetaSharedPtr rowset_meta); // this is non-public because all clients should use RowsetFactory to obtain pointer to initialized Rowset @@ -284,7 +284,7 @@ class Rowset : public std::enable_shared_from_this { // allow subclass to add custom logic when rowset is being published virtual void make_visible_extra(Version version) {} - const TabletSchema* _schema; + TabletSchemaSPtr _schema; std::string _tablet_path; RowsetMetaSharedPtr _rowset_meta; // init in constructor diff --git a/be/src/olap/rowset/rowset_factory.cpp b/be/src/olap/rowset/rowset_factory.cpp index 32e84f9c0edd2f..9c8c75b2dc345d 100644 --- a/be/src/olap/rowset/rowset_factory.cpp +++ b/be/src/olap/rowset/rowset_factory.cpp @@ -26,7 +26,7 @@ namespace doris { -Status RowsetFactory::create_rowset(const TabletSchema* schema, const std::string& tablet_path, +Status RowsetFactory::create_rowset(TabletSchemaSPtr schema, const std::string& tablet_path, RowsetMetaSharedPtr rowset_meta, RowsetSharedPtr* rowset) { if (rowset_meta->rowset_type() == ALPHA_ROWSET) { return Status::OLAPInternalError(OLAP_ERR_ROWSET_INVALID); diff --git a/be/src/olap/rowset/rowset_factory.h b/be/src/olap/rowset/rowset_factory.h index 91b36db1ed074a..e216b81f524e25 100644 --- a/be/src/olap/rowset/rowset_factory.h +++ b/be/src/olap/rowset/rowset_factory.h @@ -31,7 +31,7 @@ class RowsetFactory { public: // return OLAP_SUCCESS and set inited rowset in `*rowset`. // return others if failed to create or init rowset. - static Status create_rowset(const TabletSchema* schema, const std::string& tablet_path, + static Status create_rowset(TabletSchemaSPtr schema, const std::string& tablet_path, RowsetMetaSharedPtr rowset_meta, RowsetSharedPtr* rowset); // create and init rowset writer. diff --git a/be/src/olap/rowset/rowset_meta.h b/be/src/olap/rowset/rowset_meta.h index 93aefa4351a65f..5768b37ced61f9 100644 --- a/be/src/olap/rowset/rowset_meta.h +++ b/be/src/olap/rowset/rowset_meta.h @@ -32,6 +32,7 @@ #include "json2pb/pb_to_json.h" #include "olap/olap_common.h" #include "olap/tablet_schema.h" +#include "olap/tablet_schema_cache.h" namespace doris { @@ -53,6 +54,11 @@ class RowsetMeta { virtual bool init_from_pb(const RowsetMetaPB& rowset_meta_pb) { _rowset_meta_pb = rowset_meta_pb; + if (_rowset_meta_pb.has_tablet_schema()) { + _schema = TabletSchemaCache::instance()->insert( + _rowset_meta_pb.tablet_schema().SerializeAsString()); + _rowset_meta_pb.clear_tablet_schema(); + } _init(); return true; } @@ -245,8 +251,20 @@ class RowsetMeta { void set_num_segments(int64_t num_segments) { _rowset_meta_pb.set_num_segments(num_segments); } - void to_rowset_pb(RowsetMetaPB* rs_meta_pb) const { *rs_meta_pb = _rowset_meta_pb; } - const RowsetMetaPB& get_rowset_pb() { return _rowset_meta_pb; } + void to_rowset_pb(RowsetMetaPB* rs_meta_pb) const { + *rs_meta_pb = _rowset_meta_pb; + if (_schema) { + _schema->to_schema_pb(rs_meta_pb->mutable_tablet_schema()); + } + } + + RowsetMetaPB get_rowset_pb() { + RowsetMetaPB rowset_meta_pb = _rowset_meta_pb; + if (_schema) { + _schema->to_schema_pb(rowset_meta_pb.mutable_tablet_schema()); + } + return rowset_meta_pb; + } bool is_singleton_delta() const { return has_version() && _rowset_meta_pb.start_version() == _rowset_meta_pb.end_version(); @@ -323,25 +341,37 @@ class RowsetMeta { int64_t oldest_write_timestamp() const { return _rowset_meta_pb.oldest_write_timestamp(); } int64_t newest_write_timestamp() const { return _rowset_meta_pb.newest_write_timestamp(); } - void set_tablet_schema(const TabletSchema* tablet_schema) { - TabletSchemaPB* ts_pb = _rowset_meta_pb.mutable_tablet_schema(); - tablet_schema->to_schema_pb(ts_pb); - CHECK(_schema == nullptr); - _schema = std::make_shared(*tablet_schema); + void set_tablet_schema(const TabletSchemaSPtr& tablet_schema) { + DCHECK(_schema == nullptr); + _schema = TabletSchemaCache::instance()->insert(tablet_schema->to_key()); } - const TabletSchema* tablet_schema() { return _schema.get(); } + TabletSchemaSPtr tablet_schema() { return _schema; } private: bool _deserialize_from_pb(const std::string& value) { - return _rowset_meta_pb.ParseFromString(value); + RowsetMetaPB rowset_meta_pb; + if (!rowset_meta_pb.ParseFromString(value)) { + return false; + } + if (rowset_meta_pb.has_tablet_schema()) { + _schema = TabletSchemaCache::instance()->insert( + rowset_meta_pb.tablet_schema().SerializeAsString()); + rowset_meta_pb.clear_tablet_schema(); + } + _rowset_meta_pb = rowset_meta_pb; + return true; } bool _serialize_to_pb(std::string* value) { if (value == nullptr) { return false; } - return _rowset_meta_pb.SerializeToString(value); + RowsetMetaPB rowset_meta_pb = _rowset_meta_pb; + if (_schema) { + _schema->to_schema_pb(rowset_meta_pb.mutable_tablet_schema()); + } + return rowset_meta_pb.SerializeToString(value); } void _init() { @@ -350,10 +380,6 @@ class RowsetMeta { } else { _rowset_id.init(_rowset_meta_pb.rowset_id_v2()); } - if (_rowset_meta_pb.has_tablet_schema()) { - _schema = std::make_shared(); - _schema->init_from_pb(_rowset_meta_pb.tablet_schema()); - } } friend bool operator==(const RowsetMeta& a, const RowsetMeta& b) { diff --git a/be/src/olap/rowset/rowset_writer_context.h b/be/src/olap/rowset/rowset_writer_context.h index 2dc238b217350b..b01e6fca468cef 100644 --- a/be/src/olap/rowset/rowset_writer_context.h +++ b/be/src/olap/rowset/rowset_writer_context.h @@ -55,7 +55,7 @@ struct RowsetWriterContext { context.tablet_schema_hash = new_tablet->schema_hash(); context.rowset_type = new_rowset_type; context.tablet_path = new_tablet->tablet_path(); - context.tablet_schema = &(new_tablet->tablet_schema()); + context.tablet_schema = new_tablet->tablet_schema(); context.data_dir = new_tablet->data_dir(); context.rowset_state = VISIBLE; context.version = version; @@ -70,7 +70,7 @@ struct RowsetWriterContext { int64_t partition_id; RowsetTypePB rowset_type; std::string tablet_path; - const TabletSchema* tablet_schema; + TabletSchemaSPtr tablet_schema; // PREPARED/COMMITTED for pending rowset // VISIBLE for non-pending rowset RowsetStatePB rowset_state; diff --git a/be/src/olap/rowset/segment_v2/segment.cpp b/be/src/olap/rowset/segment_v2/segment.cpp index 892743a63d959b..fea05101728565 100644 --- a/be/src/olap/rowset/segment_v2/segment.cpp +++ b/be/src/olap/rowset/segment_v2/segment.cpp @@ -17,6 +17,8 @@ #include "olap/rowset/segment_v2/segment.h" +#include + #include #include @@ -35,7 +37,7 @@ namespace doris { namespace segment_v2 { Status Segment::open(io::FileSystem* fs, const std::string& path, uint32_t segment_id, - const TabletSchema* tablet_schema, std::shared_ptr* output) { + TabletSchemaSPtr tablet_schema, std::shared_ptr* output) { std::shared_ptr segment(new Segment(segment_id, tablet_schema)); io::FileReaderSPtr file_reader; RETURN_IF_ERROR(fs->open_file(path, &file_reader)); @@ -45,8 +47,8 @@ Status Segment::open(io::FileSystem* fs, const std::string& path, uint32_t segme return Status::OK(); } -Segment::Segment(uint32_t segment_id, const TabletSchema* tablet_schema) - : _segment_id(segment_id), _tablet_schema(*tablet_schema), _meta_mem_usage(0) {} +Segment::Segment(uint32_t segment_id, TabletSchemaSPtr tablet_schema) + : _segment_id(segment_id), _tablet_schema(tablet_schema), _meta_mem_usage(0) {} Segment::~Segment() { StorageEngine::instance()->segment_meta_mem_tracker()->release(_meta_mem_usage); @@ -64,7 +66,7 @@ Status Segment::new_iterator(const Schema& schema, const StorageReadOptions& rea // trying to prune the current segment by segment-level zone map if (read_options.conditions != nullptr) { for (auto& column_condition : read_options.conditions->columns()) { - int32_t column_unique_id = _tablet_schema.column(column_condition.first).unique_id(); + int32_t column_unique_id = _tablet_schema->column(column_condition.first).unique_id(); if (_column_readers.count(column_unique_id) < 1 || !_column_readers.at(column_unique_id)->has_zone_map()) { continue; @@ -145,7 +147,7 @@ Status Segment::load_index() { opts.stats = &tmp_stats; opts.type = INDEX_PAGE; - if (_tablet_schema.keys_type() == UNIQUE_KEYS && _footer.has_primary_key_index_meta()) { + if (_tablet_schema->keys_type() == UNIQUE_KEYS && _footer.has_primary_key_index_meta()) { _pk_index_reader.reset(new PrimaryKeyIndexReader()); RETURN_IF_ERROR( _pk_index_reader->parse(_file_reader, _footer.primary_key_index_meta())); @@ -173,15 +175,15 @@ Status Segment::_create_column_readers() { _column_id_to_footer_ordinal.emplace(column_pb.unique_id(), ordinal); } - for (uint32_t ordinal = 0; ordinal < _tablet_schema.num_columns(); ++ordinal) { - auto& column = _tablet_schema.column(ordinal); + for (uint32_t ordinal = 0; ordinal < _tablet_schema->num_columns(); ++ordinal) { + auto& column = _tablet_schema->column(ordinal); auto iter = _column_id_to_footer_ordinal.find(column.unique_id()); if (iter == _column_id_to_footer_ordinal.end()) { continue; } ColumnReaderOptions opts; - opts.kept_in_memory = _tablet_schema.is_in_memory(); + opts.kept_in_memory = _tablet_schema->is_in_memory(); std::unique_ptr reader; RETURN_IF_ERROR(ColumnReader::create(opts, _footer.columns(iter->second), _footer.num_rows(), _file_reader, &reader)); diff --git a/be/src/olap/rowset/segment_v2/segment.h b/be/src/olap/rowset/segment_v2/segment.h index 5323ace8735d08..c7d27495a0e13d 100644 --- a/be/src/olap/rowset/segment_v2/segment.h +++ b/be/src/olap/rowset/segment_v2/segment.h @@ -62,7 +62,7 @@ using SegmentSharedPtr = std::shared_ptr; class Segment : public std::enable_shared_from_this { public: static Status open(io::FileSystem* fs, const std::string& path, uint32_t segment_id, - const TabletSchema* tablet_schema, std::shared_ptr* output); + TabletSchemaSPtr tablet_schema, std::shared_ptr* output); ~Segment(); @@ -96,7 +96,7 @@ class Segment : public std::enable_shared_from_this { private: DISALLOW_COPY_AND_ASSIGN(Segment); - Segment(uint32_t segment_id, const TabletSchema* tablet_schema); + Segment(uint32_t segment_id, TabletSchemaSPtr tablet_schema); // open segment file and read the minimum amount of necessary information (footer) Status _open(); Status _parse_footer(); @@ -107,7 +107,7 @@ class Segment : public std::enable_shared_from_this { io::FileReaderSPtr _file_reader; uint32_t _segment_id; - TabletSchema _tablet_schema; + TabletSchemaSPtr _tablet_schema; int64_t _meta_mem_usage; SegmentFooterPB _footer; diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.cpp b/be/src/olap/rowset/segment_v2/segment_iterator.cpp index a90b31aad6e564..85e73b8a44e378 100644 --- a/be/src/olap/rowset/segment_v2/segment_iterator.cpp +++ b/be/src/olap/rowset/segment_v2/segment_iterator.cpp @@ -143,7 +143,7 @@ Status SegmentIterator::_init(bool is_vec) { RETURN_IF_ERROR(_init_return_column_iterators()); RETURN_IF_ERROR(_init_bitmap_index_iterators()); // z-order can not use prefix index - if (_segment->_tablet_schema.sort_type() != SortType::ZORDER) { + if (_segment->_tablet_schema->sort_type() != SortType::ZORDER) { RETURN_IF_ERROR(_get_row_ranges_by_keys()); } RETURN_IF_ERROR(_get_row_ranges_by_column_conditions()); @@ -393,7 +393,7 @@ int compare_row_with_lhs_columns(const LhsRowType& lhs, const RhsRowType& rhs) { Status SegmentIterator::_lookup_ordinal(const RowCursor& key, bool is_include, rowid_t upper_bound, rowid_t* rowid) { - if (_segment->_tablet_schema.keys_type() == UNIQUE_KEYS && + if (_segment->_tablet_schema->keys_type() == UNIQUE_KEYS && _segment->get_primary_key_index() != nullptr) { return _lookup_ordinal_from_pk_index(key, is_include, rowid); } @@ -415,7 +415,7 @@ Status SegmentIterator::_lookup_ordinal_from_sk_index(const RowCursor& key, bool DCHECK(sk_index_decoder != nullptr); std::string index_key; - encode_key_with_padding(&index_key, key, _segment->_tablet_schema.num_short_key_columns(), + encode_key_with_padding(&index_key, key, _segment->_tablet_schema->num_short_key_columns(), is_include); uint32_t start_block_id = 0; @@ -467,13 +467,13 @@ Status SegmentIterator::_lookup_ordinal_from_sk_index(const RowCursor& key, bool Status SegmentIterator::_lookup_ordinal_from_pk_index(const RowCursor& key, bool is_include, rowid_t* rowid) { - DCHECK(_segment->_tablet_schema.keys_type() == UNIQUE_KEYS); + DCHECK(_segment->_tablet_schema->keys_type() == UNIQUE_KEYS); const PrimaryKeyIndexReader* pk_index_reader = _segment->get_primary_key_index(); DCHECK(pk_index_reader != nullptr); std::string index_key; encode_key_with_padding( - &index_key, key, _segment->_tablet_schema.num_key_columns(), is_include); + &index_key, key, _segment->_tablet_schema->num_key_columns(), is_include); bool exact_match = false; std::unique_ptr index_iterator; diff --git a/be/src/olap/rowset/segment_v2/segment_writer.cpp b/be/src/olap/rowset/segment_v2/segment_writer.cpp index b0285349e2423f..2b748287073aac 100644 --- a/be/src/olap/rowset/segment_v2/segment_writer.cpp +++ b/be/src/olap/rowset/segment_v2/segment_writer.cpp @@ -40,7 +40,7 @@ const char* k_segment_magic = "D0R1"; const uint32_t k_segment_magic_length = 4; SegmentWriter::SegmentWriter(io::FileWriter* file_writer, uint32_t segment_id, - const TabletSchema* tablet_schema, DataDir* data_dir, + TabletSchemaSPtr tablet_schema, DataDir* data_dir, uint32_t max_row_per_segment, const SegmentWriterOptions& opts) : _segment_id(segment_id), _tablet_schema(tablet_schema), @@ -50,7 +50,7 @@ SegmentWriter::SegmentWriter(io::FileWriter* file_writer, uint32_t segment_id, _file_writer(file_writer), _mem_tracker(std::make_unique("SegmentWriter:Segment-" + std::to_string(segment_id))), - _olap_data_convertor(tablet_schema) { + _olap_data_convertor(tablet_schema.get()) { CHECK_NOTNULL(file_writer); if (_tablet_schema->keys_type() == UNIQUE_KEYS && _opts.enable_unique_key_merge_on_write) { _num_key_columns = _tablet_schema->num_key_columns(); @@ -92,7 +92,7 @@ Status SegmentWriter::init(uint32_t write_mbytes_per_sec __attribute__((unused)) ColumnWriterOptions opts; opts.meta = _footer.add_columns(); - init_column_meta(opts.meta, &column_id, column, _tablet_schema); + init_column_meta(opts.meta, &column_id, column, _tablet_schema.get()); // now we create zone map for key columns in AGG_KEYS or all column in UNIQUE_KEYS or DUP_KEYS // and not support zone map for array type. diff --git a/be/src/olap/rowset/segment_v2/segment_writer.h b/be/src/olap/rowset/segment_v2/segment_writer.h index f1d51bb3e97f2a..7d0bc4cf0b2654 100644 --- a/be/src/olap/rowset/segment_v2/segment_writer.h +++ b/be/src/olap/rowset/segment_v2/segment_writer.h @@ -25,6 +25,7 @@ #include "common/status.h" // Status #include "gen_cpp/segment_v2.pb.h" #include "gutil/macros.h" +#include "olap/tablet_schema.h" #include "vec/core/block.h" #include "vec/olap/olap_data_convertor.h" @@ -62,7 +63,7 @@ struct SegmentWriterOptions { class SegmentWriter { public: explicit SegmentWriter(io::FileWriter* file_writer, uint32_t segment_id, - const TabletSchema* tablet_schema, DataDir* data_dir, + TabletSchemaSPtr tablet_schema, DataDir* data_dir, uint32_t max_row_per_segment, const SegmentWriterOptions& opts); ~SegmentWriter(); @@ -102,7 +103,7 @@ class SegmentWriter { private: uint32_t _segment_id; - const TabletSchema* _tablet_schema; + TabletSchemaSPtr _tablet_schema; DataDir* _data_dir; uint32_t _max_row_per_segment; SegmentWriterOptions _opts; diff --git a/be/src/olap/schema_change.cpp b/be/src/olap/schema_change.cpp index ac203e858f04a6..7672e10eecb0cc 100644 --- a/be/src/olap/schema_change.cpp +++ b/be/src/olap/schema_change.cpp @@ -29,6 +29,7 @@ #include "olap/rowset/segment_v2/column_reader.h" #include "olap/storage_engine.h" #include "olap/tablet.h" +#include "olap/tablet_schema.h" #include "olap/types.h" #include "olap/wrapper_field.h" #include "runtime/memory/mem_tracker.h" @@ -117,7 +118,7 @@ class MultiBlockMerger { // The block version is incremental. std::stable_sort(row_refs.begin(), row_refs.end(), _cmp); - auto finalized_block = _tablet->tablet_schema().create_block(); + auto finalized_block = _tablet->tablet_schema()->create_block(); int columns = finalized_block.columns(); *merged_rows += rows; @@ -130,7 +131,7 @@ class MultiBlockMerger { for (int i = key_number; i < columns; i++) { vectorized::AggregateFunctionPtr function = - tablet_schema.column(i).get_aggregate_function( + tablet_schema->column(i).get_aggregate_function( {finalized_block.get_data_type(i)}, vectorized::AGG_LOAD_SUFFIX); agg_functions.push_back(function); // create aggregate data @@ -1094,7 +1095,7 @@ bool RowBlockMerger::merge(const std::vector& row_block_arr, RowsetWr return false; }; - if (row_cursor.init(_tablet->tablet_schema()) != Status::OK()) { + if (row_cursor.init(*_tablet->tablet_schema()) != Status::OK()) { LOG(WARNING) << "fail to init row cursor."; return merge_error(); } @@ -1104,7 +1105,7 @@ bool RowBlockMerger::merge(const std::vector& row_block_arr, RowsetWr return merge_error(); } - row_cursor.allocate_memory_for_string_type(_tablet->tablet_schema()); + row_cursor.allocate_memory_for_string_type(*_tablet->tablet_schema()); while (_heap.size() > 0) { init_row_with_others(&row_cursor, *(_heap.top().row_cursor), mem_pool.get(), agg_object_pool.get()); @@ -1254,7 +1255,7 @@ Status SchemaChangeDirectly::_inner_process(RowsetReaderSharedPtr rowset_reader, RowsetWriter* rowset_writer, TabletSharedPtr new_tablet, TabletSharedPtr base_tablet) { if (_row_block_allocator == nullptr) { - _row_block_allocator = new RowBlockAllocator(new_tablet->tablet_schema(), 0); + _row_block_allocator = new RowBlockAllocator(*new_tablet->tablet_schema(), 0); if (_row_block_allocator == nullptr) { LOG(FATAL) << "failed to malloc RowBlockAllocator. size=" << sizeof(RowBlockAllocator); return Status::OLAPInternalError(OLAP_ERR_INPUT_PARAMETER_ERROR); @@ -1268,7 +1269,7 @@ Status SchemaChangeDirectly::_inner_process(RowsetReaderSharedPtr rowset_reader, return Status::OLAPInternalError(OLAP_ERR_INPUT_PARAMETER_ERROR); } - if (!_cursor->init(new_tablet->tablet_schema())) { + if (!_cursor->init(*new_tablet->tablet_schema())) { LOG(WARNING) << "fail to init row cursor."; return Status::OLAPInternalError(OLAP_ERR_INPUT_PARAMETER_ERROR); } @@ -1324,9 +1325,9 @@ Status VSchemaChangeDirectly::_inner_process(RowsetReaderSharedPtr rowset_reader TabletSharedPtr new_tablet, TabletSharedPtr base_tablet) { auto new_block = - std::make_unique(new_tablet->tablet_schema().create_block()); + std::make_unique(new_tablet->tablet_schema()->create_block()); auto ref_block = - std::make_unique(base_tablet->tablet_schema().create_block()); + std::make_unique(base_tablet->tablet_schema()->create_block()); int origin_columns_size = ref_block->columns(); @@ -1374,7 +1375,7 @@ Status SchemaChangeWithSorting::_inner_process(RowsetReaderSharedPtr rowset_read TabletSharedPtr base_tablet) { if (_row_block_allocator == nullptr) { _row_block_allocator = - new (nothrow) RowBlockAllocator(new_tablet->tablet_schema(), _memory_limitation); + new (nothrow) RowBlockAllocator(*new_tablet->tablet_schema(), _memory_limitation); if (_row_block_allocator == nullptr) { LOG(FATAL) << "failed to malloc RowBlockAllocator. size=" << sizeof(RowBlockAllocator); return Status::OLAPInternalError(OLAP_ERR_INPUT_PARAMETER_ERROR); @@ -1554,9 +1555,9 @@ Status VSchemaChangeWithSorting::_inner_process(RowsetReaderSharedPtr rowset_rea _temp_delta_versions.first = _temp_delta_versions.second; auto new_block = - std::make_unique(new_tablet->tablet_schema().create_block()); + std::make_unique(new_tablet->tablet_schema()->create_block()); auto ref_block = - std::make_unique(base_tablet->tablet_schema().create_block()); + std::make_unique(base_tablet->tablet_schema()->create_block()); int origin_columns_size = ref_block->columns(); @@ -1601,7 +1602,7 @@ Status VSchemaChangeWithSorting::_inner_process(RowsetReaderSharedPtr rowset_rea // move unique ptr blocks.push_back( - std::make_unique(new_tablet->tablet_schema().create_block())); + std::make_unique(new_tablet->tablet_schema()->create_block())); swap(blocks.back(), new_block); ref_block->clear_column_data(origin_columns_size); @@ -1631,7 +1632,7 @@ bool SchemaChangeWithSorting::_internal_sorting( std::unique_ptr rowset_writer; if (!new_tablet->create_rowset_writer(version, VISIBLE, segments_overlap, - &new_tablet->tablet_schema(), oldest_write_timestamp, + new_tablet->tablet_schema(), oldest_write_timestamp, newest_write_timestamp, &rowset_writer)) { return false; } @@ -1658,8 +1659,8 @@ Status VSchemaChangeWithSorting::_internal_sorting( std::unique_ptr rowset_writer; RETURN_IF_ERROR(new_tablet->create_rowset_writer( - version, VISIBLE, segments_overlap, &new_tablet->tablet_schema(), - oldest_write_timestamp, newest_write_timestamp, &rowset_writer)); + version, VISIBLE, segments_overlap, new_tablet->tablet_schema(), oldest_write_timestamp, + newest_write_timestamp, &rowset_writer)); Defer defer {[&]() { new_tablet->data_dir()->remove_pending_ids(ROWSET_ID_PREFIX + @@ -1688,9 +1689,10 @@ bool SchemaChangeWithSorting::_external_sorting(vector& src_row } // get cur schema if rowset schema exist, rowset schema must be newer than tablet schema auto max_version_rowset = src_rowsets.back(); - const TabletSchema* cur_tablet_schema = max_version_rowset->rowset_meta()->tablet_schema(); + const TabletSchema* cur_tablet_schema = + max_version_rowset->rowset_meta()->tablet_schema().get(); if (cur_tablet_schema == nullptr) { - cur_tablet_schema = &(new_tablet->tablet_schema()); + cur_tablet_schema = new_tablet->tablet_schema().get(); } Merger::Statistics stats; @@ -1719,8 +1721,8 @@ Status VSchemaChangeWithSorting::_external_sorting(vector& src_ Merger::Statistics stats; RETURN_IF_ERROR(Merger::vmerge_rowsets(new_tablet, READER_ALTER_TABLE, - &new_tablet->tablet_schema(), rs_readers, rowset_writer, - &stats)); + new_tablet->tablet_schema().get(), rs_readers, + rowset_writer, &stats)); _add_merged_rows(stats.merged_rows); _add_filtered_rows(stats.filtered_rows); @@ -1807,7 +1809,8 @@ Status SchemaChangeHandler::_do_process_alter_tablet_v2(const TAlterTabletReqV2& // delete handlers for new tablet DeleteHandler delete_handler; std::vector return_columns; - auto base_tablet_schema = base_tablet->tablet_schema(); + TabletSchema base_tablet_schema; + base_tablet_schema.copy_from(*base_tablet->tablet_schema()); if (!request.columns.empty() && request.columns[0].col_unique_id >= 0) { base_tablet_schema.clear_columns(); for (const auto& column : request.columns) { @@ -2036,12 +2039,9 @@ bool SchemaChangeHandler::tablet_in_converting(int64_t tablet_id) { return _tablet_ids_in_converting.find(tablet_id) != _tablet_ids_in_converting.end(); } -Status SchemaChangeHandler::schema_version_convert(TabletSharedPtr base_tablet, - TabletSharedPtr new_tablet, - RowsetSharedPtr* base_rowset, - RowsetSharedPtr* new_rowset, - DescriptorTbl desc_tbl, - const TabletSchema* base_schema_change) { +Status SchemaChangeHandler::schema_version_convert( + TabletSharedPtr base_tablet, TabletSharedPtr new_tablet, RowsetSharedPtr* base_rowset, + RowsetSharedPtr* new_rowset, DescriptorTbl desc_tbl, TabletSchemaSPtr base_schema_change) { Status res = Status::OK(); LOG(INFO) << "begin to convert delta version for schema changing. " << "base_tablet=" << base_tablet->full_name() @@ -2049,13 +2049,13 @@ Status SchemaChangeHandler::schema_version_convert(TabletSharedPtr base_tablet, // a. Parse the Alter request and convert it into an internal representation // Do not use the delete condition specified by the DELETE_DATA command - RowBlockChanger rb_changer(new_tablet->tablet_schema(), desc_tbl); + RowBlockChanger rb_changer(*new_tablet->tablet_schema(), desc_tbl); bool sc_sorting = false; bool sc_directly = false; const std::unordered_map materialized_function_map; if (res = _parse_request(base_tablet, new_tablet, &rb_changer, &sc_sorting, &sc_directly, - materialized_function_map, desc_tbl, base_schema_change); + materialized_function_map, desc_tbl, base_schema_change.get()); !res) { LOG(WARNING) << "failed to parse the request. res=" << res; return res; @@ -2077,7 +2077,7 @@ Status SchemaChangeHandler::schema_version_convert(TabletSharedPtr base_tablet, RowsetReaderContext reader_context; reader_context.reader_type = READER_ALTER_TABLE; - reader_context.tablet_schema = base_schema_change; + reader_context.tablet_schema = base_schema_change.get(); reader_context.need_ordered_result = true; reader_context.delete_handler = &delete_handler; reader_context.return_columns = &return_columns; @@ -2093,10 +2093,9 @@ Status SchemaChangeHandler::schema_version_convert(TabletSharedPtr base_tablet, load_id.set_hi((*base_rowset)->load_id().hi()); load_id.set_lo((*base_rowset)->load_id().lo()); std::unique_ptr rowset_writer; - RETURN_NOT_OK( - new_tablet->create_rowset_writer((*base_rowset)->txn_id(), load_id, PREPARED, - (*base_rowset)->rowset_meta()->segments_overlap(), - reader_context.tablet_schema, &rowset_writer)); + RETURN_NOT_OK(new_tablet->create_rowset_writer( + (*base_rowset)->txn_id(), load_id, PREPARED, + (*base_rowset)->rowset_meta()->segments_overlap(), base_schema_change, &rowset_writer)); auto schema_version_convert_error = [&]() -> Status { if (*new_rowset != nullptr) { @@ -2170,7 +2169,7 @@ Status SchemaChangeHandler::_convert_historical_rowsets(const SchemaChangeParams // Add filter information in change, and filter column information will be set in _parse_request // And filter some data every time the row block changes - RowBlockChanger rb_changer(sc_params.new_tablet->tablet_schema(), sc_params.delete_handler, + RowBlockChanger rb_changer(*sc_params.new_tablet->tablet_schema(), sc_params.delete_handler, *sc_params.desc_tbl); bool sc_sorting = false; @@ -2220,9 +2219,9 @@ Status SchemaChangeHandler::_convert_historical_rowsets(const SchemaChangeParams std::unique_ptr rowset_writer; Status status = new_tablet->create_rowset_writer( rs_reader->version(), VISIBLE, - rs_reader->rowset()->rowset_meta()->segments_overlap(), - &new_tablet->tablet_schema(), rs_reader->oldest_write_timestamp(), - rs_reader->newest_write_timestamp(), &rowset_writer); + rs_reader->rowset()->rowset_meta()->segments_overlap(), new_tablet->tablet_schema(), + rs_reader->oldest_write_timestamp(), rs_reader->newest_write_timestamp(), + &rowset_writer); if (!status.ok()) { res = Status::OLAPInternalError(OLAP_ERR_ROWSET_BUILDER_INIT); return process_alter_exit(); @@ -2286,9 +2285,9 @@ Status SchemaChangeHandler::_parse_request( materialized_function_map, DescriptorTbl desc_tbl, const TabletSchema* base_tablet_schema) { // set column mapping - for (int i = 0, new_schema_size = new_tablet->tablet_schema().num_columns(); + for (int i = 0, new_schema_size = new_tablet->tablet_schema()->num_columns(); i < new_schema_size; ++i) { - const TabletColumn& new_column = new_tablet->tablet_schema().column(i); + const TabletColumn& new_column = new_tablet->tablet_schema()->column(i); const string& column_name = new_column.name(); ColumnMapping* column_mapping = rb_changer->get_mutable_column_mapping(i); @@ -2363,8 +2362,8 @@ Status SchemaChangeHandler::_parse_request( } const TabletSchema& ref_tablet_schema = *base_tablet_schema; - const TabletSchema& new_tablet_schema = new_tablet->tablet_schema(); - if (ref_tablet_schema.keys_type() != new_tablet_schema.keys_type()) { + TabletSchemaSPtr new_tablet_schema = new_tablet->tablet_schema(); + if (ref_tablet_schema.keys_type() != new_tablet_schema->keys_type()) { // only when base table is dup and mv is agg // the rollup job must be reagg. *sc_sorting = true; @@ -2378,7 +2377,7 @@ Status SchemaChangeHandler::_parse_request( // followings need resort: // old keys: A B C D // new keys: A B - if (new_tablet_schema.keys_type() != KeysType::DUP_KEYS && + if (new_tablet_schema->keys_type() != KeysType::DUP_KEYS && new_tablet->num_key_columns() < base_tablet_schema->num_key_columns()) { // this is a table with aggregate key type, and num of key columns in new schema // is less, which means the data in new tablet should be more aggregated. @@ -2398,7 +2397,7 @@ Status SchemaChangeHandler::_parse_request( if (column_mapping->ref_column < 0) { continue; } else { - auto column_new = new_tablet_schema.column(i); + auto column_new = new_tablet_schema->column(i); auto column_old = ref_tablet_schema.column(column_mapping->ref_column); if (column_new.type() != column_old.type() || column_new.precision() != column_old.precision() || diff --git a/be/src/olap/schema_change.h b/be/src/olap/schema_change.h index 81a6e48791930c..4ef608fb4b6af1 100644 --- a/be/src/olap/schema_change.h +++ b/be/src/olap/schema_change.h @@ -252,7 +252,7 @@ class SchemaChangeHandler { static Status schema_version_convert(TabletSharedPtr base_tablet, TabletSharedPtr new_tablet, RowsetSharedPtr* base_rowset, RowsetSharedPtr* new_rowset, DescriptorTbl desc_tbl, - const TabletSchema* base_schema_change); + TabletSchemaSPtr base_schema_change); // schema change v2, it will not set alter task in base tablet static Status process_alter_tablet_v2(const TAlterTabletReqV2& request); diff --git a/be/src/olap/snapshot_manager.cpp b/be/src/olap/snapshot_manager.cpp index 3d0e885b20c573..1aabfa8d9b2a4c 100644 --- a/be/src/olap/snapshot_manager.cpp +++ b/be/src/olap/snapshot_manager.cpp @@ -36,6 +36,7 @@ #include "olap/rowset/rowset_writer.h" #include "olap/storage_engine.h" #include "olap/tablet_meta.h" +#include "olap/tablet_schema.h" #include "runtime/thread_context.h" using std::filesystem::path; @@ -148,8 +149,9 @@ Status SnapshotManager::convert_rowset_ids(const std::string& clone_dir, int64_t new_tablet_meta_pb.set_tablet_id(tablet_id); new_tablet_meta_pb.set_replica_id(replica_id); new_tablet_meta_pb.set_schema_hash(schema_hash); - TabletSchema tablet_schema; - tablet_schema.init_from_pb(new_tablet_meta_pb.schema()); + TabletSchemaSPtr tablet_schema; + tablet_schema = + TabletSchemaCache::instance()->insert(new_tablet_meta_pb.schema().SerializeAsString()); std::unordered_map rs_version_map; std::unordered_map rowset_id_mapping; @@ -234,14 +236,14 @@ Status SnapshotManager::convert_rowset_ids(const std::string& clone_dir, int64_t Status SnapshotManager::_rename_rowset_id(const RowsetMetaPB& rs_meta_pb, const std::string& new_tablet_path, - TabletSchema& tablet_schema, const RowsetId& rowset_id, + TabletSchemaSPtr tablet_schema, const RowsetId& rowset_id, RowsetMetaPB* new_rs_meta_pb) { Status res = Status::OK(); RowsetMetaSharedPtr rowset_meta(new RowsetMeta()); rowset_meta->init_from_pb(rs_meta_pb); RowsetSharedPtr org_rowset; - RETURN_NOT_OK(RowsetFactory::create_rowset(&tablet_schema, new_tablet_path, rowset_meta, - &org_rowset)); + RETURN_NOT_OK( + RowsetFactory::create_rowset(tablet_schema, new_tablet_path, rowset_meta, &org_rowset)); // do not use cache to load index // because the index file may conflict // and the cached fd may be invalid @@ -255,7 +257,7 @@ Status SnapshotManager::_rename_rowset_id(const RowsetMetaPB& rs_meta_pb, context.rowset_type = org_rowset_meta->rowset_type(); context.tablet_path = new_tablet_path; context.tablet_schema = - org_rowset_meta->tablet_schema() ? org_rowset_meta->tablet_schema() : &tablet_schema; + org_rowset_meta->tablet_schema() ? org_rowset_meta->tablet_schema() : tablet_schema; context.rowset_state = org_rowset_meta->rowset_state(); context.version = org_rowset_meta->version(); context.oldest_write_timestamp = org_rowset_meta->oldest_write_timestamp(); diff --git a/be/src/olap/snapshot_manager.h b/be/src/olap/snapshot_manager.h index a82728207a4094..3b00220b85d19b 100644 --- a/be/src/olap/snapshot_manager.h +++ b/be/src/olap/snapshot_manager.h @@ -84,7 +84,7 @@ class SnapshotManager { Status _prepare_snapshot_dir(const TabletSharedPtr& ref_tablet, std::string* snapshot_id_path); Status _rename_rowset_id(const RowsetMetaPB& rs_meta_pb, const std::string& new_tablet_path, - TabletSchema& tablet_schema, const RowsetId& next_id, + TabletSchemaSPtr tablet_schema, const RowsetId& next_id, RowsetMetaPB* new_rs_meta_pb); private: diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp index 75e64f875ee593..2ecbd4d3cd5e0f 100644 --- a/be/src/olap/tablet.cpp +++ b/be/src/olap/tablet.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -40,6 +41,7 @@ #include "io/fs/path.h" #include "io/fs/remote_file_system.h" #include "olap/base_compaction.h" +#include "olap/base_tablet.h" #include "olap/cumulative_compaction.h" #include "olap/olap_common.h" #include "olap/olap_define.h" @@ -53,6 +55,7 @@ #include "olap/storage_policy_mgr.h" #include "olap/tablet_meta.h" #include "olap/tablet_meta_manager.h" +#include "olap/tablet_schema.h" #include "segment_loader.h" #include "util/path_util.h" #include "util/pretty_printer.h" @@ -117,7 +120,7 @@ Status Tablet::_init_once_action() { for (const auto& rs_meta : _tablet_meta->all_rs_metas()) { Version version = rs_meta->version(); RowsetSharedPtr rowset; - res = RowsetFactory::create_rowset(&_schema, _tablet_path, rs_meta, &rowset); + res = RowsetFactory::create_rowset(_schema, _tablet_path, rs_meta, &rowset); if (!res.ok()) { LOG(WARNING) << "fail to init rowset. tablet_id=" << tablet_id() << ", schema_hash=" << schema_hash() << ", version=" << version @@ -132,7 +135,7 @@ Status Tablet::_init_once_action() { for (auto& stale_rs_meta : _tablet_meta->all_stale_rs_metas()) { Version version = stale_rs_meta->version(); RowsetSharedPtr rowset; - res = RowsetFactory::create_rowset(&_schema, _tablet_path, stale_rs_meta, &rowset); + res = RowsetFactory::create_rowset(_schema, _tablet_path, stale_rs_meta, &rowset); if (!res.ok()) { LOG(WARNING) << "fail to init stale rowset. tablet_id:" << tablet_id() << ", schema_hash:" << schema_hash() << ", version=" << version @@ -142,7 +145,7 @@ Status Tablet::_init_once_action() { _stale_rs_version_map[version] = std::move(rowset); } - if (_schema.keys_type() == UNIQUE_KEYS && enable_unique_key_merge_on_write()) { + if (_schema->keys_type() == UNIQUE_KEYS && enable_unique_key_merge_on_write()) { _rowset_tree = std::make_unique(); res = _rowset_tree->Init(rowset_vec); } @@ -206,7 +209,7 @@ Status Tablet::revise_tablet_meta(const std::vector& rowset for (auto& rs_meta : rowsets_to_clone) { Version version = {rs_meta->start_version(), rs_meta->end_version()}; RowsetSharedPtr rowset; - res = RowsetFactory::create_rowset(&_schema, _tablet_path, rs_meta, &rowset); + res = RowsetFactory::create_rowset(_schema, _tablet_path, rs_meta, &rowset); if (!res.ok()) { LOG(WARNING) << "fail to init rowset. version=" << version; return res; @@ -400,7 +403,7 @@ const RowsetSharedPtr Tablet::rowset_with_max_version() const { return iter->second; } -const RowsetMetaSharedPtr Tablet::rowset_meta_with_max_schema_version( +RowsetMetaSharedPtr Tablet::rowset_meta_with_max_schema_version( const std::vector& rowset_metas) { return *std::max_element(rowset_metas.begin(), rowset_metas.end(), [](const RowsetMetaSharedPtr& a, const RowsetMetaSharedPtr& b) { @@ -970,7 +973,7 @@ Status Tablet::split_range(const OlapTuple& start_key_strings, const OlapTuple& RowCursor start_key; // 如果有startkey，用startkey初始化；反之则用minkey初始化 if (start_key_strings.size() > 0) { - if (start_key.init_scan_key(_schema, start_key_strings.values()) != Status::OK()) { + if (start_key.init_scan_key(*_schema, start_key_strings.values()) != Status::OK()) { LOG(WARNING) << "fail to initial key strings with RowCursor type."; return Status::OLAPInternalError(OLAP_ERR_INIT_FAILED); } @@ -981,12 +984,12 @@ Status Tablet::split_range(const OlapTuple& start_key_strings, const OlapTuple& } key_num = start_key_strings.size(); } else { - if (start_key.init(_schema, num_short_key_columns()) != Status::OK()) { + if (start_key.init(*_schema, num_short_key_columns()) != Status::OK()) { LOG(WARNING) << "fail to initial key strings with RowCursor type."; return Status::OLAPInternalError(OLAP_ERR_INIT_FAILED); } - start_key.allocate_memory_for_string_type(_schema); + start_key.allocate_memory_for_string_type(*_schema); start_key.build_min_key(); key_num = num_short_key_columns(); } @@ -994,7 +997,7 @@ Status Tablet::split_range(const OlapTuple& start_key_strings, const OlapTuple& RowCursor end_key; // 和startkey一样处理，没有则用maxkey初始化 if (end_key_strings.size() > 0) { - if (!end_key.init_scan_key(_schema, end_key_strings.values())) { + if (!end_key.init_scan_key(*_schema, end_key_strings.values())) { LOG(WARNING) << "fail to parse strings to key with RowCursor type."; return Status::OLAPInternalError(OLAP_ERR_INVALID_SCHEMA); } @@ -1004,12 +1007,12 @@ Status Tablet::split_range(const OlapTuple& start_key_strings, const OlapTuple& return Status::OLAPInternalError(OLAP_ERR_INVALID_SCHEMA); } } else { - if (end_key.init(_schema, num_short_key_columns()) != Status::OK()) { + if (end_key.init(*_schema, num_short_key_columns()) != Status::OK()) { LOG(WARNING) << "fail to initial key strings with RowCursor type."; return Status::OLAPInternalError(OLAP_ERR_INIT_FAILED); } - end_key.allocate_memory_for_string_type(_schema); + end_key.allocate_memory_for_string_type(*_schema); end_key.build_max_key(); } @@ -1597,8 +1600,8 @@ Status Tablet::create_initial_rowset(const int64_t req_version) { do { // there is no data in init rowset, so overlapping info is unknown. std::unique_ptr rs_writer; - res = create_rowset_writer(version, VISIBLE, OVERLAP_UNKNOWN, - &_tablet_meta->tablet_schema(), -1, -1, &rs_writer); + res = create_rowset_writer(version, VISIBLE, OVERLAP_UNKNOWN, tablet_schema(), -1, -1, + &rs_writer); if (!res.ok()) { LOG(WARNING) << "failed to init rowset writer for tablet " << full_name(); @@ -1630,8 +1633,8 @@ Status Tablet::create_initial_rowset(const int64_t req_version) { Status Tablet::create_rowset_writer(const Version& version, const RowsetStatePB& rowset_state, const SegmentsOverlapPB& overlap, - const doris::TabletSchema* tablet_schema, - int64_t oldest_write_timestamp, int64_t newest_write_timestamp, + TabletSchemaSPtr tablet_schema, int64_t oldest_write_timestamp, + int64_t newest_write_timestamp, std::unique_ptr* rowset_writer) { RowsetWriterContext context; context.version = version; @@ -1648,7 +1651,7 @@ Status Tablet::create_rowset_writer(const Version& version, const RowsetStatePB& Status Tablet::create_rowset_writer(const int64_t& txn_id, const PUniqueId& load_id, const RowsetStatePB& rowset_state, const SegmentsOverlapPB& overlap, - const doris::TabletSchema* tablet_schema, + TabletSchemaSPtr tablet_schema, std::unique_ptr* rowset_writer) { RowsetWriterContext context; context.txn_id = txn_id; @@ -1681,7 +1684,7 @@ void Tablet::_init_context_common_fields(RowsetWriterContext& context) { } Status Tablet::create_rowset(RowsetMetaSharedPtr rowset_meta, RowsetSharedPtr* rowset) { - return RowsetFactory::create_rowset(&tablet_schema(), tablet_path(), rowset_meta, rowset); + return RowsetFactory::create_rowset(tablet_schema(), tablet_path(), rowset_meta, rowset); } Status Tablet::cooldown() { @@ -1736,7 +1739,7 @@ Status Tablet::cooldown() { new_rowset_meta->set_fs(dest_fs); new_rowset_meta->set_creation_time(time(nullptr)); RowsetSharedPtr new_rowset; - RowsetFactory::create_rowset(&_schema, _tablet_path, new_rowset_meta, &new_rowset); + RowsetFactory::create_rowset(_schema, _tablet_path, new_rowset_meta, &new_rowset); std::vector to_add {std::move(new_rowset)}; std::vector to_delete {std::move(old_rowset)}; @@ -1859,11 +1862,14 @@ Status Tablet::remove_all_remote_rowsets() { return _data_dir->get_meta()->put(META_COLUMN_FAMILY_INDEX, tablet_gc_key, storage_policy()); } -const TabletSchema& Tablet::tablet_schema() const { +TabletSchemaSPtr Tablet::tablet_schema() const { std::shared_lock wrlock(_meta_lock); + if (UNLIKELY(_tablet_meta->all_rs_metas().empty())) { + return BaseTablet::tablet_schema(); + } const RowsetMetaSharedPtr rowset_meta = rowset_meta_with_max_schema_version(_tablet_meta->all_rs_metas()); - return *rowset_meta->tablet_schema(); + return rowset_meta->tablet_schema(); } Status Tablet::lookup_row_key(const Slice& encoded_key, RowLocation* row_location, diff --git a/be/src/olap/tablet.h b/be/src/olap/tablet.h index adacb11eb2f279..422c1d9cde7408 100644 --- a/be/src/olap/tablet.h +++ b/be/src/olap/tablet.h @@ -125,7 +125,7 @@ class Tablet : public BaseTablet { const RowsetSharedPtr rowset_with_max_version() const; - static const RowsetMetaSharedPtr rowset_meta_with_max_schema_version( + static RowsetMetaSharedPtr rowset_meta_with_max_schema_version( const std::vector& rowset_metas); Status add_inc_rowset(const RowsetSharedPtr& rowset); @@ -282,16 +282,16 @@ class Tablet : public BaseTablet { return _tablet_meta->all_beta(); } - const TabletSchema& tablet_schema() const override; + TabletSchemaSPtr tablet_schema() const override; Status create_rowset_writer(const Version& version, const RowsetStatePB& rowset_state, - const SegmentsOverlapPB& overlap, const TabletSchema* tablet_schema, + const SegmentsOverlapPB& overlap, TabletSchemaSPtr tablet_schema, int64_t oldest_write_timestamp, int64_t newest_write_timestamp, std::unique_ptr* rowset_writer); Status create_rowset_writer(const int64_t& txn_id, const PUniqueId& load_id, const RowsetStatePB& rowset_state, const SegmentsOverlapPB& overlap, - const TabletSchema* tablet_schema, + TabletSchemaSPtr tablet_schema, std::unique_ptr* rowset_writer); Status create_rowset(RowsetMetaSharedPtr rowset_meta, RowsetSharedPtr* rowset); @@ -494,55 +494,55 @@ inline Version Tablet::max_version() const { } inline KeysType Tablet::keys_type() const { - return _schema.keys_type(); + return _schema->keys_type(); } inline SortType Tablet::sort_type() const { - return _schema.sort_type(); + return _schema->sort_type(); } inline size_t Tablet::sort_col_num() const { - return _schema.sort_col_num(); + return _schema->sort_col_num(); } inline size_t Tablet::num_columns() const { - return _schema.num_columns(); + return _schema->num_columns(); } inline size_t Tablet::num_null_columns() const { - return _schema.num_null_columns(); + return _schema->num_null_columns(); } inline size_t Tablet::num_key_columns() const { - return _schema.num_key_columns(); + return _schema->num_key_columns(); } inline size_t Tablet::num_short_key_columns() const { - return _schema.num_short_key_columns(); + return _schema->num_short_key_columns(); } inline size_t Tablet::num_rows_per_row_block() const { - return _schema.num_rows_per_row_block(); + return _schema->num_rows_per_row_block(); } inline CompressKind Tablet::compress_kind() const { - return _schema.compress_kind(); + return _schema->compress_kind(); } inline double Tablet::bloom_filter_fpp() const { - return _schema.bloom_filter_fpp(); + return _schema->bloom_filter_fpp(); } inline size_t Tablet::next_unique_id() const { - return _schema.next_column_unique_id(); + return _schema->next_column_unique_id(); } inline int32_t Tablet::field_index(const std::string& field_name) const { - return _schema.field_index(field_name); + return _schema->field_index(field_name); } inline size_t Tablet::row_size() const { - return _schema.row_size(); + return _schema->row_size(); } } // namespace doris diff --git a/be/src/olap/tablet_manager.cpp b/be/src/olap/tablet_manager.cpp index f5c2ebdbcf92f0..cb7fe6c1bd722b 100644 --- a/be/src/olap/tablet_manager.cpp +++ b/be/src/olap/tablet_manager.cpp @@ -1134,7 +1134,7 @@ Status TabletManager::_create_tablet_meta_unlocked(const TCreateTabletReq& reque int32_t old_col_idx = base_tablet->field_index(column.column_name); if (old_col_idx != -1) { uint32_t old_unique_id = - base_tablet->tablet_schema().column(old_col_idx).unique_id(); + base_tablet->tablet_schema()->column(old_col_idx).unique_id(); col_idx_to_unique_id[new_col_idx] = old_unique_id; } else { // Not exist in old tablet, it is a new added column diff --git a/be/src/olap/tablet_schema.cpp b/be/src/olap/tablet_schema.cpp index cbeca511929925..fae5865f876218 100644 --- a/be/src/olap/tablet_schema.cpp +++ b/be/src/olap/tablet_schema.cpp @@ -17,6 +17,8 @@ #include "olap/tablet_schema.h" +#include + #include "gen_cpp/descriptors.pb.h" #include "tablet_meta.h" #include "vec/aggregate_functions/aggregate_function_reader.h" @@ -529,6 +531,18 @@ void TabletSchema::init_from_pb(const TabletSchemaPB& schema) { _schema_version = schema.schema_version(); } +void TabletSchema::copy_from(const TabletSchema& tablet_schema) { + TabletSchemaPB tablet_schema_pb; + tablet_schema.to_schema_pb(&tablet_schema_pb); + init_from_pb(tablet_schema_pb); +} + +std::string TabletSchema::to_key() const { + TabletSchemaPB pb; + to_schema_pb(&pb); + return pb.SerializeAsString(); +} + void TabletSchema::build_current_tablet_schema(int64_t index_id, const POlapTableSchemaParam& ptable_schema_param, const TabletSchema& ori_tablet_schema) { diff --git a/be/src/olap/tablet_schema.h b/be/src/olap/tablet_schema.h index 4e18012522141c..5789d9f60ff8ac 100644 --- a/be/src/olap/tablet_schema.h +++ b/be/src/olap/tablet_schema.h @@ -133,6 +133,8 @@ class TabletSchema { void init_from_pb(const TabletSchemaPB& schema); void to_schema_pb(TabletSchemaPB* tablet_meta_pb) const; void append_column(TabletColumn column); + void copy_from(const TabletSchema& tablet_schema); + std::string to_key() const; uint32_t mem_size() const; size_t row_size() const; @@ -205,4 +207,6 @@ class TabletSchema { bool operator==(const TabletSchema& a, const TabletSchema& b); bool operator!=(const TabletSchema& a, const TabletSchema& b); +using TabletSchemaSPtr = std::shared_ptr; + } // namespace doris diff --git a/be/src/olap/tablet_schema_cache.h b/be/src/olap/tablet_schema_cache.h new file mode 100644 index 00000000000000..5e530bf08c4eb7 --- /dev/null +++ b/be/src/olap/tablet_schema_cache.h @@ -0,0 +1,61 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include +#include +#include + +#include "olap/tablet_schema.h" + +namespace doris { + +class TabletSchemaCache { +public: + static void create_global_schema_cache() { + DCHECK(_s_instance == nullptr); + static TabletSchemaCache instance; + _s_instance = &instance; + } + + static TabletSchemaCache* instance() { return _s_instance; } + + TabletSchemaSPtr insert(const std::string& key) { + DCHECK(_s_instance != nullptr); + std::lock_guard guard(_mtx); + auto iter = _cache.find(key); + if (iter == _cache.end()) { + TabletSchemaSPtr tablet_schema_ptr = std::make_shared(); + TabletSchemaPB pb; + pb.ParseFromString(key); + tablet_schema_ptr->init_from_pb(pb); + _cache[key] = tablet_schema_ptr; + return tablet_schema_ptr; + } + return iter->second; + } + +private: + static inline TabletSchemaCache* _s_instance = nullptr; + std::mutex _mtx; + std::unordered_map _cache; +}; + +} // namespace doris \ No newline at end of file diff --git a/be/src/olap/task/engine_checksum_task.cpp b/be/src/olap/task/engine_checksum_task.cpp index f5d5f317f1b600..30e8d7a364de98 100644 --- a/be/src/olap/task/engine_checksum_task.cpp +++ b/be/src/olap/task/engine_checksum_task.cpp @@ -77,7 +77,7 @@ Status EngineChecksumTask::_compute_checksum() { } } - for (size_t i = 0; i < tablet->tablet_schema().num_columns(); ++i) { + for (size_t i = 0; i < tablet->tablet_schema()->num_columns(); ++i) { reader_params.return_columns.push_back(i); } @@ -90,12 +90,12 @@ Status EngineChecksumTask::_compute_checksum() { RowCursor row; std::unique_ptr mem_pool(new MemPool()); std::unique_ptr agg_object_pool(new ObjectPool()); - res = row.init(tablet->tablet_schema(), reader_params.return_columns); + res = row.init(*tablet->tablet_schema(), reader_params.return_columns); if (!res.ok()) { LOG(WARNING) << "failed to init row cursor. res = " << res; return res; } - row.allocate_memory_for_string_type(tablet->tablet_schema()); + row.allocate_memory_for_string_type(*tablet->tablet_schema()); bool eof = false; uint32_t row_checksum = 0; diff --git a/be/src/olap/task/engine_clone_task.cpp b/be/src/olap/task/engine_clone_task.cpp index 73466cb72e40f9..66300beb2e5b1c 100644 --- a/be/src/olap/task/engine_clone_task.cpp +++ b/be/src/olap/task/engine_clone_task.cpp @@ -766,9 +766,8 @@ Status EngineCloneTask::_finish_full_clone(Tablet* tablet, TabletMeta* cloned_ta // but some rowset is useless, so that remove them here for (auto& rs_meta_ptr : rs_metas_found_in_src) { RowsetSharedPtr rowset_to_remove; - auto s = - RowsetFactory::create_rowset(&(cloned_tablet_meta->tablet_schema()), - tablet->tablet_path(), rs_meta_ptr, &rowset_to_remove); + auto s = RowsetFactory::create_rowset(tablet->tablet_schema(), tablet->tablet_path(), + rs_meta_ptr, &rowset_to_remove); if (!s.ok()) { LOG(WARNING) << "failed to init rowset to remove: " << rs_meta_ptr->rowset_id().to_string(); diff --git a/be/src/service/doris_main.cpp b/be/src/service/doris_main.cpp index d62230a1651036..a87c17eef79c37 100644 --- a/be/src/service/doris_main.cpp +++ b/be/src/service/doris_main.cpp @@ -379,6 +379,7 @@ int main(int argc, char** argv) { // init exec env auto exec_env = doris::ExecEnv::GetInstance(); doris::ExecEnv::init(exec_env, paths); + doris::TabletSchemaCache::create_global_schema_cache(); // init and open storage engine doris::EngineOptions options; diff --git a/be/src/vec/exec/volap_scanner.cpp b/be/src/vec/exec/volap_scanner.cpp index 30cfa8861cb367..02fae4de8de274 100644 --- a/be/src/vec/exec/volap_scanner.cpp +++ b/be/src/vec/exec/volap_scanner.cpp @@ -68,7 +68,7 @@ Status VOlapScanner::prepare( LOG(WARNING) << ss.str(); return Status::InternalError(ss.str()); } - _tablet_schema = _tablet->tablet_schema(); + _tablet_schema.copy_from(*_tablet->tablet_schema()); if (!_parent->_olap_scan_node.columns_desc.empty() && _parent->_olap_scan_node.columns_desc[0].col_unique_id >= 0) { // Originally scanner get TabletSchema from tablet object in BE. diff --git a/be/test/olap/delete_handler_test.cpp b/be/test/olap/delete_handler_test.cpp index 7ba23ca2c18f9c..11571331ea9c99 100644 --- a/be/test/olap/delete_handler_test.cpp +++ b/be/test/olap/delete_handler_test.cpp @@ -339,7 +339,7 @@ TEST_F(TestDeleteConditionHandler, StoreCondSucceed) { conditions.push_back(condition); DeletePredicatePB del_pred; - success_res = DeleteHandler::generate_delete_predicate(tablet->tablet_schema(), conditions, + success_res = DeleteHandler::generate_delete_predicate(*tablet->tablet_schema(), conditions, &del_pred); EXPECT_EQ(Status::OK(), success_res); @@ -363,7 +363,7 @@ TEST_F(TestDeleteConditionHandler, StoreCondInvalidParameters) { // 空的过滤条件 std::vector conditions; DeletePredicatePB del_pred; - Status failed_res = DeleteHandler::generate_delete_predicate(tablet->tablet_schema(), + Status failed_res = DeleteHandler::generate_delete_predicate(*tablet->tablet_schema(), conditions, &del_pred); ; EXPECT_EQ(Status::OLAPInternalError(OLAP_ERR_DELETE_INVALID_PARAMETERS), failed_res); @@ -380,7 +380,7 @@ TEST_F(TestDeleteConditionHandler, StoreCondNonexistentColumn) { condition.condition_values.push_back("2"); conditions.push_back(condition); DeletePredicatePB del_pred; - Status failed_res = DeleteHandler::generate_delete_predicate(tablet->tablet_schema(), + Status failed_res = DeleteHandler::generate_delete_predicate(*tablet->tablet_schema(), conditions, &del_pred); ; EXPECT_EQ(Status::OLAPInternalError(OLAP_ERR_DELETE_INVALID_CONDITION), failed_res); @@ -393,7 +393,7 @@ TEST_F(TestDeleteConditionHandler, StoreCondNonexistentColumn) { condition.condition_values.push_back("5"); conditions.push_back(condition); - failed_res = DeleteHandler::generate_delete_predicate(tablet->tablet_schema(), conditions, + failed_res = DeleteHandler::generate_delete_predicate(*tablet->tablet_schema(), conditions, &del_pred); ; EXPECT_EQ(Status::OLAPInternalError(OLAP_ERR_DELETE_INVALID_CONDITION), failed_res); @@ -406,7 +406,7 @@ TEST_F(TestDeleteConditionHandler, StoreCondNonexistentColumn) { condition.condition_values.push_back("5"); conditions.push_back(condition); - Status success_res = DeleteHandler::generate_delete_predicate(dup_tablet->tablet_schema(), + Status success_res = DeleteHandler::generate_delete_predicate(*dup_tablet->tablet_schema(), conditions, &del_pred); ; EXPECT_EQ(Status::OK(), success_res); @@ -484,7 +484,7 @@ TEST_F(TestDeleteConditionHandler2, ValidConditionValue) { conditions.push_back(condition); DeletePredicatePB del_pred; - res = DeleteHandler::generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred); + res = DeleteHandler::generate_delete_predicate(*tablet->tablet_schema(), conditions, &del_pred); EXPECT_EQ(Status::OK(), res); // k5类型为int128 @@ -496,7 +496,7 @@ TEST_F(TestDeleteConditionHandler2, ValidConditionValue) { conditions.push_back(condition); DeletePredicatePB del_pred_2; - res = DeleteHandler::generate_delete_predicate(tablet->tablet_schema(), conditions, + res = DeleteHandler::generate_delete_predicate(*tablet->tablet_schema(), conditions, &del_pred_2); EXPECT_EQ(Status::OK(), res); @@ -509,28 +509,28 @@ TEST_F(TestDeleteConditionHandler2, ValidConditionValue) { conditions.push_back(condition); DeletePredicatePB del_pred_3; - res = DeleteHandler::generate_delete_predicate(tablet->tablet_schema(), conditions, + res = DeleteHandler::generate_delete_predicate(*tablet->tablet_schema(), conditions, &del_pred_3); EXPECT_EQ(Status::OK(), res); conditions[0].condition_values.clear(); conditions[0].condition_values.push_back("2"); DeletePredicatePB del_pred_4; - res = DeleteHandler::generate_delete_predicate(tablet->tablet_schema(), conditions, + res = DeleteHandler::generate_delete_predicate(*tablet->tablet_schema(), conditions, &del_pred_4); EXPECT_EQ(Status::OK(), res); conditions[0].condition_values.clear(); conditions[0].condition_values.push_back("-2"); DeletePredicatePB del_pred_5; - res = DeleteHandler::generate_delete_predicate(tablet->tablet_schema(), conditions, + res = DeleteHandler::generate_delete_predicate(*tablet->tablet_schema(), conditions, &del_pred_5); EXPECT_EQ(Status::OK(), res); conditions[0].condition_values.clear(); conditions[0].condition_values.push_back("-2.3"); DeletePredicatePB del_pred_6; - res = DeleteHandler::generate_delete_predicate(tablet->tablet_schema(), conditions, + res = DeleteHandler::generate_delete_predicate(*tablet->tablet_schema(), conditions, &del_pred_6); EXPECT_EQ(Status::OK(), res); @@ -549,7 +549,7 @@ TEST_F(TestDeleteConditionHandler2, ValidConditionValue) { conditions.push_back(condition); DeletePredicatePB del_pred_7; - res = DeleteHandler::generate_delete_predicate(tablet->tablet_schema(), conditions, + res = DeleteHandler::generate_delete_predicate(*tablet->tablet_schema(), conditions, &del_pred_7); EXPECT_EQ(Status::OK(), res); @@ -568,7 +568,7 @@ TEST_F(TestDeleteConditionHandler2, ValidConditionValue) { conditions.push_back(condition); DeletePredicatePB del_pred_8; - res = DeleteHandler::generate_delete_predicate(tablet->tablet_schema(), conditions, + res = DeleteHandler::generate_delete_predicate(*tablet->tablet_schema(), conditions, &del_pred_8); EXPECT_EQ(Status::OK(), res); } @@ -586,7 +586,7 @@ TEST_F(TestDeleteConditionHandler2, InvalidConditionValue) { conditions.push_back(condition); DeletePredicatePB del_pred_1; - res = DeleteHandler::generate_delete_predicate(tablet->tablet_schema(), conditions, + res = DeleteHandler::generate_delete_predicate(*tablet->tablet_schema(), conditions, &del_pred_1); EXPECT_EQ(Status::OLAPInternalError(OLAP_ERR_DELETE_INVALID_CONDITION), res); @@ -594,7 +594,7 @@ TEST_F(TestDeleteConditionHandler2, InvalidConditionValue) { conditions[0].condition_values.clear(); conditions[0].condition_values.push_back("-1000"); DeletePredicatePB del_pred_2; - res = DeleteHandler::generate_delete_predicate(tablet->tablet_schema(), conditions, + res = DeleteHandler::generate_delete_predicate(*tablet->tablet_schema(), conditions, &del_pred_2); EXPECT_EQ(Status::OLAPInternalError(OLAP_ERR_DELETE_INVALID_CONDITION), res); @@ -603,7 +603,7 @@ TEST_F(TestDeleteConditionHandler2, InvalidConditionValue) { conditions[0].column_name = "k2"; conditions[0].condition_values.push_back("32768"); DeletePredicatePB del_pred_3; - res = DeleteHandler::generate_delete_predicate(tablet->tablet_schema(), conditions, + res = DeleteHandler::generate_delete_predicate(*tablet->tablet_schema(), conditions, &del_pred_3); EXPECT_EQ(Status::OLAPInternalError(OLAP_ERR_DELETE_INVALID_CONDITION), res); @@ -611,7 +611,7 @@ TEST_F(TestDeleteConditionHandler2, InvalidConditionValue) { conditions[0].condition_values.clear(); conditions[0].condition_values.push_back("-32769"); DeletePredicatePB del_pred_4; - res = DeleteHandler::generate_delete_predicate(tablet->tablet_schema(), conditions, + res = DeleteHandler::generate_delete_predicate(*tablet->tablet_schema(), conditions, &del_pred_4); EXPECT_EQ(Status::OLAPInternalError(OLAP_ERR_DELETE_INVALID_CONDITION), res); @@ -620,7 +620,7 @@ TEST_F(TestDeleteConditionHandler2, InvalidConditionValue) { conditions[0].column_name = "k3"; conditions[0].condition_values.push_back("2147483648"); DeletePredicatePB del_pred_5; - res = DeleteHandler::generate_delete_predicate(tablet->tablet_schema(), conditions, + res = DeleteHandler::generate_delete_predicate(*tablet->tablet_schema(), conditions, &del_pred_5); EXPECT_EQ(Status::OLAPInternalError(OLAP_ERR_DELETE_INVALID_CONDITION), res); @@ -628,7 +628,7 @@ TEST_F(TestDeleteConditionHandler2, InvalidConditionValue) { conditions[0].condition_values.clear(); conditions[0].condition_values.push_back("-2147483649"); DeletePredicatePB del_pred_6; - res = DeleteHandler::generate_delete_predicate(tablet->tablet_schema(), conditions, + res = DeleteHandler::generate_delete_predicate(*tablet->tablet_schema(), conditions, &del_pred_6); EXPECT_EQ(Status::OLAPInternalError(OLAP_ERR_DELETE_INVALID_CONDITION), res); @@ -637,7 +637,7 @@ TEST_F(TestDeleteConditionHandler2, InvalidConditionValue) { conditions[0].column_name = "k4"; conditions[0].condition_values.push_back("9223372036854775808"); DeletePredicatePB del_pred_7; - res = DeleteHandler::generate_delete_predicate(tablet->tablet_schema(), conditions, + res = DeleteHandler::generate_delete_predicate(*tablet->tablet_schema(), conditions, &del_pred_7); EXPECT_EQ(Status::OLAPInternalError(OLAP_ERR_DELETE_INVALID_CONDITION), res); @@ -645,7 +645,7 @@ TEST_F(TestDeleteConditionHandler2, InvalidConditionValue) { conditions[0].condition_values.clear(); conditions[0].condition_values.push_back("-9223372036854775809"); DeletePredicatePB del_pred_8; - res = DeleteHandler::generate_delete_predicate(tablet->tablet_schema(), conditions, + res = DeleteHandler::generate_delete_predicate(*tablet->tablet_schema(), conditions, &del_pred_8); EXPECT_EQ(Status::OLAPInternalError(OLAP_ERR_DELETE_INVALID_CONDITION), res); @@ -654,7 +654,7 @@ TEST_F(TestDeleteConditionHandler2, InvalidConditionValue) { conditions[0].column_name = "k5"; conditions[0].condition_values.push_back("170141183460469231731687303715884105728"); DeletePredicatePB del_pred_9; - res = DeleteHandler::generate_delete_predicate(tablet->tablet_schema(), conditions, + res = DeleteHandler::generate_delete_predicate(*tablet->tablet_schema(), conditions, &del_pred_9); EXPECT_EQ(Status::OLAPInternalError(OLAP_ERR_DELETE_INVALID_CONDITION), res); @@ -662,7 +662,7 @@ TEST_F(TestDeleteConditionHandler2, InvalidConditionValue) { conditions[0].condition_values.clear(); conditions[0].condition_values.push_back("-170141183460469231731687303715884105729"); DeletePredicatePB del_pred_10; - res = DeleteHandler::generate_delete_predicate(tablet->tablet_schema(), conditions, + res = DeleteHandler::generate_delete_predicate(*tablet->tablet_schema(), conditions, &del_pred_10); EXPECT_EQ(Status::OLAPInternalError(OLAP_ERR_DELETE_INVALID_CONDITION), res); @@ -671,7 +671,7 @@ TEST_F(TestDeleteConditionHandler2, InvalidConditionValue) { conditions[0].column_name = "k9"; conditions[0].condition_values.push_back("12347876.5"); DeletePredicatePB del_pred_11; - res = DeleteHandler::generate_delete_predicate(tablet->tablet_schema(), conditions, + res = DeleteHandler::generate_delete_predicate(*tablet->tablet_schema(), conditions, &del_pred_11); EXPECT_EQ(Status::OLAPInternalError(OLAP_ERR_DELETE_INVALID_CONDITION), res); @@ -679,7 +679,7 @@ TEST_F(TestDeleteConditionHandler2, InvalidConditionValue) { conditions[0].condition_values.clear(); conditions[0].condition_values.push_back("1.2345678"); DeletePredicatePB del_pred_12; - res = DeleteHandler::generate_delete_predicate(tablet->tablet_schema(), conditions, + res = DeleteHandler::generate_delete_predicate(*tablet->tablet_schema(), conditions, &del_pred_12); EXPECT_EQ(Status::OLAPInternalError(OLAP_ERR_DELETE_INVALID_CONDITION), res); @@ -687,7 +687,7 @@ TEST_F(TestDeleteConditionHandler2, InvalidConditionValue) { conditions[0].condition_values.clear(); conditions[0].condition_values.push_back("1."); DeletePredicatePB del_pred_13; - res = DeleteHandler::generate_delete_predicate(tablet->tablet_schema(), conditions, + res = DeleteHandler::generate_delete_predicate(*tablet->tablet_schema(), conditions, &del_pred_13); EXPECT_EQ(Status::OLAPInternalError(OLAP_ERR_DELETE_INVALID_CONDITION), res); @@ -696,21 +696,21 @@ TEST_F(TestDeleteConditionHandler2, InvalidConditionValue) { conditions[0].column_name = "k10"; conditions[0].condition_values.push_back("20130101"); DeletePredicatePB del_pred_14; - res = DeleteHandler::generate_delete_predicate(tablet->tablet_schema(), conditions, + res = DeleteHandler::generate_delete_predicate(*tablet->tablet_schema(), conditions, &del_pred_14); EXPECT_EQ(Status::OLAPInternalError(OLAP_ERR_DELETE_INVALID_CONDITION), res); conditions[0].condition_values.clear(); conditions[0].condition_values.push_back("2013-64-01"); DeletePredicatePB del_pred_15; - res = DeleteHandler::generate_delete_predicate(tablet->tablet_schema(), conditions, + res = DeleteHandler::generate_delete_predicate(*tablet->tablet_schema(), conditions, &del_pred_15); EXPECT_EQ(Status::OLAPInternalError(OLAP_ERR_DELETE_INVALID_CONDITION), res); conditions[0].condition_values.clear(); conditions[0].condition_values.push_back("2013-01-40"); DeletePredicatePB del_pred_16; - res = DeleteHandler::generate_delete_predicate(tablet->tablet_schema(), conditions, + res = DeleteHandler::generate_delete_predicate(*tablet->tablet_schema(), conditions, &del_pred_16); EXPECT_EQ(Status::OLAPInternalError(OLAP_ERR_DELETE_INVALID_CONDITION), res); @@ -719,42 +719,42 @@ TEST_F(TestDeleteConditionHandler2, InvalidConditionValue) { conditions[0].column_name = "k11"; conditions[0].condition_values.push_back("20130101 00:00:00"); DeletePredicatePB del_pred_17; - res = DeleteHandler::generate_delete_predicate(tablet->tablet_schema(), conditions, + res = DeleteHandler::generate_delete_predicate(*tablet->tablet_schema(), conditions, &del_pred_17); EXPECT_EQ(Status::OLAPInternalError(OLAP_ERR_DELETE_INVALID_CONDITION), res); conditions[0].condition_values.clear(); conditions[0].condition_values.push_back("2013-64-01 00:00:00"); DeletePredicatePB del_pred_18; - res = DeleteHandler::generate_delete_predicate(tablet->tablet_schema(), conditions, + res = DeleteHandler::generate_delete_predicate(*tablet->tablet_schema(), conditions, &del_pred_18); EXPECT_EQ(Status::OLAPInternalError(OLAP_ERR_DELETE_INVALID_CONDITION), res); conditions[0].condition_values.clear(); conditions[0].condition_values.push_back("2013-01-40 00:00:00"); DeletePredicatePB del_pred_19; - res = DeleteHandler::generate_delete_predicate(tablet->tablet_schema(), conditions, + res = DeleteHandler::generate_delete_predicate(*tablet->tablet_schema(), conditions, &del_pred_19); EXPECT_EQ(Status::OLAPInternalError(OLAP_ERR_DELETE_INVALID_CONDITION), res); conditions[0].condition_values.clear(); conditions[0].condition_values.push_back("2013-01-01 24:00:00"); DeletePredicatePB del_pred_20; - res = DeleteHandler::generate_delete_predicate(tablet->tablet_schema(), conditions, + res = DeleteHandler::generate_delete_predicate(*tablet->tablet_schema(), conditions, &del_pred_20); EXPECT_EQ(Status::OLAPInternalError(OLAP_ERR_DELETE_INVALID_CONDITION), res); conditions[0].condition_values.clear(); conditions[0].condition_values.push_back("2013-01-01 00:60:00"); DeletePredicatePB del_pred_21; - res = DeleteHandler::generate_delete_predicate(tablet->tablet_schema(), conditions, + res = DeleteHandler::generate_delete_predicate(*tablet->tablet_schema(), conditions, &del_pred_21); EXPECT_EQ(Status::OLAPInternalError(OLAP_ERR_DELETE_INVALID_CONDITION), res); conditions[0].condition_values.clear(); conditions[0].condition_values.push_back("2013-01-01 00:00:60"); DeletePredicatePB del_pred_22; - res = DeleteHandler::generate_delete_predicate(tablet->tablet_schema(), conditions, + res = DeleteHandler::generate_delete_predicate(*tablet->tablet_schema(), conditions, &del_pred_22); EXPECT_EQ(Status::OLAPInternalError(OLAP_ERR_DELETE_INVALID_CONDITION), res); @@ -766,7 +766,7 @@ TEST_F(TestDeleteConditionHandler2, InvalidConditionValue) { "FhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYW" "FhYWFhYWFhYWFhYWFhYWFhYWFhYWE=;k13=YWFhYQ=="); DeletePredicatePB del_pred_23; - res = DeleteHandler::generate_delete_predicate(tablet->tablet_schema(), conditions, + res = DeleteHandler::generate_delete_predicate(*tablet->tablet_schema(), conditions, &del_pred_23); EXPECT_EQ(Status::OLAPInternalError(OLAP_ERR_DELETE_INVALID_CONDITION), res); @@ -777,7 +777,7 @@ TEST_F(TestDeleteConditionHandler2, InvalidConditionValue) { "FhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYW" "FhYWFhYWFhYWFhYWFhYWFhYWFhYWE=;k13=YWFhYQ=="); DeletePredicatePB del_pred_24; - res = DeleteHandler::generate_delete_predicate(tablet->tablet_schema(), conditions, + res = DeleteHandler::generate_delete_predicate(*tablet->tablet_schema(), conditions, &del_pred_24); EXPECT_EQ(Status::OLAPInternalError(OLAP_ERR_DELETE_INVALID_CONDITION), res); } @@ -811,8 +811,8 @@ class TestDeleteHandler : public testing::Test { EXPECT_TRUE(tablet != nullptr); _tablet_path = tablet->tablet_path(); - _data_row_cursor.init(tablet->tablet_schema()); - _data_row_cursor.allocate_memory_for_string_type(tablet->tablet_schema()); + _data_row_cursor.init(*tablet->tablet_schema()); + _data_row_cursor.allocate_memory_for_string_type(*tablet->tablet_schema()); } void TearDown() { @@ -856,7 +856,7 @@ TEST_F(TestDeleteHandler, InitSuccess) { conditions.push_back(condition); DeletePredicatePB del_pred; - res = DeleteHandler::generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred); + res = DeleteHandler::generate_delete_predicate(*tablet->tablet_schema(), conditions, &del_pred); EXPECT_EQ(Status::OK(), res); tablet->add_delete_predicate(del_pred, 1); @@ -868,7 +868,7 @@ TEST_F(TestDeleteHandler, InitSuccess) { conditions.push_back(condition); DeletePredicatePB del_pred_2; - res = DeleteHandler::generate_delete_predicate(tablet->tablet_schema(), conditions, + res = DeleteHandler::generate_delete_predicate(*tablet->tablet_schema(), conditions, &del_pred_2); EXPECT_EQ(Status::OK(), res); tablet->add_delete_predicate(del_pred_2, 2); @@ -881,7 +881,7 @@ TEST_F(TestDeleteHandler, InitSuccess) { conditions.push_back(condition); DeletePredicatePB del_pred_3; - res = DeleteHandler::generate_delete_predicate(tablet->tablet_schema(), conditions, + res = DeleteHandler::generate_delete_predicate(*tablet->tablet_schema(), conditions, &del_pred_3); EXPECT_EQ(Status::OK(), res); tablet->add_delete_predicate(del_pred_3, 3); @@ -894,13 +894,13 @@ TEST_F(TestDeleteHandler, InitSuccess) { conditions.push_back(condition); DeletePredicatePB del_pred_4; - res = DeleteHandler::generate_delete_predicate(tablet->tablet_schema(), conditions, + res = DeleteHandler::generate_delete_predicate(*tablet->tablet_schema(), conditions, &del_pred_4); EXPECT_EQ(Status::OK(), res); tablet->add_delete_predicate(del_pred_4, 4); // 从header文件中取出版本号小于等于7的过滤条件 - res = _delete_handler.init(tablet->tablet_schema(), tablet->delete_predicates(), 4); + res = _delete_handler.init(*tablet->tablet_schema(), tablet->delete_predicates(), 4); EXPECT_EQ(Status::OK(), res); EXPECT_EQ(4, _delete_handler.conditions_num()); std::vector conds_version = _delete_handler.get_conds_version(); @@ -936,12 +936,12 @@ TEST_F(TestDeleteHandler, FilterDataSubconditions) { conditions.push_back(condition); DeletePredicatePB del_pred; - res = DeleteHandler::generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred); + res = DeleteHandler::generate_delete_predicate(*tablet->tablet_schema(), conditions, &del_pred); EXPECT_EQ(Status::OK(), res); tablet->add_delete_predicate(del_pred, 1); // 指定版本号为10以载入Header中的所有过滤条件(在这个case中，只有过滤条件1) - res = _delete_handler.init(tablet->tablet_schema(), tablet->delete_predicates(), 4); + res = _delete_handler.init(*tablet->tablet_schema(), tablet->delete_predicates(), 4); EXPECT_EQ(Status::OK(), res); EXPECT_EQ(1, _delete_handler.conditions_num()); @@ -996,7 +996,7 @@ TEST_F(TestDeleteHandler, FilterDataConditions) { conditions.push_back(condition); DeletePredicatePB del_pred; - res = DeleteHandler::generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred); + res = DeleteHandler::generate_delete_predicate(*tablet->tablet_schema(), conditions, &del_pred); EXPECT_EQ(Status::OK(), res); tablet->add_delete_predicate(del_pred, 1); @@ -1009,7 +1009,7 @@ TEST_F(TestDeleteHandler, FilterDataConditions) { conditions.push_back(condition); DeletePredicatePB del_pred_2; - res = DeleteHandler::generate_delete_predicate(tablet->tablet_schema(), conditions, + res = DeleteHandler::generate_delete_predicate(*tablet->tablet_schema(), conditions, &del_pred_2); EXPECT_EQ(Status::OK(), res); tablet->add_delete_predicate(del_pred_2, 2); @@ -1023,13 +1023,13 @@ TEST_F(TestDeleteHandler, FilterDataConditions) { conditions.push_back(condition); DeletePredicatePB del_pred_3; - res = DeleteHandler::generate_delete_predicate(tablet->tablet_schema(), conditions, + res = DeleteHandler::generate_delete_predicate(*tablet->tablet_schema(), conditions, &del_pred_3); EXPECT_EQ(Status::OK(), res); tablet->add_delete_predicate(del_pred_3, 3); // 指定版本号为4以载入meta中的所有过滤条件(在这个case中，只有过滤条件1) - res = _delete_handler.init(tablet->tablet_schema(), tablet->delete_predicates(), 4); + res = _delete_handler.init(*tablet->tablet_schema(), tablet->delete_predicates(), 4); EXPECT_EQ(Status::OK(), res); EXPECT_EQ(3, _delete_handler.conditions_num()); @@ -1075,7 +1075,7 @@ TEST_F(TestDeleteHandler, FilterDataVersion) { conditions.push_back(condition); DeletePredicatePB del_pred; - res = DeleteHandler::generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred); + res = DeleteHandler::generate_delete_predicate(*tablet->tablet_schema(), conditions, &del_pred); EXPECT_EQ(Status::OK(), res); tablet->add_delete_predicate(del_pred, 3); @@ -1088,13 +1088,13 @@ TEST_F(TestDeleteHandler, FilterDataVersion) { conditions.push_back(condition); DeletePredicatePB del_pred_2; - res = DeleteHandler::generate_delete_predicate(tablet->tablet_schema(), conditions, + res = DeleteHandler::generate_delete_predicate(*tablet->tablet_schema(), conditions, &del_pred_2); EXPECT_EQ(Status::OK(), res); tablet->add_delete_predicate(del_pred_2, 4); // 指定版本号为4以载入meta中的所有过滤条件(过滤条件1，过滤条件2) - res = _delete_handler.init(tablet->tablet_schema(), tablet->delete_predicates(), 4); + res = _delete_handler.init(*tablet->tablet_schema(), tablet->delete_predicates(), 4); EXPECT_EQ(Status::OK(), res); EXPECT_EQ(2, _delete_handler.conditions_num()); diff --git a/be/test/olap/engine_storage_migration_task_test.cpp b/be/test/olap/engine_storage_migration_task_test.cpp index a36f0ff0c4b118..d1785b26407379 100644 --- a/be/test/olap/engine_storage_migration_task_test.cpp +++ b/be/test/olap/engine_storage_migration_task_test.cpp @@ -67,7 +67,6 @@ static void set_up() { options.store_paths = paths; Status s = doris::StorageEngine::open(options, &k_engine); EXPECT_TRUE(s.ok()) << s.to_string(); - ExecEnv* exec_env = doris::ExecEnv::GetInstance(); exec_env->set_storage_engine(k_engine); k_engine->start_bg_threads(); diff --git a/be/test/olap/rowset/beta_rowset_test.cpp b/be/test/olap/rowset/beta_rowset_test.cpp index 605e9dfaff5351..4d849b4ac2be9f 100644 --- a/be/test/olap/rowset/beta_rowset_test.cpp +++ b/be/test/olap/rowset/beta_rowset_test.cpp @@ -94,7 +94,7 @@ class BetaRowsetTest : public testing::Test { OlapReaderStatistics _stats; // (k1 int, k2 varchar(20), k3 int) duplicated key (k1, k2) - void create_tablet_schema(TabletSchema* tablet_schema) { + void create_tablet_schema(TabletSchemaSPtr tablet_schema) { TabletSchemaPB tablet_schema_pb; tablet_schema_pb.set_keys_type(DUP_KEYS); tablet_schema_pb.set_num_short_key_columns(2); @@ -137,7 +137,7 @@ class BetaRowsetTest : public testing::Test { tablet_schema->init_from_pb(tablet_schema_pb); } - void create_rowset_writer_context(TabletSchema* tablet_schema, + void create_rowset_writer_context(TabletSchemaSPtr tablet_schema, RowsetWriterContext* rowset_writer_context) { RowsetId rowset_id; rowset_id.init(10000); @@ -170,8 +170,8 @@ class BetaRowsetTest : public testing::Test { TEST_F(BetaRowsetTest, BasicFunctionTest) { Status s; - TabletSchema tablet_schema; - create_tablet_schema(&tablet_schema); + TabletSchemaSPtr tablet_schema = std::make_shared(); + create_tablet_schema(tablet_schema); RowsetSharedPtr rowset; const int num_segments = 3; @@ -179,14 +179,14 @@ TEST_F(BetaRowsetTest, BasicFunctionTest) { std::vector segment_num_rows; { // write `num_segments * rows_per_segment` rows to rowset RowsetWriterContext writer_context; - create_rowset_writer_context(&tablet_schema, &writer_context); + create_rowset_writer_context(tablet_schema, &writer_context); std::unique_ptr rowset_writer; s = RowsetFactory::create_rowset_writer(writer_context, &rowset_writer); EXPECT_EQ(Status::OK(), s); RowCursor input_row; - input_row.init(tablet_schema); + input_row.init(*tablet_schema); // for segment "i", row "rid" // k1 := rid*10 + i @@ -216,7 +216,7 @@ TEST_F(BetaRowsetTest, BasicFunctionTest) { { // test return ordered results and return k1 and k2 RowsetReaderContext reader_context; - reader_context.tablet_schema = &tablet_schema; + reader_context.tablet_schema = tablet_schema.get(); reader_context.need_ordered_result = true; std::vector return_columns = {0, 1}; reader_context.return_columns = &return_columns; @@ -306,7 +306,7 @@ TEST_F(BetaRowsetTest, BasicFunctionTest) { { // test return unordered data and only k3 RowsetReaderContext reader_context; - reader_context.tablet_schema = &tablet_schema; + reader_context.tablet_schema = tablet_schema.get(); reader_context.need_ordered_result = false; std::vector return_columns = {2}; reader_context.return_columns = &return_columns; diff --git a/be/test/olap/rowset/rowset_tree_test.cpp b/be/test/olap/rowset/rowset_tree_test.cpp index 0dd20a64fd92d3..de5dc451243885 100644 --- a/be/test/olap/rowset/rowset_tree_test.cpp +++ b/be/test/olap/rowset/rowset_tree_test.cpp @@ -40,6 +40,7 @@ #include "gutil/strings/substitute.h" #include "olap/rowset/rowset.h" #include "olap/rowset/unique_rowset_id_generator.h" +#include "olap/tablet_schema.h" #include "testutil/mock_rowset.h" #include "testutil/test_util.h" #include "util/slice.h" @@ -59,9 +60,10 @@ class TestRowsetTree : public testing::Test { TestRowsetTree() : rowset_id_generator_({0, 0}) {} void SetUp() { + schema_ = std::make_shared(); TabletSchemaPB schema_pb; schema_pb.set_keys_type(UNIQUE_KEYS); - schema_.init_from_pb(schema_pb); + schema_->init_from_pb(schema_pb); } // Generates random rowsets with keys between 0 and 10000 @@ -88,12 +90,12 @@ class TestRowsetTree : public testing::Test { RowsetMetaSharedPtr meta_ptr = make_shared(); meta_ptr->init_from_pb(rs_meta_pb); RowsetSharedPtr res_ptr; - MockRowset::create_rowset(&schema_, rowset_path_, meta_ptr, &res_ptr, is_mem_rowset); + MockRowset::create_rowset(schema_, rowset_path_, meta_ptr, &res_ptr, is_mem_rowset); return res_ptr; } private: - TabletSchema schema_; + TabletSchemaSPtr schema_; std::string rowset_path_; UniqueRowsetIdGenerator rowset_id_generator_; }; diff --git a/be/test/olap/rowset/segment_v2/segment_test.cpp b/be/test/olap/rowset/segment_v2/segment_test.cpp index b566a6d5f9b072..a3767a18e22022 100644 --- a/be/test/olap/rowset/segment_v2/segment_test.cpp +++ b/be/test/olap/rowset/segment_v2/segment_test.cpp @@ -22,6 +22,7 @@ #include #include #include +#include #include #include "common/logging.h" @@ -100,28 +101,28 @@ class SegmentReaderWriterTest : public ::testing::Test { } } - TabletSchema create_schema(const std::vector& columns, - KeysType keys_type = DUP_KEYS, int num_custom_key_columns = -1) { - TabletSchema res; + TabletSchemaSPtr create_schema(const std::vector& columns, + KeysType keys_type = DUP_KEYS, int num_custom_key_columns = -1) { + TabletSchemaSPtr res = std::make_shared(); int num_key_columns = 0; for (auto& col : columns) { if (col.is_key()) { num_key_columns++; } - res._cols.push_back(col); + res->_cols.push_back(col); } - res._num_columns = columns.size(); - res._num_key_columns = num_key_columns; - res._num_short_key_columns = + res->_num_columns = columns.size(); + res->_num_key_columns = num_key_columns; + res->_num_short_key_columns = num_custom_key_columns != -1 ? num_custom_key_columns : num_key_columns; - res._keys_type = keys_type; - res.init_field_index_for_test(); + res->_keys_type = keys_type; + res->init_field_index_for_test(); return res; } - void build_segment(SegmentWriterOptions opts, const TabletSchema& build_schema, - const TabletSchema& query_schema, size_t nrows, - const ValueGenerator& generator, shared_ptr* res) { + void build_segment(SegmentWriterOptions opts, TabletSchemaSPtr build_schema, + TabletSchemaSPtr query_schema, size_t nrows, const ValueGenerator& generator, + shared_ptr* res) { static int seg_id = 0; // must use unique filename for each segment, otherwise page cache kicks in and produces // the wrong answer (it use (filename,offset) as cache key) @@ -134,16 +135,16 @@ class SegmentReaderWriterTest : public ::testing::Test { EXPECT_TRUE(st.ok()); DataDir data_dir(kSegmentDir); data_dir.init(); - SegmentWriter writer(file_writer.get(), 0, &build_schema, &data_dir, INT32_MAX, opts); + SegmentWriter writer(file_writer.get(), 0, build_schema, &data_dir, INT32_MAX, opts); st = writer.init(10); EXPECT_TRUE(st.ok()); RowCursor row; - auto olap_st = row.init(build_schema); + auto olap_st = row.init(*build_schema); EXPECT_EQ(Status::OK(), olap_st); for (size_t rid = 0; rid < nrows; ++rid) { - for (int cid = 0; cid < build_schema.num_columns(); ++cid) { + for (int cid = 0; cid < build_schema->num_columns(); ++cid) { int row_block_id = rid / opts.num_rows_per_block; RowCursorCell cell = row.cell(cid); generator(rid, cid, row_block_id, cell); @@ -156,31 +157,31 @@ class SegmentReaderWriterTest : public ::testing::Test { EXPECT_TRUE(st.ok()); EXPECT_TRUE(file_writer->close().ok()); // Check min/max key generation - if (build_schema.keys_type() == UNIQUE_KEYS && opts.enable_unique_key_merge_on_write) { + if (build_schema->keys_type() == UNIQUE_KEYS && opts.enable_unique_key_merge_on_write) { // Create min row - for (int cid = 0; cid < build_schema.num_key_columns(); ++cid) { + for (int cid = 0; cid < build_schema->num_key_columns(); ++cid) { RowCursorCell cell = row.cell(cid); generator(0, cid, 0 / opts.num_rows_per_block, cell); } std::string min_encoded_key; encode_key(&min_encoded_key, row, - build_schema.num_key_columns()); + build_schema->num_key_columns()); EXPECT_EQ(min_encoded_key, writer.min_encoded_key().to_string()); // Create max row - for (int cid = 0; cid < build_schema.num_key_columns(); ++cid) { + for (int cid = 0; cid < build_schema->num_key_columns(); ++cid) { RowCursorCell cell = row.cell(cid); generator(nrows - 1, cid, (nrows - 1) / opts.num_rows_per_block, cell); } std::string max_encoded_key; encode_key(&max_encoded_key, row, - build_schema.num_key_columns()); + build_schema->num_key_columns()); EXPECT_EQ(max_encoded_key, writer.max_encoded_key().to_string()); } else { EXPECT_EQ("", writer.min_encoded_key().to_string()); EXPECT_EQ("", writer.max_encoded_key().to_string()); } - st = Segment::open(fs, path, 0, &query_schema, res); + st = Segment::open(fs, path, 0, query_schema, res); EXPECT_TRUE(st.ok()); EXPECT_EQ(nrows, (*res)->num_rows()); } @@ -194,9 +195,10 @@ TEST_F(SegmentReaderWriterTest, normal) { std::vector enable_unique_key_merge_on_write_vec = {false, true}; for (auto keys_type : keys_type_vec) { for (auto enable_unique_key_merge_on_write : enable_unique_key_merge_on_write_vec) { - TabletSchema tablet_schema = create_schema({create_int_key(1), create_int_key(2), - create_int_value(3), create_int_value(4)}, - keys_type); + TabletSchemaSPtr tablet_schema = + create_schema({create_int_key(1), create_int_key(2), create_int_value(3), + create_int_value(4)}, + keys_type); SegmentWriterOptions opts; opts.enable_unique_key_merge_on_write = enable_unique_key_merge_on_write; opts.num_rows_per_block = 10; @@ -206,13 +208,13 @@ TEST_F(SegmentReaderWriterTest, normal) { // reader { - Schema schema(tablet_schema); + Schema schema(*tablet_schema); OlapReaderStatistics stats; // scan all rows { StorageReadOptions read_opts; read_opts.stats = &stats; - read_opts.tablet_schema = &tablet_schema; + read_opts.tablet_schema = tablet_schema.get(); std::unique_ptr iter; ASSERT_TRUE(segment->new_iterator(schema, read_opts, &iter).ok()); @@ -245,7 +247,7 @@ TEST_F(SegmentReaderWriterTest, normal) { { // lower bound std::unique_ptr lower_bound(new RowCursor()); - lower_bound->init(tablet_schema, 2); + lower_bound->init(*tablet_schema, 2); { auto cell = lower_bound->cell(0); cell.set_not_null(); @@ -259,7 +261,7 @@ TEST_F(SegmentReaderWriterTest, normal) { // upper bound std::unique_ptr upper_bound(new RowCursor()); - upper_bound->init(tablet_schema, 1); + upper_bound->init(*tablet_schema, 1); { auto cell = upper_bound->cell(0); cell.set_not_null(); @@ -268,7 +270,7 @@ TEST_F(SegmentReaderWriterTest, normal) { StorageReadOptions read_opts; read_opts.stats = &stats; - read_opts.tablet_schema = &tablet_schema; + read_opts.tablet_schema = tablet_schema.get(); read_opts.key_ranges.emplace_back(lower_bound.get(), false, upper_bound.get(), true); std::unique_ptr iter; @@ -287,7 +289,7 @@ TEST_F(SegmentReaderWriterTest, normal) { { // lower bound std::unique_ptr lower_bound(new RowCursor()); - lower_bound->init(tablet_schema, 2); + lower_bound->init(*tablet_schema, 2); { auto cell = lower_bound->cell(0); cell.set_not_null(); @@ -301,7 +303,7 @@ TEST_F(SegmentReaderWriterTest, normal) { // upper bound std::unique_ptr upper_bound(new RowCursor()); - upper_bound->init(tablet_schema, 2); + upper_bound->init(*tablet_schema, 2); { auto cell = upper_bound->cell(0); cell.set_not_null(); @@ -316,7 +318,7 @@ TEST_F(SegmentReaderWriterTest, normal) { // include upper key StorageReadOptions read_opts; read_opts.stats = &stats; - read_opts.tablet_schema = &tablet_schema; + read_opts.tablet_schema = tablet_schema.get(); read_opts.key_ranges.emplace_back(lower_bound.get(), true, upper_bound.get(), true); std::unique_ptr iter; @@ -334,7 +336,7 @@ TEST_F(SegmentReaderWriterTest, normal) { // not include upper key StorageReadOptions read_opts1; read_opts1.stats = &stats; - read_opts1.tablet_schema = &tablet_schema; + read_opts1.tablet_schema = tablet_schema.get(); read_opts1.key_ranges.emplace_back(lower_bound.get(), true, upper_bound.get(), false); std::unique_ptr iter1; @@ -349,7 +351,7 @@ TEST_F(SegmentReaderWriterTest, normal) { { // lower bound std::unique_ptr lower_bound(new RowCursor()); - lower_bound->init(tablet_schema, 1); + lower_bound->init(*tablet_schema, 1); { auto cell = lower_bound->cell(0); cell.set_not_null(); @@ -358,7 +360,7 @@ TEST_F(SegmentReaderWriterTest, normal) { StorageReadOptions read_opts; read_opts.stats = &stats; - read_opts.tablet_schema = &tablet_schema; + read_opts.tablet_schema = tablet_schema.get(); read_opts.key_ranges.emplace_back(lower_bound.get(), false, nullptr, false); std::unique_ptr iter; ASSERT_TRUE(segment->new_iterator(schema, read_opts, &iter).ok()); @@ -371,7 +373,7 @@ TEST_F(SegmentReaderWriterTest, normal) { { // lower bound std::unique_ptr lower_bound(new RowCursor()); - lower_bound->init(tablet_schema, 1); + lower_bound->init(*tablet_schema, 1); { auto cell = lower_bound->cell(0); cell.set_not_null(); @@ -379,7 +381,7 @@ TEST_F(SegmentReaderWriterTest, normal) { } std::unique_ptr upper_bound(new RowCursor()); - upper_bound->init(tablet_schema, 1); + upper_bound->init(*tablet_schema, 1); { auto cell = upper_bound->cell(0); cell.set_not_null(); @@ -388,7 +390,7 @@ TEST_F(SegmentReaderWriterTest, normal) { StorageReadOptions read_opts; read_opts.stats = &stats; - read_opts.tablet_schema = &tablet_schema; + read_opts.tablet_schema = tablet_schema.get(); read_opts.key_ranges.emplace_back(lower_bound.get(), false, upper_bound.get(), false); std::unique_ptr iter; @@ -404,7 +406,7 @@ TEST_F(SegmentReaderWriterTest, normal) { } TEST_F(SegmentReaderWriterTest, LazyMaterialization) { - TabletSchema tablet_schema = create_schema({create_int_key(1), create_int_value(2)}); + TabletSchemaSPtr tablet_schema = create_schema({create_int_key(1), create_int_value(2)}); ValueGenerator data_gen = [](size_t rid, int cid, int block_id, RowCursorCell& cell) { cell.set_not_null(); if (cid == 0) { @@ -421,7 +423,7 @@ TEST_F(SegmentReaderWriterTest, LazyMaterialization) { { // lazy enabled when predicate is subset of returned columns: // select c1, c2 where c2 = 30; - Schema read_schema(tablet_schema); + Schema read_schema(*tablet_schema); std::unique_ptr predicate(new EqualPredicate(1, 30)); const std::vector predicates = {predicate.get()}; @@ -429,7 +431,7 @@ TEST_F(SegmentReaderWriterTest, LazyMaterialization) { StorageReadOptions read_opts; read_opts.column_predicates = predicates; read_opts.stats = &stats; - read_opts.tablet_schema = &tablet_schema; + read_opts.tablet_schema = tablet_schema.get(); std::unique_ptr iter; ASSERT_TRUE(segment->new_iterator(read_schema, read_opts, &iter).ok()); @@ -445,7 +447,7 @@ TEST_F(SegmentReaderWriterTest, LazyMaterialization) { { // lazy disabled when all return columns have predicates: // select c1, c2 where c1 = 10 and c2 = 100; - Schema read_schema(tablet_schema); + Schema read_schema(*tablet_schema); std::unique_ptr p0(new EqualPredicate(0, 10)); std::unique_ptr p1(new EqualPredicate(1, 100)); const std::vector predicates = {p0.get(), p1.get()}; @@ -454,7 +456,7 @@ TEST_F(SegmentReaderWriterTest, LazyMaterialization) { StorageReadOptions read_opts; read_opts.column_predicates = predicates; read_opts.stats = &stats; - read_opts.tablet_schema = &tablet_schema; + read_opts.tablet_schema = tablet_schema.get(); std::unique_ptr iter; ASSERT_TRUE(segment->new_iterator(read_schema, read_opts, &iter).ok()); @@ -471,11 +473,11 @@ TEST_F(SegmentReaderWriterTest, LazyMaterialization) { // lazy disabled when no predicate: // select c2 std::vector read_cols = {1}; - Schema read_schema(tablet_schema.columns(), read_cols); + Schema read_schema(tablet_schema->columns(), read_cols); OlapReaderStatistics stats; StorageReadOptions read_opts; read_opts.stats = &stats; - read_opts.tablet_schema = &tablet_schema; + read_opts.tablet_schema = tablet_schema.get(); std::unique_ptr iter; ASSERT_TRUE(segment->new_iterator(read_schema, read_opts, &iter).ok()); @@ -500,7 +502,7 @@ TEST_F(SegmentReaderWriterTest, LazyMaterialization) { { // lazy disabled when all predicates are removed by bitmap index: // select c1, c2 where c2 = 30; - Schema read_schema(tablet_schema); + Schema read_schema(*tablet_schema); std::unique_ptr predicate(new EqualPredicate(0, 20)); const std::vector predicates = {predicate.get()}; @@ -508,7 +510,7 @@ TEST_F(SegmentReaderWriterTest, LazyMaterialization) { StorageReadOptions read_opts; read_opts.column_predicates = predicates; read_opts.stats = &stats; - read_opts.tablet_schema = &tablet_schema; + read_opts.tablet_schema = tablet_schema.get(); std::unique_ptr iter; ASSERT_TRUE(segment->new_iterator(read_schema, read_opts, &iter).ok()); @@ -526,8 +528,9 @@ TEST_F(SegmentReaderWriterTest, LazyMaterialization) { } TEST_F(SegmentReaderWriterTest, TestIndex) { - TabletSchema tablet_schema = create_schema({create_int_key(1), create_int_key(2, true, true), - create_int_key(3), create_int_value(4)}); + TabletSchemaSPtr tablet_schema = + create_schema({create_int_key(1), create_int_key(2, true, true), create_int_key(3), + create_int_value(4)}); SegmentWriterOptions opts; opts.num_rows_per_block = 10; @@ -554,7 +557,7 @@ TEST_F(SegmentReaderWriterTest, TestIndex) { // reader with condition { - Schema schema(tablet_schema); + Schema schema(*tablet_schema); OlapReaderStatistics stats; // test empty segment iterator { @@ -565,12 +568,12 @@ TEST_F(SegmentReaderWriterTest, TestIndex) { std::vector vals = {"2"}; condition.__set_condition_values(vals); std::shared_ptr conditions(new Conditions()); - conditions->set_tablet_schema(&tablet_schema); + conditions->set_tablet_schema(tablet_schema.get()); EXPECT_EQ(Status::OK(), conditions->append_condition(condition)); StorageReadOptions read_opts; read_opts.stats = &stats; - read_opts.tablet_schema = &tablet_schema; + read_opts.tablet_schema = tablet_schema.get(); read_opts.conditions = conditions.get(); std::unique_ptr iter; @@ -589,12 +592,12 @@ TEST_F(SegmentReaderWriterTest, TestIndex) { std::vector vals = {"100"}; condition.__set_condition_values(vals); std::shared_ptr conditions(new Conditions()); - conditions->set_tablet_schema(&tablet_schema); + conditions->set_tablet_schema(tablet_schema.get()); EXPECT_EQ(Status::OK(), conditions->append_condition(condition)); StorageReadOptions read_opts; read_opts.stats = &stats; - read_opts.tablet_schema = &tablet_schema; + read_opts.tablet_schema = tablet_schema.get(); read_opts.conditions = conditions.get(); std::unique_ptr iter; @@ -639,7 +642,7 @@ TEST_F(SegmentReaderWriterTest, TestIndex) { std::vector vals = {"165000"}; condition.__set_condition_values(vals); std::shared_ptr conditions(new Conditions()); - conditions->set_tablet_schema(&tablet_schema); + conditions->set_tablet_schema(tablet_schema.get()); EXPECT_EQ(Status::OK(), conditions->append_condition(condition)); // the second page read will be pruned by the following delete predicate @@ -649,12 +652,12 @@ TEST_F(SegmentReaderWriterTest, TestIndex) { std::vector vals2 = {"164001"}; delete_condition.__set_condition_values(vals2); std::shared_ptr delete_conditions(new Conditions()); - delete_conditions->set_tablet_schema(&tablet_schema); + delete_conditions->set_tablet_schema(tablet_schema.get()); EXPECT_EQ(Status::OK(), delete_conditions->append_condition(delete_condition)); StorageReadOptions read_opts; read_opts.stats = &stats; - read_opts.tablet_schema = &tablet_schema; + read_opts.tablet_schema = tablet_schema.get(); read_opts.conditions = conditions.get(); read_opts.delete_conditions.push_back(delete_conditions.get()); @@ -696,7 +699,7 @@ TEST_F(SegmentReaderWriterTest, TestIndex) { { StorageReadOptions read_opts; read_opts.stats = &stats; - read_opts.tablet_schema = &tablet_schema; + read_opts.tablet_schema = tablet_schema.get(); TCondition condition; condition.__set_column_name("2"); condition.__set_condition_op("="); @@ -704,7 +707,7 @@ TEST_F(SegmentReaderWriterTest, TestIndex) { std::vector vals = {"102"}; condition.__set_condition_values(vals); std::shared_ptr conditions(new Conditions()); - conditions->set_tablet_schema(&tablet_schema); + conditions->set_tablet_schema(tablet_schema.get()); EXPECT_EQ(Status::OK(), conditions->append_condition(condition)); read_opts.conditions = conditions.get(); std::unique_ptr iter; @@ -741,7 +744,7 @@ TEST_F(SegmentReaderWriterTest, estimate_segment_size) { EXPECT_TRUE(st.ok()) << st.to_string(); DataDir data_dir(kSegmentDir); data_dir.init(); - SegmentWriter writer(file_writer.get(), 0, tablet_schema.get(), &data_dir, INT32_MAX, opts); + SegmentWriter writer(file_writer.get(), 0, tablet_schema, &data_dir, INT32_MAX, opts); st = writer.init(10); EXPECT_TRUE(st.ok()) << st.to_string(); @@ -778,25 +781,25 @@ TEST_F(SegmentReaderWriterTest, estimate_segment_size) { TEST_F(SegmentReaderWriterTest, TestDefaultValueColumn) { std::vector columns = {create_int_key(1), create_int_key(2), create_int_value(3), create_int_value(4)}; - TabletSchema build_schema = create_schema(columns); + TabletSchemaSPtr build_schema = create_schema(columns); // add a column with null default value { std::vector read_columns = columns; read_columns.push_back(create_int_value(5, OLAP_FIELD_AGGREGATION_SUM, true, "NULL")); - TabletSchema query_schema = create_schema(read_columns); + TabletSchemaSPtr query_schema = create_schema(read_columns); std::shared_ptr segment; build_segment(SegmentWriterOptions(), build_schema, query_schema, 4096, DefaultIntGenerator, &segment); - Schema schema(query_schema); + Schema schema(*query_schema); OlapReaderStatistics stats; // scan all rows { StorageReadOptions read_opts; read_opts.stats = &stats; - read_opts.tablet_schema = &query_schema; + read_opts.tablet_schema = query_schema.get(); std::unique_ptr iter; ASSERT_TRUE(segment->new_iterator(schema, read_opts, &iter).ok()); @@ -835,19 +838,19 @@ TEST_F(SegmentReaderWriterTest, TestDefaultValueColumn) { { std::vector read_columns = columns; read_columns.push_back(create_int_value(5, OLAP_FIELD_AGGREGATION_SUM, true, "10086")); - TabletSchema query_schema = create_schema(read_columns); + TabletSchemaSPtr query_schema = create_schema(read_columns); std::shared_ptr segment; build_segment(SegmentWriterOptions(), build_schema, query_schema, 4096, DefaultIntGenerator, &segment); - Schema schema(query_schema); + Schema schema(*query_schema); OlapReaderStatistics stats; // scan all rows { StorageReadOptions read_opts; read_opts.stats = &stats; - read_opts.tablet_schema = &query_schema; + read_opts.tablet_schema = query_schema.get(); std::unique_ptr iter; ASSERT_TRUE(segment->new_iterator(schema, read_opts, &iter).ok()); @@ -909,7 +912,7 @@ TEST_F(SegmentReaderWriterTest, TestStringDict) { EXPECT_TRUE(st.ok()); DataDir data_dir(kSegmentDir); data_dir.init(); - SegmentWriter writer(file_writer.get(), 0, tablet_schema.get(), &data_dir, INT32_MAX, opts); + SegmentWriter writer(file_writer.get(), 0, tablet_schema, &data_dir, INT32_MAX, opts); st = writer.init(10); EXPECT_TRUE(st.ok()); @@ -940,7 +943,7 @@ TEST_F(SegmentReaderWriterTest, TestStringDict) { { std::shared_ptr segment; - st = Segment::open(fs, fname, 0, tablet_schema.get(), &segment); + st = Segment::open(fs, fname, 0, tablet_schema, &segment); EXPECT_TRUE(st.ok()); EXPECT_EQ(4096, segment->num_rows()); Schema schema(*tablet_schema); @@ -1134,9 +1137,9 @@ TEST_F(SegmentReaderWriterTest, TestStringDict) { } TEST_F(SegmentReaderWriterTest, TestBitmapPredicate) { - TabletSchema tablet_schema = create_schema({create_int_key(1, true, false, true), - create_int_key(2, true, false, true), - create_int_value(3), create_int_value(4)}); + TabletSchemaSPtr tablet_schema = create_schema({create_int_key(1, true, false, true), + create_int_key(2, true, false, true), + create_int_value(3), create_int_value(4)}); SegmentWriterOptions opts; shared_ptr segment; @@ -1145,7 +1148,7 @@ TEST_F(SegmentReaderWriterTest, TestBitmapPredicate) { EXPECT_TRUE(column_contains_index(segment->footer().columns(1), BITMAP_INDEX)); { - Schema schema(tablet_schema); + Schema schema(*tablet_schema); // test where v1=10 { @@ -1157,7 +1160,7 @@ TEST_F(SegmentReaderWriterTest, TestBitmapPredicate) { OlapReaderStatistics stats; read_opts.column_predicates = column_predicates; read_opts.stats = &stats; - read_opts.tablet_schema = &tablet_schema; + read_opts.tablet_schema = tablet_schema.get(); std::unique_ptr iter; ASSERT_TRUE(segment->new_iterator(schema, read_opts, &iter).ok()); @@ -1180,7 +1183,7 @@ TEST_F(SegmentReaderWriterTest, TestBitmapPredicate) { OlapReaderStatistics stats; read_opts.column_predicates = column_predicates; read_opts.stats = &stats; - read_opts.tablet_schema = &tablet_schema; + read_opts.tablet_schema = tablet_schema.get(); std::unique_ptr iter; ASSERT_TRUE(segment->new_iterator(schema, read_opts, &iter).ok()); @@ -1203,7 +1206,7 @@ TEST_F(SegmentReaderWriterTest, TestBitmapPredicate) { OlapReaderStatistics stats; read_opts.column_predicates = column_predicates; read_opts.stats = &stats; - read_opts.tablet_schema = &tablet_schema; + read_opts.tablet_schema = tablet_schema.get(); std::unique_ptr iter; ASSERT_TRUE(segment->new_iterator(schema, read_opts, &iter).ok()); @@ -1228,7 +1231,7 @@ TEST_F(SegmentReaderWriterTest, TestBitmapPredicate) { OlapReaderStatistics stats; read_opts.column_predicates = column_predicates; read_opts.stats = &stats; - read_opts.tablet_schema = &tablet_schema; + read_opts.tablet_schema = tablet_schema.get(); std::unique_ptr iter; ASSERT_TRUE(segment->new_iterator(schema, read_opts, &iter).ok()); @@ -1252,7 +1255,7 @@ TEST_F(SegmentReaderWriterTest, TestBitmapPredicate) { OlapReaderStatistics stats; read_opts.column_predicates = column_predicates; read_opts.stats = &stats; - read_opts.tablet_schema = &tablet_schema; + read_opts.tablet_schema = tablet_schema.get(); std::unique_ptr iter; ASSERT_TRUE(segment->new_iterator(schema, read_opts, &iter).ok()); @@ -1270,7 +1273,7 @@ TEST_F(SegmentReaderWriterTest, TestBitmapPredicate) { } TEST_F(SegmentReaderWriterTest, TestBloomFilterIndexUniqueModel) { - TabletSchema schema = + TabletSchemaSPtr schema = create_schema({create_int_key(1), create_int_key(2), create_int_key(3), create_int_value(4, OLAP_FIELD_AGGREGATION_REPLACE, true, "", true)}); @@ -1287,4 +1290,4 @@ TEST_F(SegmentReaderWriterTest, TestBloomFilterIndexUniqueModel) { EXPECT_TRUE(column_contains_index(seg2->footer().columns(3), BLOOM_FILTER_INDEX)); } } // namespace segment_v2 -} // namespace doris +} // namespace doris \ No newline at end of file diff --git a/be/test/olap/tablet_meta_test.cpp b/be/test/olap/tablet_meta_test.cpp index fffe32f382f0ce..aa46ce62504ed0 100644 --- a/be/test/olap/tablet_meta_test.cpp +++ b/be/test/olap/tablet_meta_test.cpp @@ -19,8 +19,10 @@ #include +#include #include +#include "olap/tablet_schema.h" #include "testutil/mock_rowset.h" namespace doris { @@ -60,8 +62,8 @@ TEST(TabletMetaTest, TestReviseMeta) { RowsetMetaSharedPtr meta_ptr = std::make_shared(); meta_ptr->init_from_pb(rs_meta_pb); RowsetSharedPtr rowset_ptr; - TabletSchema schema; - MockRowset::create_rowset(&schema, "", meta_ptr, &rowset_ptr, false); + TabletSchemaSPtr schema = std::make_shared(); + MockRowset::create_rowset(schema, "", meta_ptr, &rowset_ptr, false); src_rowsets.push_back(rowset_ptr); tablet_meta.add_rs_meta(rowset_ptr->rowset_meta()); } diff --git a/be/test/olap/tablet_test.cpp b/be/test/olap/tablet_test.cpp index 077190c3c17344..2156ca87ccf688 100644 --- a/be/test/olap/tablet_test.cpp +++ b/be/test/olap/tablet_test.cpp @@ -26,6 +26,7 @@ #include "olap/storage_engine.h" #include "olap/storage_policy_mgr.h" #include "olap/tablet_meta.h" +#include "olap/tablet_schema_cache.h" #include "testutil/mock_rowset.h" #include "util/time.h" @@ -379,19 +380,21 @@ TEST_F(TestTablet, rowset_tree_update) { RowsetMetaSharedPtr rsm1(new RowsetMeta()); init_rs_meta(rsm1, 6, 7, convert_key_bounds({{"100", "200"}, {"300", "400"}})); + rsm1->set_tablet_schema(tablet->tablet_schema()); RowsetId id1; id1.init(10010); RowsetSharedPtr rs_ptr1; - MockRowset::create_rowset(&tablet_meta->tablet_schema(), "", rsm1, &rs_ptr1, false); + MockRowset::create_rowset(tablet->tablet_schema(), "", rsm1, &rs_ptr1, false); tablet->add_inc_rowset(rs_ptr1); RowsetMetaSharedPtr rsm2(new RowsetMeta()); init_rs_meta(rsm2, 8, 8, convert_key_bounds({{"500", "999"}})); + rsm2->set_tablet_schema(tablet->tablet_schema()); RowsetId id2; id2.init(10086); rsm2->set_rowset_id(id2); RowsetSharedPtr rs_ptr2; - MockRowset::create_rowset(&tablet_meta->tablet_schema(), "", rsm2, &rs_ptr2, false); + MockRowset::create_rowset(tablet->tablet_schema(), "", rsm2, &rs_ptr2, false); tablet->add_inc_rowset(rs_ptr2); RowLocation loc; diff --git a/be/test/olap/txn_manager_test.cpp b/be/test/olap/txn_manager_test.cpp index 18917353b998f9..88b33c2a514f91 100644 --- a/be/test/olap/txn_manager_test.cpp +++ b/be/test/olap/txn_manager_test.cpp @@ -132,10 +132,10 @@ class TxnManagerTest : public testing::Test { RowsetMetaSharedPtr rowset_meta(new RowsetMeta()); rowset_meta->init_from_json(_json_rowset_meta); EXPECT_EQ(rowset_meta->rowset_id(), rowset_id); - EXPECT_EQ(Status::OK(), RowsetFactory::create_rowset(_schema.get(), rowset_meta_path, - rowset_meta, &_rowset)); - EXPECT_EQ(Status::OK(), RowsetFactory::create_rowset(_schema.get(), rowset_meta_path, - rowset_meta, &_rowset_same_id)); + EXPECT_EQ(Status::OK(), + RowsetFactory::create_rowset(_schema, rowset_meta_path, rowset_meta, &_rowset)); + EXPECT_EQ(Status::OK(), RowsetFactory::create_rowset(_schema, rowset_meta_path, rowset_meta, + &_rowset_same_id)); // init rowset meta 2 _json_rowset_meta = ""; @@ -150,7 +150,7 @@ class TxnManagerTest : public testing::Test { RowsetMetaSharedPtr rowset_meta2(new RowsetMeta()); rowset_meta2->init_from_json(_json_rowset_meta); EXPECT_EQ(rowset_meta2->rowset_id(), rowset_id); - EXPECT_EQ(Status::OK(), RowsetFactory::create_rowset(_schema.get(), rowset_meta_path_2, + EXPECT_EQ(Status::OK(), RowsetFactory::create_rowset(_schema, rowset_meta_path_2, rowset_meta2, &_rowset_diff_id)); _tablet_uid = TabletUid(10, 10); } @@ -170,7 +170,7 @@ class TxnManagerTest : public testing::Test { SchemaHash schema_hash = 333; TabletUid _tablet_uid {0, 0}; PUniqueId load_id; - std::unique_ptr _schema; + TabletSchemaSPtr _schema; RowsetSharedPtr _rowset; RowsetSharedPtr _rowset_same_id; RowsetSharedPtr _rowset_diff_id; diff --git a/be/test/testutil/mock_rowset.h b/be/test/testutil/mock_rowset.h index 7a1bc4a746b6f5..71a17876fa7287 100644 --- a/be/test/testutil/mock_rowset.h +++ b/be/test/testutil/mock_rowset.h @@ -18,6 +18,7 @@ #pragma once #include "olap/rowset/rowset.h" +#include "olap/tablet_schema.h" namespace doris { @@ -64,7 +65,7 @@ class MockRowset : public Rowset { return Rowset::get_segments_key_bounds(segments_key_bounds); } - static Status create_rowset(const TabletSchema* schema, const std::string& rowset_path, + static Status create_rowset(TabletSchemaSPtr schema, const std::string& rowset_path, RowsetMetaSharedPtr rowset_meta, RowsetSharedPtr* rowset, bool is_mem_rowset = false) { rowset->reset(new MockRowset(schema, rowset_path, rowset_meta)); @@ -73,7 +74,7 @@ class MockRowset : public Rowset { } protected: - MockRowset(const TabletSchema* schema, const std::string& rowset_path, + MockRowset(TabletSchemaSPtr schema, const std::string& rowset_path, RowsetMetaSharedPtr rowset_meta) : Rowset(schema, rowset_path, rowset_meta) {} diff --git a/be/test/testutil/run_all_tests.cpp b/be/test/testutil/run_all_tests.cpp index b84cc6c375723f..a3a4ce8b7373ed 100644 --- a/be/test/testutil/run_all_tests.cpp +++ b/be/test/testutil/run_all_tests.cpp @@ -32,6 +32,7 @@ int main(int argc, char** argv) { doris::MemTrackerLimiter* process_mem_tracker = new doris::MemTrackerLimiter(-1, "Process"); doris::ExecEnv::GetInstance()->set_process_mem_tracker(process_mem_tracker); doris::thread_context()->_thread_mem_tracker_mgr->init(); + doris::TabletSchemaCache::create_global_schema_cache(); doris::StoragePageCache::create_global_cache(1 << 30, 10); doris::SegmentLoader::create_global_instance(1000); std::string conf = std::string(getenv("DORIS_HOME")) + "/conf/be.conf";