diff --git a/.asf.yaml b/.asf.yaml index e71e55de23fc83c..6ff16967c2e4157 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -49,7 +49,6 @@ github: strict: false contexts: - License Check - - Clang Formatter - CheckStyle - P0 Regression (Doris Regression) - External Regression (Doris External Regression) @@ -87,7 +86,6 @@ github: strict: false contexts: - License Check - - Clang Formatter - CheckStyle - Build Broker - ShellCheck @@ -109,7 +107,6 @@ github: strict: false contexts: - License Check - - Clang Formatter - CheckStyle - P0 Regression (Doris Regression) - External Regression (Doris External Regression) diff --git a/.github/workflows/clang-format.yml b/.github/workflows/clang-format.yml index ef39fa6c45fe11c..f49f1987b9ed41b 100644 --- a/.github/workflows/clang-format.yml +++ b/.github/workflows/clang-format.yml @@ -67,6 +67,11 @@ jobs: git checkout 6adbe14579e5b8e19eb3e31e5ff2479f3bd302c7 popd &>/dev/null + - name: Install Python dependencies + uses: actions/setup-python@v5 + with: + python-version: '3.10' # Adjust if needed + - name: "Format it!" if: ${{ steps.filter.outputs.changes == 'true' }} uses: ./.github/actions/clang-format-lint-action diff --git a/.github/workflows/code-checks.yml b/.github/workflows/code-checks.yml index 2762480a0cb5669..4fe4090b516a1ea 100644 --- a/.github/workflows/code-checks.yml +++ b/.github/workflows/code-checks.yml @@ -50,7 +50,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: sh_checker_comment: true - sh_checker_exclude: .git .github ^docker ^thirdparty/src ^thirdparty/installed ^ui ^docs/node_modules ^tools/clickbench-tools ^extension ^output ^fs_brokers/apache_hdfs_broker/output (^|.*/)Dockerfile$ ^be/src/apache-orc ^be/src/clucene ^pytest + sh_checker_exclude: .git .github ^docker ^thirdparty/src ^thirdparty/installed ^ui ^docs/node_modules ^tools/clickbench-tools ^extension ^output ^fs_brokers/apache_hdfs_broker/output (^|.*/)Dockerfile$ ^be/src/apache-orc ^be/src/clucene ^pytest ^samples preparation: name: "Clang Tidy Preparation" diff --git a/.github/workflows/scope-label.yml b/.github/workflows/scope-label.yml index 77f0e50a44f705c..ba88dbbb02f7c09 100644 --- a/.github/workflows/scope-label.yml +++ b/.github/workflows/scope-label.yml @@ -35,7 +35,7 @@ jobs: github.event_name == 'pull_request_target' && (github.event.action == 'opened' || github.event.action == 'synchronize') - uses: actions/labeler@v5.5.0 + uses: actions/labeler@2.2.0 with: repo-token: ${{ secrets.GITHUB_TOKEN }} configuration-path: .github/workflows/labeler/scope-label-conf.yml diff --git a/be/src/cloud/cloud_meta_mgr.cpp b/be/src/cloud/cloud_meta_mgr.cpp index bc6c3a0f098e7e8..8138ea52421102c 100644 --- a/be/src/cloud/cloud_meta_mgr.cpp +++ b/be/src/cloud/cloud_meta_mgr.cpp @@ -715,7 +715,10 @@ Status CloudMetaMgr::prepare_rowset(const RowsetMeta& rs_meta, RowsetMetaSharedPtr* existed_rs_meta) { VLOG_DEBUG << "prepare rowset, tablet_id: " << rs_meta.tablet_id() << ", rowset_id: " << rs_meta.rowset_id() << " txn_id: " << rs_meta.txn_id(); - + { + Status ret_st; + TEST_INJECTION_POINT_RETURN_WITH_VALUE("CloudMetaMgr::prepare_rowset", ret_st); + } CreateRowsetRequest req; CreateRowsetResponse resp; req.set_cloud_unique_id(config::cloud_unique_id); @@ -741,6 +744,10 @@ Status CloudMetaMgr::commit_rowset(const RowsetMeta& rs_meta, RowsetMetaSharedPtr* existed_rs_meta) { VLOG_DEBUG << "commit rowset, tablet_id: " << rs_meta.tablet_id() << ", rowset_id: " << rs_meta.rowset_id() << " txn_id: " << rs_meta.txn_id(); + { + Status ret_st; + TEST_INJECTION_POINT_RETURN_WITH_VALUE("CloudMetaMgr::commit_rowset", ret_st); + } CreateRowsetRequest req; CreateRowsetResponse resp; req.set_cloud_unique_id(config::cloud_unique_id); @@ -841,6 +848,10 @@ static void send_stats_to_fe_async(const int64_t db_id, const int64_t txn_id, Status CloudMetaMgr::commit_txn(const StreamLoadContext& ctx, bool is_2pc) { VLOG_DEBUG << "commit txn, db_id: " << ctx.db_id << ", txn_id: " << ctx.txn_id << ", label: " << ctx.label << ", is_2pc: " << is_2pc; + { + Status ret_st; + TEST_INJECTION_POINT_RETURN_WITH_VALUE("CloudMetaMgr::commit_txn", ret_st); + } CommitTxnRequest req; CommitTxnResponse res; req.set_cloud_unique_id(config::cloud_unique_id); @@ -860,6 +871,10 @@ Status CloudMetaMgr::commit_txn(const StreamLoadContext& ctx, bool is_2pc) { Status CloudMetaMgr::abort_txn(const StreamLoadContext& ctx) { VLOG_DEBUG << "abort txn, db_id: " << ctx.db_id << ", txn_id: " << ctx.txn_id << ", label: " << ctx.label; + { + Status ret_st; + TEST_INJECTION_POINT_RETURN_WITH_VALUE("CloudMetaMgr::abort_txn", ret_st); + } AbortTxnRequest req; AbortTxnResponse res; req.set_cloud_unique_id(config::cloud_unique_id); @@ -879,6 +894,10 @@ Status CloudMetaMgr::abort_txn(const StreamLoadContext& ctx) { Status CloudMetaMgr::precommit_txn(const StreamLoadContext& ctx) { VLOG_DEBUG << "precommit txn, db_id: " << ctx.db_id << ", txn_id: " << ctx.txn_id << ", label: " << ctx.label; + { + Status ret_st; + TEST_INJECTION_POINT_RETURN_WITH_VALUE("CloudMetaMgr::precommit_txn", ret_st); + } PrecommitTxnRequest req; PrecommitTxnResponse res; req.set_cloud_unique_id(config::cloud_unique_id); diff --git a/be/src/cloud/cloud_stream_load_executor.cpp b/be/src/cloud/cloud_stream_load_executor.cpp index 1b8167c96ebd487..1352b4aac81a5f9 100644 --- a/be/src/cloud/cloud_stream_load_executor.cpp +++ b/be/src/cloud/cloud_stream_load_executor.cpp @@ -129,7 +129,7 @@ void CloudStreamLoadExecutor::rollback_txn(StreamLoadContext* ctx) { std::stringstream ss; ss << "db_id=" << ctx->db_id << " txn_id=" << ctx->txn_id << " label=" << ctx->label; std::string op_info = ss.str(); - LOG(INFO) << "rollback stream laod txn " << op_info; + LOG(INFO) << "rollback stream load txn " << op_info; TxnOpParamType topt = ctx->txn_id > 0 ? TxnOpParamType::WITH_TXN_ID : !ctx->label.empty() ? TxnOpParamType::WITH_LABEL : TxnOpParamType::ILLEGAL; diff --git a/be/src/cloud/injection_point_action.cpp b/be/src/cloud/injection_point_action.cpp index be90ee23afddae2..1880f14a3b77d1a 100644 --- a/be/src/cloud/injection_point_action.cpp +++ b/be/src/cloud/injection_point_action.cpp @@ -124,6 +124,17 @@ void register_suites() { *arg0 = Status::Corruption("test_file_segment_cache_corruption injection error"); }); }); + // curl be_ip:http_port/api/injection_point/apply_suite?name=test_cloud_meta_mgr_commit_txn' + suite_map.emplace("test_cloud_meta_mgr_commit_txn", [] { + auto* sp = SyncPoint::get_instance(); + sp->set_call_back("CloudMetaMgr::commit_txn", [](auto&& args) { + LOG(INFO) << "injection CloudMetaMgr::commit_txn"; + auto* arg0 = try_any_cast_ret(args); + arg0->first = Status::InternalError( + "test_file_segment_cache_corruption injection error"); + arg0->second = true; + }); + }); } void set_sleep(const std::string& point, HttpRequest* req) { @@ -139,7 +150,8 @@ void set_sleep(const std::string& point, HttpRequest* req) { } } auto sp = SyncPoint::get_instance(); - sp->set_call_back(point, [duration](auto&& args) { + sp->set_call_back(point, [point, duration](auto&& args) { + LOG(INFO) << "injection point hit, point=" << point << " sleep milliseconds=" << duration; std::this_thread::sleep_for(std::chrono::milliseconds(duration)); }); HttpChannel::send_reply(req, HttpStatus::OK, "OK"); @@ -147,8 +159,9 @@ void set_sleep(const std::string& point, HttpRequest* req) { void set_return(const std::string& point, HttpRequest* req) { auto sp = SyncPoint::get_instance(); - sp->set_call_back(point, [](auto&& args) { + sp->set_call_back(point, [point](auto&& args) { try { + LOG(INFO) << "injection point hit, point=" << point << " return void"; auto pred = try_any_cast(args.back()); *pred = true; } catch (const std::bad_any_cast&) { @@ -160,8 +173,9 @@ void set_return(const std::string& point, HttpRequest* req) { void set_return_ok(const std::string& point, HttpRequest* req) { auto sp = SyncPoint::get_instance(); - sp->set_call_back(point, [](auto&& args) { + sp->set_call_back(point, [point](auto&& args) { try { + LOG(INFO) << "injection point hit, point=" << point << " return ok"; auto* pair = try_any_cast_ret(args); pair->first = Status::OK(); pair->second = true; @@ -188,8 +202,9 @@ void set_return_error(const std::string& point, HttpRequest* req) { } auto sp = SyncPoint::get_instance(); - sp->set_call_back(point, [code](auto&& args) { + sp->set_call_back(point, [code, point](auto&& args) { try { + LOG(INFO) << "injection point hit, point=" << point << " return error code=" << code; auto* pair = try_any_cast_ret(args); pair->first = Status::Error(code, "injected error"); pair->second = true; @@ -243,7 +258,7 @@ void handle_clear(HttpRequest* req) { HttpChannel::send_reply(req, HttpStatus::OK, "OK"); } -void handle_suite(HttpRequest* req) { +void handle_apply_suite(HttpRequest* req) { auto& suite = req->param("name"); if (suite.empty()) { HttpChannel::send_reply(req, HttpStatus::BAD_REQUEST, "empty suite name"); @@ -253,10 +268,11 @@ void handle_suite(HttpRequest* req) { std::call_once(register_suites_once, register_suites); if (auto it = suite_map.find(suite); it != suite_map.end()) { it->second(); // set injection callbacks - HttpChannel::send_reply(req, HttpStatus::OK, "OK"); + HttpChannel::send_reply(req, HttpStatus::OK, "OK apply suite " + suite + "\n"); return; } - HttpChannel::send_reply(req, HttpStatus::INTERNAL_SERVER_ERROR, "unknown suite: " + suite); + HttpChannel::send_reply(req, HttpStatus::INTERNAL_SERVER_ERROR, + "unknown suite: " + suite + "\n"); } void handle_enable(HttpRequest* req) { @@ -273,6 +289,37 @@ void handle_disable(HttpRequest* req) { InjectionPointAction::InjectionPointAction() = default; +// +// enable/disable injection point +// ``` +// curl "be_ip:http_port/api/injection_point/enable" +// curl "be_ip:http_port/api/injection_point/disable" +// ``` +// +// clear all injection points +// ``` +// curl "be_ip:http_port/api/injection_point/clear" +// ``` +// +// apply/activate specific suite with registered action, see `register_suites()` for more details +// ``` +// curl "be_ip:http_port/api/injection_point/apply_suite?name=${suite_name}" +// ``` +// +// set predifined action for specific injection point, supported actions are: +// * sleep: for injection point with callback, accepted param is `duration` in milliseconds +// * return: for injection point without return value (return void) +// * return_ok: for injection point with return value, always return Status::OK +// * return_error: for injection point with return value, accepted param is `code`, +// which is an int, valid values can be found in status.h, e.g. -235 or -230, +// if `code` is not present return Status::InternalError +// ``` +// curl "be_ip:http_port/api/injection_point/set?name=${injection_point_name}&behavior=sleep&duration=${x_millsec}" # sleep x millisecs +// curl "be_ip:http_port/api/injection_point/set?name=${injection_point_name}&behavior=return" # return void +// curl "be_ip:http_port/api/injection_point/set?name=${injection_point_name}&behavior=return_ok" # return ok +// curl "be_ip:http_port/api/injection_point/set?name=${injection_point_name}&behavior=return_error" # internal error +// curl "be_ip:http_port/api/injection_point/set?name=${injection_point_name}&behavior=return_error&code=${code}" # -235 +// ``` void InjectionPointAction::handle(HttpRequest* req) { LOG(INFO) << "handle InjectionPointAction " << req->debug_string(); auto& op = req->param("op"); @@ -283,7 +330,7 @@ void InjectionPointAction::handle(HttpRequest* req) { handle_clear(req); return; } else if (op == "apply_suite") { - handle_suite(req); + handle_apply_suite(req); return; } else if (op == "enable") { handle_enable(req); diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 6c38a95bdc91c6b..67a12fc02f1d700 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -296,7 +296,7 @@ DEFINE_mInt32(default_num_rows_per_column_file_block, "1024"); // pending data policy DEFINE_mInt32(pending_data_expire_time_sec, "1800"); // inc_rowset snapshot rs sweep time interval -DEFINE_mInt32(tablet_rowset_stale_sweep_time_sec, "300"); +DEFINE_mInt32(tablet_rowset_stale_sweep_time_sec, "600"); // tablet stale rowset sweep by threshold size DEFINE_Bool(tablet_rowset_stale_sweep_by_size, "false"); DEFINE_mInt32(tablet_rowset_stale_sweep_threshold_size, "100"); @@ -966,10 +966,6 @@ DEFINE_Int32(pipeline_executor_size, "0"); DEFINE_Bool(enable_workload_group_for_scan, "false"); DEFINE_mInt64(workload_group_scan_task_wait_timeout_ms, "10000"); -// Temp config. True to use optimization for bitmap_index apply predicate except leaf node of the and node. -// Will remove after fully test. -DEFINE_Bool(enable_index_apply_preds_except_leafnode_of_andnode, "true"); - DEFINE_mDouble(variant_ratio_of_defaults_as_sparse_column, "1"); DEFINE_mInt64(variant_threshold_rows_to_estimate_sparse_column, "2048"); DEFINE_mBool(variant_throw_exeception_on_invalid_json, "false"); diff --git a/be/src/common/config.h b/be/src/common/config.h index 017eab0302ac901..f381abca4bcf63e 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -1026,10 +1026,6 @@ DECLARE_Bool(enable_debug_points); DECLARE_Int32(pipeline_executor_size); -// Temp config. True to use optimization for bitmap_index apply predicate except leaf node of the and node. -// Will remove after fully test. -DECLARE_Bool(enable_index_apply_preds_except_leafnode_of_andnode); - // block file cache DECLARE_Bool(enable_file_cache); // format: [{"path":"/path/to/file_cache","total_size":21474836480,"query_limit":10737418240}] diff --git a/be/src/exec/decompressor.cpp b/be/src/exec/decompressor.cpp index 9102765458bdf15..a830eb3cbe70b59 100644 --- a/be/src/exec/decompressor.cpp +++ b/be/src/exec/decompressor.cpp @@ -468,7 +468,7 @@ Status Lz4BlockDecompressor::decompress(uint8_t* input, size_t input_len, size_t } std::size_t decompressed_large_block_len = 0; - do { + while (remaining_decompressed_large_block_len > 0) { // Check that input length should not be negative. if (input_len < sizeof(uint32_t)) { *more_input_bytes = sizeof(uint32_t) - input_len; @@ -505,8 +505,7 @@ Status Lz4BlockDecompressor::decompress(uint8_t* input, size_t input_len, size_t output_ptr += decompressed_small_block_len; remaining_decompressed_large_block_len -= decompressed_small_block_len; decompressed_large_block_len += decompressed_small_block_len; - - } while (remaining_decompressed_large_block_len > 0); + }; if (*more_input_bytes != 0) { // Need more input buffer @@ -586,7 +585,7 @@ Status SnappyBlockDecompressor::decompress(uint8_t* input, size_t input_len, } std::size_t decompressed_large_block_len = 0; - do { + while (remaining_decompressed_large_block_len > 0) { // Check that input length should not be negative. if (input_len < sizeof(uint32_t)) { *more_input_bytes = sizeof(uint32_t) - input_len; @@ -630,8 +629,7 @@ Status SnappyBlockDecompressor::decompress(uint8_t* input, size_t input_len, output_ptr += decompressed_small_block_len; remaining_decompressed_large_block_len -= decompressed_small_block_len; decompressed_large_block_len += decompressed_small_block_len; - - } while (remaining_decompressed_large_block_len > 0); + }; if (*more_input_bytes != 0) { // Need more input buffer diff --git a/be/src/exec/olap_common.h b/be/src/exec/olap_common.h index 76d5ec83d7d04b5..9ba1b81d1b8f11e 100644 --- a/be/src/exec/olap_common.h +++ b/be/src/exec/olap_common.h @@ -113,16 +113,8 @@ class ColumnValueRange { Status add_range(SQLFilterOp op, CppType value); - Status add_compound_value(SQLFilterOp op, CppType value); - - bool is_in_compound_value_range() const; - - Status add_match_value(MatchType match_type, const CppType& value); - bool is_fixed_value_range() const; - bool is_match_value_range() const; - bool is_scope_value_range() const; bool is_empty_value_range() const; @@ -195,15 +187,9 @@ class ColumnValueRange { size_t get_fixed_value_size() const { return _fixed_values.size(); } void to_olap_filter(std::vector& filters) { - if (is_fixed_value_range() || is_match_value_range()) { + if (is_fixed_value_range()) { // 1. convert to in filter condition - if (is_fixed_value_range()) { - to_in_condition(filters, true); - } - - if (is_match_value_range()) { - to_match_condition(filters); - } + to_in_condition(filters, true); } else if (_low_value < _high_value) { // 2. convert to min max filter condition TCondition null_pred; @@ -277,59 +263,6 @@ class ColumnValueRange { } } - void to_condition_in_compound(std::vector& filters) { - for (const auto& compound_value : _compound_values) { - TCondition condition; - condition.__set_column_name(_column_name); - if (compound_value.first == FILTER_LARGER) { - condition.__set_condition_op(">>"); - } else if (compound_value.first == FILTER_LARGER_OR_EQUAL) { - condition.__set_condition_op(">="); - } else if (compound_value.first == FILTER_LESS) { - condition.__set_condition_op("<<"); - } else if (compound_value.first == FILTER_LESS_OR_EQUAL) { - condition.__set_condition_op("<="); - } else if (compound_value.first == FILTER_IN) { - condition.__set_condition_op("*="); - } else if (compound_value.first == FILTER_NOT_IN) { - condition.__set_condition_op("!*="); - } - for (const auto& value : compound_value.second) { - condition.condition_values.push_back( - cast_to_string(value, _scale)); - } - if (condition.condition_values.size() != 0) { - filters.push_back(std::move(condition)); - } - } - } - - void to_match_condition(std::vector& filters) { - for (const auto& value : _match_values) { - TCondition condition; - condition.__set_column_name(_column_name); - - if (value.first == MatchType::MATCH_ANY) { - condition.__set_condition_op("match_any"); - } else if (value.first == MatchType::MATCH_ALL) { - condition.__set_condition_op("match_all"); - } else if (value.first == MatchType::MATCH_PHRASE) { - condition.__set_condition_op("match_phrase"); - } else if (value.first == MatchType::MATCH_PHRASE_PREFIX) { - condition.__set_condition_op("match_phrase_prefix"); - } else if (value.first == MatchType::MATCH_REGEXP) { - condition.__set_condition_op("match_regexp"); - } else if (value.first == MatchType::MATCH_PHRASE_EDGE) { - condition.__set_condition_op("match_phrase_edge"); - } - condition.condition_values.push_back( - cast_to_string(value.second, _scale)); - if (condition.condition_values.size() != 0) { - filters.push_back(std::move(condition)); - } - } - } - void set_whole_value_range() { _fixed_values.clear(); _low_value = TYPE_MIN; @@ -386,16 +319,6 @@ class ColumnValueRange { static_cast(range.add_range(op, *value)); } - static void add_compound_value_range(ColumnValueRange& range, SQLFilterOp op, - CppType* value) { - static_cast(range.add_compound_value(op, *value)); - } - - static void add_match_value_range(ColumnValueRange& range, MatchType match_type, - CppType* match_value) { - static_cast(range.add_match_value(match_type, *match_value)); - } - static ColumnValueRange create_empty_column_value_range(bool is_nullable_col, int precision, int scale) { @@ -422,8 +345,7 @@ class ColumnValueRange { CppType _high_value; // Column's high value, open interval at right SQLFilterOp _low_op; SQLFilterOp _high_op; - std::set _fixed_values; // Column's fixed int value - std::set> _match_values; // match value using in full-text search + std::set _fixed_values; // Column's fixed int value bool _is_nullable_col; bool _contain_null; @@ -440,8 +362,6 @@ class ColumnValueRange { primitive_type == PrimitiveType::TYPE_DATETIME || primitive_type == PrimitiveType::TYPE_DATETIMEV2; - // range value except leaf node of and node in compound expr tree - std::map> _compound_values; bool _marked_runtime_filter_predicate = false; }; @@ -592,28 +512,6 @@ Status ColumnValueRange::add_fixed_value(const CppType& value) { return Status::OK(); } -template -Status ColumnValueRange::add_compound_value(SQLFilterOp op, CppType value) { - _compound_values[op].insert(value); - _contain_null = false; - - _high_value = TYPE_MIN; - _low_value = TYPE_MAX; - return Status::OK(); -} - -template -Status ColumnValueRange::add_match_value(MatchType match_type, - const CppType& value) { - std::pair match_value(match_type, value); - _match_values.insert(match_value); - _contain_null = false; - - // _high_value = TYPE_MIN; - // _low_value = TYPE_MAX; - return Status::OK(); -} - template void ColumnValueRange::remove_fixed_value(const CppType& value) { _fixed_values.erase(value); @@ -624,16 +522,6 @@ bool ColumnValueRange::is_fixed_value_range() const { return _fixed_values.size() != 0; } -template -bool ColumnValueRange::is_in_compound_value_range() const { - return _compound_values.size() != 0; -} - -template -bool ColumnValueRange::is_match_value_range() const { - return _match_values.size() != 0; -} - template bool ColumnValueRange::is_scope_value_range() const { return _high_value > _low_value; @@ -645,8 +533,7 @@ bool ColumnValueRange::is_empty_value_range() const { return true; } - return (!is_fixed_value_range() && !is_scope_value_range() && !contain_null() && - !is_match_value_range()); + return (!is_fixed_value_range() && !is_scope_value_range() && !contain_null()); } template @@ -989,7 +876,7 @@ void ColumnValueRange::intersection(ColumnValueRange result_values; // 3. fixed_value intersection, fixed value range do not contain null - if (is_fixed_value_range() || range.is_fixed_value_range() || range.is_match_value_range()) { + if (is_fixed_value_range() || range.is_fixed_value_range()) { if (is_fixed_value_range() && range.is_fixed_value_range()) { set_intersection(_fixed_values.begin(), _fixed_values.end(), range._fixed_values.begin(), range._fixed_values.end(), @@ -1018,10 +905,6 @@ void ColumnValueRange::intersection(ColumnValueRange(add_match_value(value.first, value.second)); - } } else { set_empty_value_range(); } @@ -1151,21 +1034,6 @@ Status OlapScanKeys::extend_scan_key(ColumnValueRange& range, } } - // extend ScanKey with MatchValueRange - if (range.is_match_value_range() && _begin_scan_keys.empty()) { - _begin_scan_keys.emplace_back(); - _begin_scan_keys.back().add_value( - cast_to_string(type_limit::min(), 0)); - _end_scan_keys.emplace_back(); - _end_scan_keys.back().add_value( - cast_to_string(type_limit::max(), 0)); - _begin_include = true; - _end_include = true; - *exact_value = false; - // not empty, do nothing - return Status::OK(); - } - // 3.1 extend ScanKey with FixedValueRange if (range.is_fixed_value_range()) { // 3.1.1 construct num of fixed value ScanKey (begin_key == end_key) diff --git a/be/src/exec/schema_scanner.cpp b/be/src/exec/schema_scanner.cpp index b5cd484b650b3f1..cce150670353d9d 100644 --- a/be/src/exec/schema_scanner.cpp +++ b/be/src/exec/schema_scanner.cpp @@ -28,6 +28,7 @@ #include "exec/schema_scanner/schema_active_queries_scanner.h" #include "exec/schema_scanner/schema_backend_active_tasks.h" +#include "exec/schema_scanner/schema_catalog_meta_cache_stats_scanner.h" #include "exec/schema_scanner/schema_charsets_scanner.h" #include "exec/schema_scanner/schema_collations_scanner.h" #include "exec/schema_scanner/schema_columns_scanner.h" @@ -240,6 +241,8 @@ std::unique_ptr SchemaScanner::create(TSchemaTableType::type type return SchemaBackendWorkloadGroupResourceUsage::create_unique(); case TSchemaTableType::SCH_TABLE_PROPERTIES: return SchemaTablePropertiesScanner::create_unique(); + case TSchemaTableType::SCH_CATALOG_META_CACHE_STATISTICS: + return SchemaCatalogMetaCacheStatsScanner::create_unique(); default: return SchemaDummyScanner::create_unique(); break; diff --git a/be/src/exec/schema_scanner/schema_catalog_meta_cache_stats_scanner.cpp b/be/src/exec/schema_scanner/schema_catalog_meta_cache_stats_scanner.cpp new file mode 100644 index 000000000000000..4c067057729f215 --- /dev/null +++ b/be/src/exec/schema_scanner/schema_catalog_meta_cache_stats_scanner.cpp @@ -0,0 +1,135 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "exec/schema_scanner/schema_catalog_meta_cache_stats_scanner.h" + +#include "runtime/client_cache.h" +#include "runtime/exec_env.h" +#include "runtime/query_context.h" +#include "runtime/runtime_state.h" +#include "util/thrift_rpc_helper.h" +#include "vec/common/string_ref.h" +#include "vec/core/block.h" +#include "vec/data_types/data_type_factory.hpp" + +namespace doris { +std::vector SchemaCatalogMetaCacheStatsScanner::_s_tbls_columns = { + {"CATALOG_NAME", TYPE_STRING, sizeof(StringRef), true}, + {"CACHE_NAME", TYPE_STRING, sizeof(StringRef), true}, + {"METRIC_NAME", TYPE_STRING, sizeof(StringRef), true}, + {"METRIC_VALUE", TYPE_STRING, sizeof(StringRef), true}, +}; + +SchemaCatalogMetaCacheStatsScanner::SchemaCatalogMetaCacheStatsScanner() + : SchemaScanner(_s_tbls_columns, TSchemaTableType::SCH_CATALOG_META_CACHE_STATISTICS) {} + +SchemaCatalogMetaCacheStatsScanner::~SchemaCatalogMetaCacheStatsScanner() {} + +Status SchemaCatalogMetaCacheStatsScanner::start(RuntimeState* state) { + _block_rows_limit = state->batch_size(); + _rpc_timeout = state->execution_timeout() * 1000; + _fe_addr = state->get_query_ctx()->current_connect_fe; + return Status::OK(); +} + +Status SchemaCatalogMetaCacheStatsScanner::_get_meta_cache_from_fe() { + TSchemaTableRequestParams schema_table_request_params; + for (int i = 0; i < _s_tbls_columns.size(); i++) { + schema_table_request_params.__isset.columns_name = true; + schema_table_request_params.columns_name.emplace_back(_s_tbls_columns[i].name); + } + schema_table_request_params.__set_current_user_ident(*_param->common_param->current_user_ident); + + TFetchSchemaTableDataRequest request; + request.__set_schema_table_name(TSchemaTableName::CATALOG_META_CACHE_STATS); + request.__set_schema_table_params(schema_table_request_params); + + TFetchSchemaTableDataResult result; + + RETURN_IF_ERROR(ThriftRpcHelper::rpc( + _fe_addr.hostname, _fe_addr.port, + [&request, &result](FrontendServiceConnection& client) { + client->fetchSchemaTableData(result, request); + }, + _rpc_timeout)); + + Status status(Status::create(result.status)); + if (!status.ok()) { + LOG(WARNING) << "fetch catalog meta cache stats from FE(" << _fe_addr.hostname + << ") failed, errmsg=" << status; + return status; + } + std::vector result_data = result.data_batch; + + _block = vectorized::Block::create_unique(); + for (int i = 0; i < _s_tbls_columns.size(); ++i) { + TypeDescriptor descriptor(_s_tbls_columns[i].type); + auto data_type = vectorized::DataTypeFactory::instance().create_data_type(descriptor, true); + _block->insert(vectorized::ColumnWithTypeAndName(data_type->create_column(), data_type, + _s_tbls_columns[i].name)); + } + + _block->reserve(_block_rows_limit); + + if (result_data.size() > 0) { + int col_size = result_data[0].column_value.size(); + if (col_size != _s_tbls_columns.size()) { + return Status::InternalError( + "catalog meta cache stats schema is not match for FE and BE"); + } + } + + for (int i = 0; i < result_data.size(); i++) { + TRow row = result_data[i]; + for (int j = 0; j < _s_tbls_columns.size(); j++) { + RETURN_IF_ERROR(insert_block_column(row.column_value[j], j, _block.get(), + _s_tbls_columns[j].type)); + } + } + return Status::OK(); +} + +Status SchemaCatalogMetaCacheStatsScanner::get_next_block_internal(vectorized::Block* block, + bool* eos) { + if (!_is_init) { + return Status::InternalError("Used before initialized."); + } + + if (nullptr == block || nullptr == eos) { + return Status::InternalError("input pointer is nullptr."); + } + + if (_block == nullptr) { + RETURN_IF_ERROR(_get_meta_cache_from_fe()); + _total_rows = _block->rows(); + } + + if (_row_idx == _total_rows) { + *eos = true; + return Status::OK(); + } + + int current_batch_rows = std::min(_block_rows_limit, _total_rows - _row_idx); + vectorized::MutableBlock mblock = vectorized::MutableBlock::build_mutable_block(block); + RETURN_IF_ERROR(mblock.add_rows(_block.get(), _row_idx, current_batch_rows)); + _row_idx += current_batch_rows; + + *eos = _row_idx == _total_rows; + return Status::OK(); +} + +} // namespace doris diff --git a/be/src/exec/schema_scanner/schema_catalog_meta_cache_stats_scanner.h b/be/src/exec/schema_scanner/schema_catalog_meta_cache_stats_scanner.h new file mode 100644 index 000000000000000..dd6c908deb326d7 --- /dev/null +++ b/be/src/exec/schema_scanner/schema_catalog_meta_cache_stats_scanner.h @@ -0,0 +1,54 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include "common/status.h" +#include "exec/schema_scanner.h" + +namespace doris { +class RuntimeState; +namespace vectorized { +class Block; +} // namespace vectorized + +class SchemaCatalogMetaCacheStatsScanner : public SchemaScanner { + ENABLE_FACTORY_CREATOR(SchemaCatalogMetaCacheStatsScanner); + +public: + SchemaCatalogMetaCacheStatsScanner(); + ~SchemaCatalogMetaCacheStatsScanner() override; + + Status start(RuntimeState* state) override; + Status get_next_block_internal(vectorized::Block* block, bool* eos) override; + + static std::vector _s_tbls_columns; + +private: + Status _get_meta_cache_from_fe(); + + TNetworkAddress _fe_addr; + + int _block_rows_limit = 4096; + int _row_idx = 0; + int _total_rows = 0; + std::unique_ptr _block = nullptr; + int _rpc_timeout = 3000; +}; +}; // namespace doris diff --git a/be/src/olap/compaction.cpp b/be/src/olap/compaction.cpp index 1fed2253b69fbf1..963b885a26b4add 100644 --- a/be/src/olap/compaction.cpp +++ b/be/src/olap/compaction.cpp @@ -686,9 +686,9 @@ Status Compaction::do_inverted_index_compaction() { << st; return st; } - for (const auto& writer : inverted_index_file_writers) { - writer->set_file_writer_opts(ctx.get_file_writer_options()); - } + } + for (const auto& writer : inverted_index_file_writers) { + writer->set_file_writer_opts(ctx.get_file_writer_options()); } // use tmp file dir to store index files diff --git a/be/src/olap/cumulative_compaction_time_series_policy.cpp b/be/src/olap/cumulative_compaction_time_series_policy.cpp index 64e51c77641311a..6fa4b8d014313f0 100644 --- a/be/src/olap/cumulative_compaction_time_series_policy.cpp +++ b/be/src/olap/cumulative_compaction_time_series_policy.cpp @@ -27,14 +27,11 @@ namespace doris { uint32_t TimeSeriesCumulativeCompactionPolicy::calc_cumulative_compaction_score(Tablet* tablet) { uint32_t score = 0; - uint32_t level0_score = 0; bool base_rowset_exist = false; const int64_t point = tablet->cumulative_layer_point(); - int64_t level0_total_size = 0; RowsetMetaSharedPtr first_meta; int64_t first_version = INT64_MAX; - std::list checked_rs_metas; // NOTE: tablet._meta_lock is hold auto& rs_metas = tablet->tablet_meta()->all_rs_metas(); // check the base rowset and collect the rowsets of cumulative part @@ -53,12 +50,6 @@ uint32_t TimeSeriesCumulativeCompactionPolicy::calc_cumulative_compaction_score( } else { // collect the rowsets of cumulative part score += rs_meta->get_compaction_score(); - if (rs_meta->compaction_level() == 0) { - level0_total_size += rs_meta->total_disk_size(); - level0_score += rs_meta->get_compaction_score(); - } else { - checked_rs_metas.push_back(rs_meta); - } } } @@ -73,64 +64,7 @@ uint32_t TimeSeriesCumulativeCompactionPolicy::calc_cumulative_compaction_score( return 0; } - // Condition 1: the size of input files for compaction meets the requirement of parameter compaction_goal_size - int64_t compaction_goal_size_mbytes = - tablet->tablet_meta()->time_series_compaction_goal_size_mbytes(); - if (level0_total_size >= compaction_goal_size_mbytes * 1024 * 1024) { - return score; - } - - // Condition 2: the number of input files reaches the threshold specified by parameter compaction_file_count_threshold - if (level0_score >= tablet->tablet_meta()->time_series_compaction_file_count_threshold()) { - return score; - } - - // Condition 3: level1 achieve compaction_goal_size - if (tablet->tablet_meta()->time_series_compaction_level_threshold() >= 2) { - checked_rs_metas.sort([](const RowsetMetaSharedPtr& a, const RowsetMetaSharedPtr& b) { - return a->version().first < b->version().first; - }); - int32_t rs_meta_count = 0; - int64_t continuous_size = 0; - for (const auto& rs_meta : checked_rs_metas) { - rs_meta_count++; - continuous_size += rs_meta->total_disk_size(); - if (rs_meta_count >= 2) { - if (continuous_size >= compaction_goal_size_mbytes * 1024 * 1024) { - return score; - } - } - } - } - - int64_t now = UnixMillis(); - int64_t last_cumu = tablet->last_cumu_compaction_success_time(); - if (last_cumu != 0) { - int64_t cumu_interval = now - last_cumu; - - // Condition 4: the time interval between compactions exceeds the value specified by parameter _compaction_time_threshold_second - if (cumu_interval > - (tablet->tablet_meta()->time_series_compaction_time_threshold_seconds() * 1000)) { - return score; - } - } else if (score > 0) { - // If the compaction process has not been successfully executed, - // the condition for triggering compaction based on the last successful compaction time (condition 3) will never be met - tablet->set_last_cumu_compaction_success_time(now); - } - - // Condition 5: If there is a continuous set of empty rowsets, prioritize merging. - std::vector input_rowsets; - std::vector candidate_rowsets = - tablet->pick_candidate_rowsets_to_cumulative_compaction(); - tablet->calc_consecutive_empty_rowsets( - &input_rowsets, candidate_rowsets, - tablet->tablet_meta()->time_series_compaction_empty_rowsets_threshold()); - if (!input_rowsets.empty()) { - return score; - } - - return 0; + return score; } void TimeSeriesCumulativeCompactionPolicy::calculate_cumulative_point( diff --git a/be/src/olap/iterators.h b/be/src/olap/iterators.h index 4f103e6cad17b30..1d3c2ddf6b614f4 100644 --- a/be/src/olap/iterators.h +++ b/be/src/olap/iterators.h @@ -87,7 +87,6 @@ class StorageReadOptions { // reader's column predicate, nullptr if not existed // used to fiter rows in row block std::vector column_predicates; - std::vector column_predicates_except_leafnode_of_andnode; std::unordered_map> col_id_to_predicates; std::unordered_map> del_predicates_for_zone_map; TPushAggOp::type push_down_agg_type_opt = TPushAggOp::NONE; diff --git a/be/src/olap/rowset/beta_rowset_reader.cpp b/be/src/olap/rowset/beta_rowset_reader.cpp index e90b553896a6a41..5fdb2d7c41a681d 100644 --- a/be/src/olap/rowset/beta_rowset_reader.cpp +++ b/be/src/olap/rowset/beta_rowset_reader.cpp @@ -149,27 +149,6 @@ Status BetaRowsetReader::get_segment_iterators(RowsetReaderContext* read_context } } - if (_read_context->predicates_except_leafnode_of_andnode != nullptr) { - bool should_push_down = true; - bool should_push_down_value_predicates = _should_push_down_value_predicates(); - for (auto pred : *_read_context->predicates_except_leafnode_of_andnode) { - if (_rowset->keys_type() == UNIQUE_KEYS && !should_push_down_value_predicates && - !_read_context->tablet_schema->column(pred->column_id()).is_key()) { - VLOG_DEBUG << "do not push down except_leafnode_of_andnode value pred " - << pred->debug_string(); - should_push_down = false; - break; - } - } - - if (should_push_down) { - _read_options.column_predicates_except_leafnode_of_andnode.insert( - _read_options.column_predicates_except_leafnode_of_andnode.end(), - _read_context->predicates_except_leafnode_of_andnode->begin(), - _read_context->predicates_except_leafnode_of_andnode->end()); - } - } - // Take a delete-bitmap for each segment, the bitmap contains all deletes // until the max read version, which is read_context->version.second if (_read_context->delete_bitmap != nullptr) { diff --git a/be/src/olap/rowset/rowset_reader_context.h b/be/src/olap/rowset/rowset_reader_context.h index 2de76df80401859..0d4f5897772ad5c 100644 --- a/be/src/olap/rowset/rowset_reader_context.h +++ b/be/src/olap/rowset/rowset_reader_context.h @@ -54,7 +54,6 @@ struct RowsetReaderContext { // column name -> column predicate // adding column_name for predicate to make use of column selectivity const std::vector* predicates = nullptr; - const std::vector* predicates_except_leafnode_of_andnode = nullptr; // value column predicate in UNIQUE table const std::vector* value_predicates = nullptr; const std::vector* lower_bound_keys = nullptr; diff --git a/be/src/olap/rowset/segment_v2/segment.cpp b/be/src/olap/rowset/segment_v2/segment.cpp index 64f58e546c22bac..2556597419ef876 100644 --- a/be/src/olap/rowset/segment_v2/segment.cpp +++ b/be/src/olap/rowset/segment_v2/segment.cpp @@ -553,7 +553,7 @@ Status Segment::_create_column_readers(const SegmentFooterPB& footer) { vectorized::PathInData path; path.from_protobuf(spase_column_pb.column_path_info()); // Read from root column, so reader is nullptr - _sparse_column_tree[column.unique_id()].add( + _sparse_column_tree[unique_id].add( path.copy_pop_front(), SubcolumnReader {nullptr, vectorized::DataTypeFactory::instance().create_data_type( @@ -617,9 +617,10 @@ Status Segment::new_column_iterator_with_path(const TabletColumn& tablet_column, const auto* node = tablet_column.has_path_info() ? _sub_column_tree[unique_id].find_exact(relative_path) : nullptr; - const auto* sparse_node = tablet_column.has_path_info() - ? _sparse_column_tree[unique_id].find_exact(relative_path) - : nullptr; + const auto* sparse_node = + tablet_column.has_path_info() && _sparse_column_tree.contains(unique_id) + ? _sparse_column_tree[unique_id].find_exact(relative_path) + : nullptr; // Currently only compaction and checksum need to read flat leaves // They both use tablet_schema_with_merged_max_schema_version as read schema auto type_to_read_flat_leaves = [](ReaderType type) { diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.cpp b/be/src/olap/rowset/segment_v2/segment_iterator.cpp index af473bb5ecdd8d5..f0d9cc6605d767e 100644 --- a/be/src/olap/rowset/segment_v2/segment_iterator.cpp +++ b/be/src/olap/rowset/segment_v2/segment_iterator.cpp @@ -297,17 +297,6 @@ Status SegmentIterator::_init_impl(const StorageReadOptions& opts) { // Read options will not change, so that just resize here _block_rowids.resize(_opts.block_row_max); - // compound predicates - //TODO: need to check can_apply_predicate_safely in expr - /*_col_preds_except_leafnode_of_andnode.clear(); - for (const auto& predicate : opts.column_predicates_except_leafnode_of_andnode) { - if (!_segment->can_apply_predicate_safely(predicate->column_id(), predicate, *_schema, - _opts.io_ctx.reader_type)) { - continue; - } - _col_preds_except_leafnode_of_andnode.push_back(predicate); - }*/ - _remaining_conjunct_roots = opts.remaining_conjunct_roots; if (_schema->rowid_col_idx() > 0) { @@ -514,35 +503,41 @@ Status SegmentIterator::_get_row_ranges_by_column_conditions() { } RETURN_IF_ERROR(_apply_bitmap_index()); - RETURN_IF_ERROR(_apply_inverted_index()); - RETURN_IF_ERROR(_apply_index_expr()); - size_t input_rows = _row_bitmap.cardinality(); - for (auto it = _common_expr_ctxs_push_down.begin(); it != _common_expr_ctxs_push_down.end();) { - if ((*it)->all_expr_inverted_index_evaluated()) { - const auto* result = - (*it)->get_inverted_index_context()->get_inverted_index_result_for_expr( - (*it)->root().get()); - if (result != nullptr) { - _row_bitmap &= *result->get_data_bitmap(); - auto root = (*it)->root(); - auto iter_find = std::find(_remaining_conjunct_roots.begin(), - _remaining_conjunct_roots.end(), root); - if (iter_find != _remaining_conjunct_roots.end()) { - _remaining_conjunct_roots.erase(iter_find); + { + if (_opts.runtime_state && + _opts.runtime_state->query_options().enable_inverted_index_query) { + SCOPED_RAW_TIMER(&_opts.stats->inverted_index_filter_timer); + size_t input_rows = _row_bitmap.cardinality(); + RETURN_IF_ERROR(_apply_inverted_index()); + RETURN_IF_ERROR(_apply_index_expr()); + for (auto it = _common_expr_ctxs_push_down.begin(); + it != _common_expr_ctxs_push_down.end();) { + if ((*it)->all_expr_inverted_index_evaluated()) { + const auto* result = + (*it)->get_inverted_index_context()->get_inverted_index_result_for_expr( + (*it)->root().get()); + if (result != nullptr) { + _row_bitmap &= *result->get_data_bitmap(); + auto root = (*it)->root(); + auto iter_find = std::find(_remaining_conjunct_roots.begin(), + _remaining_conjunct_roots.end(), root); + if (iter_find != _remaining_conjunct_roots.end()) { + _remaining_conjunct_roots.erase(iter_find); + } + it = _common_expr_ctxs_push_down.erase(it); + } + } else { + ++it; } - it = _common_expr_ctxs_push_down.erase(it); } - } else { - ++it; - } - } - - _opts.stats->rows_inverted_index_filtered += (input_rows - _row_bitmap.cardinality()); - for (auto cid : _schema->column_ids()) { - bool result_true = _check_all_conditions_passed_inverted_index_for_column(cid); + _opts.stats->rows_inverted_index_filtered += (input_rows - _row_bitmap.cardinality()); + for (auto cid : _schema->column_ids()) { + bool result_true = _check_all_conditions_passed_inverted_index_for_column(cid); - if (result_true) { - _need_read_data_indices[cid] = false; + if (result_true) { + _need_read_data_indices[cid] = false; + } + } } } if (!_row_bitmap.isEmpty() && @@ -745,18 +740,7 @@ Status SegmentIterator::_extract_common_expr_columns(const vectorized::VExprSPtr return Status::OK(); } -bool SegmentIterator::_check_apply_by_inverted_index(ColumnId col_id) { - if (_opts.runtime_state && !_opts.runtime_state->query_options().enable_inverted_index_query) { - return false; - } - if (_inverted_index_iterators[col_id] == nullptr) { - //this column without inverted index - return false; - } - return true; -} - -bool SegmentIterator::_check_apply_by_inverted_index(ColumnPredicate* pred, bool pred_in_compound) { +bool SegmentIterator::_check_apply_by_inverted_index(ColumnPredicate* pred) { if (_opts.runtime_state && !_opts.runtime_state->query_options().enable_inverted_index_query) { return false; } @@ -776,11 +760,6 @@ bool SegmentIterator::_check_apply_by_inverted_index(ColumnPredicate* pred, bool if (predicate_param->marked_by_runtime_filter) { return false; } - // the in_list or not_in_list value count cannot be greater than threshold - int32_t threshold = _opts.runtime_state->query_options().in_list_value_count_threshold; - if (pred_in_compound && predicate_param->values.size() > threshold) { - return false; - } } // UNTOKENIZED strings exceed ignore_above, they are written as null, causing range query errors @@ -799,15 +778,11 @@ bool SegmentIterator::_check_apply_by_inverted_index(ColumnPredicate* pred, bool bool handle_by_fulltext = _column_has_fulltext_index(pred_column_id); if (handle_by_fulltext) { - // when predicate in compound condition which except leafNode of andNode, - // only can apply match query for fulltext index, // when predicate is leafNode of andNode, - // can apply 'match qeury' and 'equal query' and 'list query' for fulltext index. - return (pred_in_compound ? pred->type() == PredicateType::MATCH - : (pred->type() == PredicateType::MATCH || - pred->type() == PredicateType::IS_NULL || - pred->type() == PredicateType::IS_NOT_NULL || - PredicateTypeTraits::is_equal_or_list(pred->type()))); + // can apply 'match query' and 'equal query' and 'list query' for fulltext index. + return pred->type() == PredicateType::MATCH || pred->type() == PredicateType::IS_NULL || + pred->type() == PredicateType::IS_NOT_NULL || + PredicateTypeTraits::is_equal_or_list(pred->type()); } return true; @@ -979,11 +954,6 @@ bool SegmentIterator::_is_target_expr_match_predicate(const vectorized::VExprSPt } Status SegmentIterator::_apply_inverted_index() { - SCOPED_RAW_TIMER(&_opts.stats->inverted_index_filter_timer); - if (_opts.runtime_state && !_opts.runtime_state->query_options().enable_inverted_index_query) { - return Status::OK(); - } - size_t input_rows = _row_bitmap.cardinality(); std::vector remaining_predicates; std::set no_need_to_pass_column_predicate_set; @@ -1001,7 +971,6 @@ Status SegmentIterator::_apply_inverted_index() { } _col_predicates = std::move(remaining_predicates); - _opts.stats->rows_inverted_index_filtered += (input_rows - _row_bitmap.cardinality()); return Status::OK(); } diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.h b/be/src/olap/rowset/segment_v2/segment_iterator.h index 6ffd1666f602e22..f5c133485aa012c 100644 --- a/be/src/olap/rowset/segment_v2/segment_iterator.h +++ b/be/src/olap/rowset/segment_v2/segment_iterator.h @@ -298,8 +298,7 @@ class SegmentIterator : public RowwiseIterator { void _convert_dict_code_for_predicate_if_necessary_impl(ColumnPredicate* predicate); - bool _check_apply_by_inverted_index(ColumnId col_id); - bool _check_apply_by_inverted_index(ColumnPredicate* pred, bool pred_in_compound = false); + bool _check_apply_by_inverted_index(ColumnPredicate* pred); void _output_index_result_column_for_expr(uint16_t* sel_rowid_idx, uint16_t select_size, vectorized::Block* block); diff --git a/be/src/olap/rowset/segment_v2/segment_writer.cpp b/be/src/olap/rowset/segment_v2/segment_writer.cpp index 105433d2689e019..9f5811f67909bc0 100644 --- a/be/src/olap/rowset/segment_v2/segment_writer.cpp +++ b/be/src/olap/rowset/segment_v2/segment_writer.cpp @@ -596,7 +596,8 @@ Status SegmentWriter::append_block_with_partial_content(const vectorized::Block* segment_pos); } else { - if (!_opts.rowset_ctx->partial_update_info->can_insert_new_rows_in_partial_update) { + if (!_opts.rowset_ctx->partial_update_info->can_insert_new_rows_in_partial_update && + !have_delete_sign) { std::string error_column; for (auto cid : _opts.rowset_ctx->partial_update_info->missing_cids) { const TabletColumn& col = _tablet_schema->column(cid); diff --git a/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp b/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp index 64f72bc0c4669d4..7ec9236a4d8cd56 100644 --- a/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp +++ b/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp @@ -455,7 +455,8 @@ Status VerticalSegmentWriter::_append_block_with_partial_content(RowsInBlock& da DeleteBitmap::TEMP_VERSION_COMMON}, segment_pos); } else { - if (!_opts.rowset_ctx->partial_update_info->can_insert_new_rows_in_partial_update) { + if (!_opts.rowset_ctx->partial_update_info->can_insert_new_rows_in_partial_update && + !have_delete_sign) { std::string error_column; for (auto cid : _opts.rowset_ctx->partial_update_info->missing_cids) { const TabletColumn& col = _tablet_schema->column(cid); diff --git a/be/src/olap/tablet_reader.cpp b/be/src/olap/tablet_reader.cpp index d158e6a5ac526ef..5966feeaaf2d2c6 100644 --- a/be/src/olap/tablet_reader.cpp +++ b/be/src/olap/tablet_reader.cpp @@ -116,9 +116,6 @@ TabletReader::~TabletReader() { for (auto* pred : _value_col_predicates) { delete pred; } - for (auto* pred : _col_preds_except_leafnode_of_andnode) { - delete pred; - } } Status TabletReader::init(const ReaderParams& read_params) { @@ -242,7 +239,6 @@ Status TabletReader::_capture_rs_readers(const ReaderParams& read_params) { _reader_context.read_orderby_key_columns = !_orderby_key_columns.empty() ? &_orderby_key_columns : nullptr; _reader_context.predicates = &_col_predicates; - _reader_context.predicates_except_leafnode_of_andnode = &_col_preds_except_leafnode_of_andnode; _reader_context.value_predicates = &_value_col_predicates; _reader_context.lower_bound_keys = &_keys_param.start_keys; _reader_context.is_lower_keys_included = &_is_lower_keys_included; @@ -290,7 +286,6 @@ Status TabletReader::_init_params(const ReaderParams& read_params) { _reader_context.target_cast_type_for_variants = read_params.target_cast_type_for_variants; RETURN_IF_ERROR(_init_conditions_param(read_params)); - RETURN_IF_ERROR(_init_conditions_param_except_leafnode_of_andnode(read_params)); Status res = _init_delete_condition(read_params); if (!res.ok()) { @@ -561,25 +556,6 @@ Status TabletReader::_init_conditions_param(const ReaderParams& read_params) { _col_predicates.push_back(predicate); } } - return Status::OK(); -} - -Status TabletReader::_init_conditions_param_except_leafnode_of_andnode( - const ReaderParams& read_params) { - for (const auto& condition : read_params.conditions_except_leafnode_of_andnode) { - TCondition tmp_cond = condition; - const auto& column = *DORIS_TRY(_tablet_schema->column(tmp_cond.column_name)); - const auto& mcolumn = materialize_column(column); - uint32_t index = _tablet_schema->field_index(tmp_cond.column_name); - ColumnPredicate* predicate = - parse_to_predicate(mcolumn, index, tmp_cond, _predicate_arena.get()); - if (predicate != nullptr) { - auto predicate_params = predicate->predicate_params(); - predicate_params->marked_by_runtime_filter = condition.marked_by_runtime_filter; - predicate_params->values = condition.condition_values; - _col_preds_except_leafnode_of_andnode.push_back(predicate); - } - } for (int id : read_params.topn_filter_source_node_ids) { auto& runtime_predicate = diff --git a/be/src/olap/tablet_reader.h b/be/src/olap/tablet_reader.h index 18ebb9653ccde73..f4c8438fca8766d 100644 --- a/be/src/olap/tablet_reader.h +++ b/be/src/olap/tablet_reader.h @@ -132,7 +132,6 @@ class TabletReader { std::vector>> bloom_filters; std::vector>> bitmap_filters; std::vector>> in_filters; - std::vector conditions_except_leafnode_of_andnode; std::vector function_filters; std::vector delete_predicates; // slots that cast may be eliminated in storage layer @@ -243,8 +242,6 @@ class TabletReader { Status _init_conditions_param(const ReaderParams& read_params); - Status _init_conditions_param_except_leafnode_of_andnode(const ReaderParams& read_params); - ColumnPredicate* _parse_to_predicate( const std::pair>& bloom_filter); @@ -287,7 +284,6 @@ class TabletReader { std::vector _is_lower_keys_included; std::vector _is_upper_keys_included; std::vector _col_predicates; - std::vector _col_preds_except_leafnode_of_andnode; std::vector _value_col_predicates; DeleteHandler _delete_handler; diff --git a/be/src/pipeline/exec/olap_scan_operator.cpp b/be/src/pipeline/exec/olap_scan_operator.cpp index 95bfd3980417b91..002e085ac0a9c7c 100644 --- a/be/src/pipeline/exec/olap_scan_operator.cpp +++ b/be/src/pipeline/exec/olap_scan_operator.cpp @@ -332,7 +332,6 @@ Status OlapScanLocalState::_init_scanners(std::list* s for (auto& scanner : *scanners) { auto* olap_scanner = assert_cast(scanner.get()); RETURN_IF_ERROR(olap_scanner->prepare(state(), _conjuncts)); - olap_scanner->set_compound_filters(_compound_filters); } return Status::OK(); } @@ -354,7 +353,6 @@ Status OlapScanLocalState::_init_scanners(std::list* s p._olap_scan_node.is_preaggregation, }); RETURN_IF_ERROR(scanner->prepare(state(), _conjuncts)); - scanner->set_compound_filters(_compound_filters); scanners->push_back(std::move(scanner)); return Status::OK(); }; @@ -516,22 +514,6 @@ Status OlapScanLocalState::_build_key_ranges_and_filters() { } } - for (auto& iter : _compound_value_ranges) { - std::vector filters; - std::visit( - [&](auto&& range) { - if (range.is_in_compound_value_range()) { - range.to_condition_in_compound(filters); - } else if (range.is_match_value_range()) { - range.to_match_condition(filters); - } - }, - iter); - for (const auto& filter : filters) { - _compound_filters.push_back(filter); - } - } - // Append value ranges in "_not_in_value_ranges" for (auto& range : _not_in_value_ranges) { std::visit([&](auto&& the_range) { the_range.to_in_condition(_olap_filters, false); }, diff --git a/be/src/pipeline/exec/olap_scan_operator.h b/be/src/pipeline/exec/olap_scan_operator.h index 83f838dd0fc47c6..0dffa1b7efea3a1 100644 --- a/be/src/pipeline/exec/olap_scan_operator.h +++ b/be/src/pipeline/exec/olap_scan_operator.h @@ -92,9 +92,6 @@ class OlapScanLocalState final : public ScanLocalState { std::vector> _cond_ranges; OlapScanKeys _scan_keys; std::vector _olap_filters; - // _compound_filters store conditions in the one compound relationship in conjunct expr tree except leaf node of `and` node, - // such as: "(a or b) and (c or d)", conditions for a,b,c,d will be stored - std::vector _compound_filters; // If column id in this set, indicate that we need to read data after index filtering std::set _maybe_read_column_ids; diff --git a/be/src/pipeline/exec/result_sink_operator.h b/be/src/pipeline/exec/result_sink_operator.h index 06b961b2a316941..33e32e93633453f 100644 --- a/be/src/pipeline/exec/result_sink_operator.h +++ b/be/src/pipeline/exec/result_sink_operator.h @@ -58,6 +58,7 @@ struct ResultFileOptions { std::string file_suffix; //Bring BOM when exporting to CSV format bool with_bom = false; + int64_t orc_writer_version = 0; ResultFileOptions(const TResultFileSinkOptions& t_opt) { file_path = t_opt.file_path; @@ -108,6 +109,9 @@ struct ResultFileOptions { if (t_opt.__isset.orc_compression_type) { orc_compression_type = t_opt.orc_compression_type; } + if (t_opt.__isset.orc_writer_version) { + orc_writer_version = t_opt.orc_writer_version; + } } }; diff --git a/be/src/pipeline/exec/scan_operator.cpp b/be/src/pipeline/exec/scan_operator.cpp index d7af005394479e4..73cd02b5a5dd187 100644 --- a/be/src/pipeline/exec/scan_operator.cpp +++ b/be/src/pipeline/exec/scan_operator.cpp @@ -45,6 +45,8 @@ namespace doris::pipeline { +const static int32_t ADAPTIVE_PIPELINE_TASK_SERIAL_READ_ON_LIMIT_DEFAULT = 10000; + #define RETURN_IF_PUSH_DOWN(stmt, status) \ if (pdt == PushDownType::UNACCEPTABLE) { \ status = stmt; \ @@ -1147,12 +1149,6 @@ ScanOperatorX::ScanOperatorX(ObjectPool* pool, const TPlanNode& : OperatorX(pool, tnode, operator_id, descs), _runtime_filter_descs(tnode.runtime_filters), _parallel_tasks(parallel_tasks) { - if (!tnode.__isset.conjuncts || tnode.conjuncts.empty()) { - // Which means the request could be fullfilled in a single segment iterator request. - if (tnode.limit > 0 && tnode.limit < 1024) { - _should_run_serial = true; - } - } if (tnode.__isset.push_down_count) { _push_down_count = tnode.push_down_count; } @@ -1185,6 +1181,34 @@ Status ScanOperatorX::init(const TPlanNode& tnode, RuntimeState* if (tnode.__isset.topn_filter_source_node_ids) { topn_filter_source_node_ids = tnode.topn_filter_source_node_ids; } + + // The first branch is kept for compatibility with the old version of the FE + if (!query_options.__isset.enable_adaptive_pipeline_task_serial_read_on_limit) { + if (!tnode.__isset.conjuncts || tnode.conjuncts.empty()) { + // Which means the request could be fullfilled in a single segment iterator request. + if (tnode.limit > 0 && + tnode.limit <= ADAPTIVE_PIPELINE_TASK_SERIAL_READ_ON_LIMIT_DEFAULT) { + _should_run_serial = true; + } + } + } else { + DCHECK(query_options.__isset.adaptive_pipeline_task_serial_read_on_limit); + // The set of enable_adaptive_pipeline_task_serial_read_on_limit + // is checked in previous branch. + if (query_options.enable_adaptive_pipeline_task_serial_read_on_limit) { + int32_t adaptive_pipeline_task_serial_read_on_limit = + ADAPTIVE_PIPELINE_TASK_SERIAL_READ_ON_LIMIT_DEFAULT; + if (query_options.__isset.adaptive_pipeline_task_serial_read_on_limit) { + adaptive_pipeline_task_serial_read_on_limit = + query_options.adaptive_pipeline_task_serial_read_on_limit; + } + + if (tnode.limit > 0 && tnode.limit <= adaptive_pipeline_task_serial_read_on_limit) { + _should_run_serial = true; + } + } + } + return Status::OK(); } diff --git a/be/src/pipeline/exec/scan_operator.h b/be/src/pipeline/exec/scan_operator.h index 823440f52c0e8e3..e106c5a2b513836 100644 --- a/be/src/pipeline/exec/scan_operator.h +++ b/be/src/pipeline/exec/scan_operator.h @@ -333,19 +333,6 @@ class ScanLocalState : public ScanLocalStateBase { // We use _colname_to_value_range to store a column and its conresponding value ranges. std::unordered_map _colname_to_value_range; - /** - * _colname_to_value_range only store the leaf of and in the conjunct expr tree, - * we use _compound_value_ranges to store conresponding value ranges - * in the one compound relationship except the leaf of and node, - * such as `where a > 1 or b > 10 and c < 200`, the expr tree like: - * or - * / \ - * a and - * / \ - * b c - * the value ranges of column a,b,c will all store into _compound_value_ranges - */ - std::vector _compound_value_ranges; // But if a col is with value range, eg: 1 < col < 10, which is "!is_fixed_range", // in this case we can not merge "1 < col < 10" with "col not in (2)". // So we have to save "col not in (2)" to another structure: "_not_in_value_ranges". diff --git a/be/src/pipeline/pipeline_fragment_context.cpp b/be/src/pipeline/pipeline_fragment_context.cpp index 51cfd73e549d796..3bf54ed7ece00b7 100644 --- a/be/src/pipeline/pipeline_fragment_context.cpp +++ b/be/src/pipeline/pipeline_fragment_context.cpp @@ -1031,6 +1031,10 @@ Status PipelineFragmentContext::_create_data_sink(ObjectPool* pool, const TDataS } case TDataSinkType::GROUP_COMMIT_BLOCK_SINK: { DCHECK(thrift_sink.__isset.olap_table_sink); +#ifndef NDEBUG + DCHECK(state->get_query_ctx() != nullptr); + state->get_query_ctx()->query_mem_tracker->is_group_commit_load = true; +#endif _sink.reset( new GroupCommitBlockSinkOperatorX(next_sink_operator_id(), row_desc, output_exprs)); break; @@ -1177,6 +1181,10 @@ Status PipelineFragmentContext::_create_operator(ObjectPool* pool, const TPlanNo break; } case TPlanNodeType::GROUP_COMMIT_SCAN_NODE: { +#ifndef NDEBUG + DCHECK(_query_ctx != nullptr); + _query_ctx->query_mem_tracker->is_group_commit_load = true; +#endif op.reset(new GroupCommitOperatorX(pool, tnode, next_operator_id(), descs, _num_instances)); RETURN_IF_ERROR(cur_pipe->add_operator(op)); if (request.__isset.parallel_instances) { diff --git a/be/src/runtime/buffer_control_block.cpp b/be/src/runtime/buffer_control_block.cpp index 6f8022a00342a32..be03db04d67f0c7 100644 --- a/be/src/runtime/buffer_control_block.cpp +++ b/be/src/runtime/buffer_control_block.cpp @@ -248,6 +248,11 @@ Status BufferControlBlock::get_arrow_batch(std::shared_ptr* Status BufferControlBlock::close(const TUniqueId& id, Status exec_status) { std::unique_lock l(_lock); + // close will be called multiple times and error status needs to be collected. + if (!exec_status.ok()) { + _status = exec_status; + } + auto it = _result_sink_dependencys.find(id); if (it != _result_sink_dependencys.end()) { it->second->set_always_ready(); @@ -258,7 +263,6 @@ Status BufferControlBlock::close(const TUniqueId& id, Status exec_status) { } _is_close = true; - _status = exec_status; _arrow_data_arrival.notify_all(); if (!_waiting_rpc.empty()) { diff --git a/be/src/runtime/memory/mem_tracker.h b/be/src/runtime/memory/mem_tracker.h index 8a574398e0eaed5..8a977e49388d523 100644 --- a/be/src/runtime/memory/mem_tracker.h +++ b/be/src/runtime/memory/mem_tracker.h @@ -146,8 +146,6 @@ class MemTracker { // Creates and adds the tracker to the mem_tracker_pool. MemTracker(const std::string& label, MemTrackerLimiter* parent = nullptr); - // For MemTrackerLimiter - MemTracker() { _parent_group_num = -1; } virtual ~MemTracker(); @@ -203,6 +201,9 @@ class MemTracker { } protected: + // Only used by MemTrackerLimiter + MemTracker() { _parent_group_num = -1; } + void bind_parent(MemTrackerLimiter* parent); Type _type; diff --git a/be/src/runtime/memory/mem_tracker_limiter.cpp b/be/src/runtime/memory/mem_tracker_limiter.cpp index cc695a6fdd51e1c..774e06eb7f5ac4e 100644 --- a/be/src/runtime/memory/mem_tracker_limiter.cpp +++ b/be/src/runtime/memory/mem_tracker_limiter.cpp @@ -114,6 +114,8 @@ MemTrackerLimiter::~MemTrackerLimiter() { "mem tracker not equal to 0 when mem tracker destruct, this usually means that " "memory tracking is inaccurate and SCOPED_ATTACH_TASK and " "SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER are not used correctly. " + "If the log is truncated, search for `Address Sanitizer` in the be.INFO log to see " + "more information." "1. For query and load, memory leaks may have occurred, it is expected that the query " "mem tracker will be bound to the thread context using SCOPED_ATTACH_TASK and " "SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER before all memory alloc and free. " @@ -127,7 +129,7 @@ MemTrackerLimiter::~MemTrackerLimiter() { if (_consumption->current_value() != 0) { // TODO, expect mem tracker equal to 0 at the load/compaction/etc. task end. #ifndef NDEBUG - if (_type == Type::QUERY) { + if (_type == Type::QUERY || (_type == Type::LOAD && !is_group_commit_load)) { std::string err_msg = fmt::format("mem tracker label: {}, consumption: {}, peak consumption: {}, {}.", label(), _consumption->current_value(), _consumption->peak_value(), @@ -140,11 +142,11 @@ MemTrackerLimiter::~MemTrackerLimiter() { } _consumption->set(0); #ifndef NDEBUG - } else if (!_address_sanitizers.empty()) { - LOG(INFO) << "[Address Sanitizer] consumption is 0, but address sanitizers not empty. " - << ", mem tracker label: " << _label - << ", peak consumption: " << _consumption->peak_value() - << print_address_sanitizers(); + } else if (!_address_sanitizers.empty() && !is_group_commit_load) { + LOG(FATAL) << "[Address Sanitizer] consumption is 0, but address sanitizers not empty. " + << ", mem tracker label: " << _label + << ", peak consumption: " << _consumption->peak_value() + << print_address_sanitizers(); #endif } memory_memtrackerlimiter_cnt << -1; @@ -152,17 +154,17 @@ MemTrackerLimiter::~MemTrackerLimiter() { #ifndef NDEBUG void MemTrackerLimiter::add_address_sanitizers(void* buf, size_t size) { - if (_type == Type::QUERY) { + if (_type == Type::QUERY || (_type == Type::LOAD && !is_group_commit_load)) { std::lock_guard l(_address_sanitizers_mtx); auto it = _address_sanitizers.find(buf); if (it != _address_sanitizers.end()) { - LOG(INFO) << "[Address Sanitizer] memory buf repeat add, mem tracker label: " << _label - << ", consumption: " << _consumption->current_value() - << ", peak consumption: " << _consumption->peak_value() << ", buf: " << buf - << ", size: " << size << ", old buf: " << it->first - << ", old size: " << it->second.size - << ", new stack_trace: " << get_stack_trace(1, "DISABLED") - << ", old stack_trace: " << it->second.stack_trace; + _error_address_sanitizers.emplace_back( + fmt::format("[Address Sanitizer] memory buf repeat add, mem tracker label: {}, " + "consumption: {}, peak consumption: {}, buf: {}, size: {}, old " + "buf: {}, old size: {}, new stack_trace: {}, old stack_trace: {}.", + _label, _consumption->current_value(), _consumption->peak_value(), + buf, size, it->first, it->second.size, + get_stack_trace(1, "FULL_WITH_INLINE"), it->second.stack_trace)); } // if alignment not equal to 0, maybe usable_size > size. @@ -174,26 +176,26 @@ void MemTrackerLimiter::add_address_sanitizers(void* buf, size_t size) { } void MemTrackerLimiter::remove_address_sanitizers(void* buf, size_t size) { - if (_type == Type::QUERY) { + if (_type == Type::QUERY || (_type == Type::LOAD && !is_group_commit_load)) { std::lock_guard l(_address_sanitizers_mtx); auto it = _address_sanitizers.find(buf); if (it != _address_sanitizers.end()) { if (it->second.size != size) { - LOG(INFO) << "[Address Sanitizer] free memory buf size inaccurate, mem tracker " - "label: " - << _label << ", consumption: " << _consumption->current_value() - << ", peak consumption: " << _consumption->peak_value() - << ", buf: " << buf << ", size: " << size << ", old buf: " << it->first - << ", old size: " << it->second.size - << ", new stack_trace: " << get_stack_trace(1, "DISABLED") - << ", old stack_trace: " << it->second.stack_trace; + _error_address_sanitizers.emplace_back(fmt::format( + "[Address Sanitizer] free memory buf size inaccurate, mem tracker label: " + "{}, consumption: {}, peak consumption: {}, buf: {}, size: {}, old buf: " + "{}, old size: {}, new stack_trace: {}, old stack_trace: {}.", + _label, _consumption->current_value(), _consumption->peak_value(), buf, + size, it->first, it->second.size, get_stack_trace(1, "FULL_WITH_INLINE"), + it->second.stack_trace)); } _address_sanitizers.erase(buf); } else { - LOG(INFO) << "[Address Sanitizer] memory buf not exist, mem tracker label: " << _label - << ", consumption: " << _consumption->current_value() - << ", peak consumption: " << _consumption->peak_value() << ", buf: " << buf - << ", size: " << size << ", stack_trace: " << get_stack_trace(1, "DISABLED"); + _error_address_sanitizers.emplace_back(fmt::format( + "[Address Sanitizer] memory buf not exist, mem tracker label: {}, consumption: " + "{}, peak consumption: {}, buf: {}, size: {}, stack_trace: {}.", + _label, _consumption->current_value(), _consumption->peak_value(), buf, size, + get_stack_trace(1, "FULL_WITH_INLINE"))); } } } @@ -201,9 +203,20 @@ void MemTrackerLimiter::remove_address_sanitizers(void* buf, size_t size) { std::string MemTrackerLimiter::print_address_sanitizers() { std::lock_guard l(_address_sanitizers_mtx); std::string detail = "[Address Sanitizer]:"; + detail += "\n memory not be freed:"; for (const auto& it : _address_sanitizers) { - detail += fmt::format("\n {}, size {}, strack trace: {}", it.first, it.second.size, - it.second.stack_trace); + auto msg = fmt::format( + "\n [Address Sanitizer] buf not be freed, mem tracker label: {}, consumption: " + "{}, peak consumption: {}, buf: {}, size {}, strack trace: {}", + _label, _consumption->current_value(), _consumption->peak_value(), it.first, + it.second.size, it.second.stack_trace); + LOG(INFO) << msg; + detail += msg; + } + detail += "\n incorrect memory alloc and free:"; + for (const auto& err_msg : _error_address_sanitizers) { + LOG(INFO) << err_msg; + detail += fmt::format("\n {}", err_msg); } return detail; } @@ -323,7 +336,6 @@ void MemTrackerLimiter::make_process_snapshots(std::vector snapshot.cur_consumption = GlobalMemoryArbitrator::process_reserved_memory(); snapshot.peak_consumption = -1; (*snapshots).emplace_back(snapshot); - all_trackers_mem_sum += GlobalMemoryArbitrator::process_reserved_memory(); snapshot.type = "overview"; snapshot.label = "sum_of_all_trackers"; // is virtual memory diff --git a/be/src/runtime/memory/mem_tracker_limiter.h b/be/src/runtime/memory/mem_tracker_limiter.h index e5c5cb1bc0369c6..344f3dc92b6670c 100644 --- a/be/src/runtime/memory/mem_tracker_limiter.h +++ b/be/src/runtime/memory/mem_tracker_limiter.h @@ -209,6 +209,7 @@ class MemTrackerLimiter final : public MemTracker { void add_address_sanitizers(void* buf, size_t size); void remove_address_sanitizers(void* buf, size_t size); std::string print_address_sanitizers(); + bool is_group_commit_load {false}; #endif std::string debug_string() override { @@ -260,6 +261,7 @@ class MemTrackerLimiter final : public MemTracker { std::mutex _address_sanitizers_mtx; std::unordered_map _address_sanitizers; + std::vector _error_address_sanitizers; #endif }; diff --git a/be/src/service/point_query_executor.cpp b/be/src/service/point_query_executor.cpp index 88293ba9b036752..8058e1f1be63024 100644 --- a/be/src/service/point_query_executor.cpp +++ b/be/src/service/point_query_executor.cpp @@ -49,7 +49,9 @@ #include "runtime/thread_context.h" #include "util/key_util.h" #include "util/runtime_profile.h" +#include "util/simd/bits.h" #include "util/thrift_util.h" +#include "vec/columns/columns_number.h" #include "vec/data_types/serde/data_type_serde.h" #include "vec/exprs/vexpr.h" #include "vec/exprs/vexpr_context.h" @@ -143,6 +145,9 @@ Status Reusable::init(const TDescriptorTable& t_desc_tbl, const std::vectorroot(), tuple_desc(), output_slot_descs); } + // get the delete sign idx in block + _delete_sign_idx = _col_uid_to_idx[schema.columns()[schema.delete_sign_idx()]->unique_id()]; + if (schema.have_column(BeConsts::ROW_STORE_COL)) { const auto& column = *DORIS_TRY(schema.column(BeConsts::ROW_STORE_COL)); _row_store_column_ids = column.unique_id(); @@ -483,6 +488,19 @@ Status PointQueryExecutor::_lookup_row_data() { } } } + // filter rows by delete sign + if (_row_hits > 0 && _reusable->delete_sign_idx() != -1) { + vectorized::ColumnPtr delete_filter_columns = + _result_block->get_columns()[_reusable->delete_sign_idx()]; + const auto& filter = + assert_cast(delete_filter_columns.get())->get_data(); + size_t count = filter.size() - simd::count_zero_num((int8_t*)filter.data(), filter.size()); + if (count == filter.size()) { + _result_block->clear(); + } else if (count > 0) { + return Status::NotSupported("Not implemented since only single row at present"); + } + } return Status::OK(); } diff --git a/be/src/service/point_query_executor.h b/be/src/service/point_query_executor.h index 29c7348e3d5b973..19954479c97ec78 100644 --- a/be/src/service/point_query_executor.h +++ b/be/src/service/point_query_executor.h @@ -100,6 +100,9 @@ class Reusable { RuntimeState* runtime_state() { return _runtime_state.get(); } + // delete sign idx in block + int32_t delete_sign_idx() const { return _delete_sign_idx; } + private: // caching TupleDescriptor, output_expr, etc... std::unique_ptr _runtime_state; @@ -118,6 +121,8 @@ class Reusable { std::unordered_set _missing_col_uids; // included cids in rowstore(column group) std::unordered_set _include_col_uids; + // delete sign idx in block + int32_t _delete_sign_idx = -1; }; // RowCache is a LRU cache for row store diff --git a/be/src/vec/data_types/data_type_decimal.h b/be/src/vec/data_types/data_type_decimal.h index 580eb5fb3ea55d7..b18487d1fb09668 100644 --- a/be/src/vec/data_types/data_type_decimal.h +++ b/be/src/vec/data_types/data_type_decimal.h @@ -595,10 +595,11 @@ void convert_from_decimal(typename ToDataType::FieldType* dst, dst[i] = static_cast(src[i].value) / multiplier.value; } } - FromDataType from_data_type(precision, scale); if constexpr (narrow_integral) { + FromDataType from_data_type(precision, scale); for (size_t i = 0; i < size; i++) { - if (dst[i] < min_result || dst[i] > max_result) { + if (std::isnan(dst[i]) || std::isinf(dst[i]) || dst[i] < min_result || + dst[i] > max_result) { THROW_DECIMAL_CONVERT_OVERFLOW_EXCEPTION(from_data_type.to_string(src[i]), from_data_type.get_name(), ToDataType {}.get_name()); diff --git a/be/src/vec/data_types/get_least_supertype.cpp b/be/src/vec/data_types/get_least_supertype.cpp index 7d8b3ddbe507c9d..82bea452923ed5c 100644 --- a/be/src/vec/data_types/get_least_supertype.cpp +++ b/be/src/vec/data_types/get_least_supertype.cpp @@ -41,6 +41,7 @@ #include "vec/data_types/data_type_number.h" #include "vec/data_types/data_type_object.h" #include "vec/data_types/data_type_string.h" +#include "vec/data_types/data_type_time_v2.h" namespace doris::vectorized { @@ -284,6 +285,14 @@ void get_least_supertype_jsonb(const TypeIndexSet& types, DataTypePtr* type) { *type = std::make_shared(); return; } + if (which.is_date_v2()) { + *type = std::make_shared(); + return; + } + if (which.is_date_time_v2()) { + *type = std::make_shared(); + return; + } *type = std::make_shared(); return; } diff --git a/be/src/vec/exec/scan/new_olap_scanner.cpp b/be/src/vec/exec/scan/new_olap_scanner.cpp index 0b1382dc1dba2ab..a01bfaa92d0fd3e 100644 --- a/be/src/vec/exec/scan/new_olap_scanner.cpp +++ b/be/src/vec/exec/scan/new_olap_scanner.cpp @@ -84,7 +84,6 @@ NewOlapScanner::NewOlapScanner(pipeline::ScanLocalStateBase* parent, .bloom_filters {}, .bitmap_filters {}, .in_filters {}, - .conditions_except_leafnode_of_andnode {}, .function_filters {}, .delete_predicates {}, .target_cast_type_for_variants {}, @@ -245,10 +244,6 @@ Status NewOlapScanner::open(RuntimeState* state) { return Status::OK(); } -void NewOlapScanner::set_compound_filters(const std::vector& compound_filters) { - _compound_filters = compound_filters; -} - // it will be called under tablet read lock because capture rs readers need Status NewOlapScanner::_init_tablet_reader_params( const std::vector& key_ranges, const std::vector& filters, @@ -294,10 +289,6 @@ Status NewOlapScanner::_init_tablet_reader_params( _tablet_reader_params.conditions.push_back(filter); } - std::copy(_compound_filters.cbegin(), _compound_filters.cend(), - std::inserter(_tablet_reader_params.conditions_except_leafnode_of_andnode, - _tablet_reader_params.conditions_except_leafnode_of_andnode.begin())); - std::copy(filter_predicates.bloom_filters.cbegin(), filter_predicates.bloom_filters.cend(), std::inserter(_tablet_reader_params.bloom_filters, _tablet_reader_params.bloom_filters.begin())); diff --git a/be/src/vec/exec/scan/new_olap_scanner.h b/be/src/vec/exec/scan/new_olap_scanner.h index df6ff0411436062..52c664c026afd82 100644 --- a/be/src/vec/exec/scan/new_olap_scanner.h +++ b/be/src/vec/exec/scan/new_olap_scanner.h @@ -75,8 +75,6 @@ class NewOlapScanner : public VScanner { Status close(RuntimeState* state) override; - void set_compound_filters(const std::vector& compound_filters); - doris::TabletStorageType get_storage_type() override; protected: @@ -101,7 +99,6 @@ class NewOlapScanner : public VScanner { std::vector _return_columns; std::unordered_set _tablet_columns_convert_to_null_set; - std::vector _compound_filters; // ========= profiles ========== int64_t _compressed_bytes_read = 0; diff --git a/be/src/vec/exec/scan/vmeta_scanner.h b/be/src/vec/exec/scan/vmeta_scanner.h index 8256dff9b910a25..a9975300cdc5002 100644 --- a/be/src/vec/exec/scan/vmeta_scanner.h +++ b/be/src/vec/exec/scan/vmeta_scanner.h @@ -86,7 +86,6 @@ class VMetaScanner : public VScanner { TFetchSchemaTableDataRequest* request); Status _build_queries_metadata_request(const TMetaScanRange& meta_scan_range, TFetchSchemaTableDataRequest* request); - bool _meta_eos; TupleId _tuple_id; TUserIdentity _user_identity; diff --git a/be/src/vec/exprs/vexpr.h b/be/src/vec/exprs/vexpr.h index 90048511c908565..382713b2afc0188 100644 --- a/be/src/vec/exprs/vexpr.h +++ b/be/src/vec/exprs/vexpr.h @@ -321,7 +321,6 @@ class VExpr { uint32_t _index_unique_id = 0; bool _can_fast_execute = false; bool _enable_inverted_index_query = true; - uint32_t _in_list_value_count_threshold = 10; }; } // namespace vectorized diff --git a/be/src/vec/exprs/vin_predicate.cpp b/be/src/vec/exprs/vin_predicate.cpp index 361f55f038ca4c1..9e00a3afbaef434 100644 --- a/be/src/vec/exprs/vin_predicate.cpp +++ b/be/src/vec/exprs/vin_predicate.cpp @@ -81,6 +81,14 @@ Status VInPredicate::prepare(RuntimeState* state, const RowDescriptor& desc, VExpr::register_function_context(state, context); _prepare_finished = true; + + if (state->query_options().__isset.in_list_value_count_threshold) { + _in_list_value_count_threshold = state->query_options().in_list_value_count_threshold; + } + + const auto in_list_value_count = _children.size() - 1; + // When the number of values in the IN condition exceeds this threshold, fast_execute will not be used + _can_fast_execute = in_list_value_count <= _in_list_value_count_threshold; return Status::OK(); } @@ -94,10 +102,24 @@ Status VInPredicate::open(RuntimeState* state, VExprContext* context, if (scope == FunctionContext::FRAGMENT_LOCAL) { RETURN_IF_ERROR(VExpr::get_const_col(context, nullptr)); } + + _is_args_all_constant = std::all_of(_children.begin() + 1, _children.end(), + [](const VExprSPtr& expr) { return expr->is_constant(); }); _open_finished = true; return Status::OK(); } +size_t VInPredicate::skip_constant_args_size() const { + if (_is_args_all_constant && !_can_fast_execute) { + // This is an optimization. For expressions like colA IN (1, 2, 3, 4), + // where all values inside the IN clause are constants, + // a hash set is created during open, and it will not be accessed again during execute + // Here, _children[0] is colA + return 1; + } + return _children.size(); +} + void VInPredicate::close(VExprContext* context, FunctionContext::FunctionStateScope scope) { VExpr::close_function_context(context, scope, _function); VExpr::close(context, scope); @@ -116,9 +138,8 @@ Status VInPredicate::execute(VExprContext* context, Block* block, int* result_co return Status::OK(); } DCHECK(_open_finished || _getting_const_col); - // TODO: not execute const expr again, but use the const column in function context - doris::vectorized::ColumnNumbers arguments(_children.size()); - for (int i = 0; i < _children.size(); ++i) { + doris::vectorized::ColumnNumbers arguments(skip_constant_args_size()); + for (int i = 0; i < skip_constant_args_size(); ++i) { int column_id = -1; RETURN_IF_ERROR(_children[i]->execute(context, block, &column_id)); arguments[i] = column_id; diff --git a/be/src/vec/exprs/vin_predicate.h b/be/src/vec/exprs/vin_predicate.h index 4d227510b910cea..024ad68f2ba576c 100644 --- a/be/src/vec/exprs/vin_predicate.h +++ b/be/src/vec/exprs/vin_predicate.h @@ -51,6 +51,8 @@ class VInPredicate final : public VExpr { std::string debug_string() const override; + size_t skip_constant_args_size() const; + const FunctionBasePtr function() { return _function; } bool is_not_in() const { return _is_not_in; }; @@ -62,5 +64,7 @@ class VInPredicate final : public VExpr { const bool _is_not_in; static const constexpr char* function_name = "in"; + uint32_t _in_list_value_count_threshold = 10; + bool _is_args_all_constant = false; }; } // namespace doris::vectorized \ No newline at end of file diff --git a/be/src/vec/functions/function_collection_in.h b/be/src/vec/functions/function_collection_in.h index 1e86ce25b3405f8..33a4a2570800a97 100644 --- a/be/src/vec/functions/function_collection_in.h +++ b/be/src/vec/functions/function_collection_in.h @@ -112,9 +112,8 @@ class FunctionCollectionIn : public IFunction { // FE should make element type consistent and // equalize the length of the elements in struct const auto& const_column_ptr = context->get_constant_col(i); - if (const_column_ptr == nullptr) { - break; - } + // Types like struct, array, and map only support constant expressions. + DCHECK(const_column_ptr != nullptr); const auto& [col, _] = unpack_if_const(const_column_ptr->column_ptr); if (col->is_nullable()) { auto* null_col = vectorized::check_and_get_column(col); diff --git a/be/src/vec/functions/function_math_unary_alway_nullable.h b/be/src/vec/functions/function_math_unary_alway_nullable.h new file mode 100644 index 000000000000000..8d2cea1bc0db87d --- /dev/null +++ b/be/src/vec/functions/function_math_unary_alway_nullable.h @@ -0,0 +1,94 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "vec/columns/column.h" +#include "vec/columns/column_decimal.h" +#include "vec/columns/column_nullable.h" +#include "vec/columns/columns_number.h" +#include "vec/core/call_on_type_index.h" +#include "vec/core/types.h" +#include "vec/data_types/data_type_decimal.h" +#include "vec/data_types/data_type_nullable.h" +#include "vec/data_types/data_type_number.h" +#include "vec/functions/function.h" +#include "vec/functions/function_helpers.h" +#include "vec/utils/util.hpp" + +namespace doris::vectorized { + +template +class FunctionMathUnaryAlwayNullable : public IFunction { +public: + using IFunction::execute; + + static constexpr auto name = Impl::name; + static FunctionPtr create() { return std::make_shared(); } + +private: + String get_name() const override { return name; } + size_t get_number_of_arguments() const override { return 1; } + + DataTypePtr get_return_type_impl(const DataTypes& arguments) const override { + return make_nullable(std::make_shared()); + } + + static void execute_in_iterations(const double* src_data, double* dst_data, size_t size) { + for (size_t i = 0; i < size; i++) { + Impl::execute(&src_data[i], &dst_data[i]); + } + } + + Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments, + size_t result, size_t input_rows_count) const override { + const ColumnFloat64* col = + assert_cast(block.get_by_position(arguments[0]).column.get()); + auto dst = ColumnFloat64::create(); + auto& dst_data = dst->get_data(); + dst_data.resize(input_rows_count); + + execute_in_iterations(col->get_data().data(), dst_data.data(), input_rows_count); + + auto result_null_map = ColumnUInt8::create(input_rows_count, 0); + + for (size_t i = 0; i < input_rows_count; i++) { + if (Impl::is_invalid_input(col->get_data()[i])) [[unlikely]] { + result_null_map->get_data().data()[i] = 1; + } + } + + block.replace_by_position( + result, ColumnNullable::create(std::move(dst), std::move(result_null_map))); + return Status::OK(); + } +}; + +template +struct UnaryFunctionPlainAlwayNullable { + using Type = DataTypeFloat64; + static constexpr auto name = Name::name; + + static constexpr bool is_invalid_input(Float64 x) { return Name::is_invalid_input(x); } + + template + static void execute(const T* src, U* dst) { + *dst = static_cast(Function(*src)); + } +}; + +} // namespace doris::vectorized diff --git a/be/src/vec/functions/functions_comparison.h b/be/src/vec/functions/functions_comparison.h index f6aa46fe1c42c95..bb1666ab8640701 100644 --- a/be/src/vec/functions/functions_comparison.h +++ b/be/src/vec/functions/functions_comparison.h @@ -546,14 +546,6 @@ class FunctionComparison : public IFunction { //NOT support comparison predicate when parser is FULLTEXT for expr inverted index evaluate. return Status::OK(); } - std::string column_name = data_type_with_name.first; - Field param_value; - arguments[0].column->get(0, param_value); - auto param_type = arguments[0].type->get_type_as_type_descriptor().type; - - std::unique_ptr query_param = nullptr; - RETURN_IF_ERROR(segment_v2::InvertedIndexQueryParamFactory::create_query_value( - param_type, ¶m_value, query_param)); segment_v2::InvertedIndexQueryType query_type; std::string_view name_view(name); if (name_view == NameEquals::name || name_view == NameNotEquals::name) { @@ -570,6 +562,19 @@ class FunctionComparison : public IFunction { return Status::InvalidArgument("invalid comparison op type {}", Name::name); } + if (segment_v2::is_range_query(query_type) && + iter->get_inverted_index_reader_type() == + segment_v2::InvertedIndexReaderType::STRING_TYPE) { + // untokenized strings exceed ignore_above, they are written as null, causing range query errors + return Status::OK(); + } + std::string column_name = data_type_with_name.first; + Field param_value; + arguments[0].column->get(0, param_value); + auto param_type = arguments[0].type->get_type_as_type_descriptor().type; + std::unique_ptr query_param = nullptr; + RETURN_IF_ERROR(segment_v2::InvertedIndexQueryParamFactory::create_query_value( + param_type, ¶m_value, query_param)); std::shared_ptr roaring = std::make_shared(); RETURN_IF_ERROR(segment_v2::InvertedIndexQueryParamFactory::create_query_value( param_type, ¶m_value, query_param)); @@ -585,7 +590,7 @@ class FunctionComparison : public IFunction { bitmap_result = result; bitmap_result.mask_out_null(); - if (name == "ne") { + if (name_view == NameNotEquals::name) { roaring::Roaring full_result; full_result.addRange(0, num_rows); bitmap_result.op_not(&full_result); diff --git a/be/src/vec/functions/functions_geo.cpp b/be/src/vec/functions/functions_geo.cpp index b389bc1636e45fa..7b833f91a8ddfde 100644 --- a/be/src/vec/functions/functions_geo.cpp +++ b/be/src/vec/functions/functions_geo.cpp @@ -28,6 +28,7 @@ #include "vec/columns/column.h" #include "vec/columns/column_nullable.h" #include "vec/columns/columns_number.h" +#include "vec/common/assert_cast.h" #include "vec/common/string_ref.h" #include "vec/core/block.h" #include "vec/core/column_with_type_and_name.h" @@ -58,14 +59,16 @@ struct StPoint { auto res = ColumnString::create(); auto null_map = ColumnUInt8::create(size, 0); auto& null_map_data = null_map->get_data(); + const auto* left_column_f64 = assert_cast(left_column.get()); + const auto* right_column_f64 = assert_cast(right_column.get()); GeoPoint point; std::string buf; if (left_const) { - const_vector(left_column, right_column, res, null_map_data, size, point, buf); + const_vector(left_column_f64, right_column_f64, res, null_map_data, size, point, buf); } else if (right_const) { - vector_const(left_column, right_column, res, null_map_data, size, point, buf); + vector_const(left_column_f64, right_column_f64, res, null_map_data, size, point, buf); } else { - vector_vector(left_column, right_column, res, null_map_data, size, point, buf); + vector_vector(left_column_f64, right_column_f64, res, null_map_data, size, point, buf); } block.replace_by_position(result, @@ -86,32 +89,32 @@ struct StPoint { res->insert_data(buf.data(), buf.size()); } - static void const_vector(const ColumnPtr& left_column, const ColumnPtr& right_column, + static void const_vector(const ColumnFloat64* left_column, const ColumnFloat64* right_column, ColumnString::MutablePtr& res, NullMap& null_map, const size_t size, GeoPoint& point, std::string& buf) { - double x = left_column->operator[](0).get(); + double x = left_column->get_element(0); for (int row = 0; row < size; ++row) { - auto cur_res = point.from_coord(x, right_column->operator[](row).get()); + auto cur_res = point.from_coord(x, right_column->get_element(row)); loop_do(cur_res, res, null_map, row, point, buf); } } - static void vector_const(const ColumnPtr& left_column, const ColumnPtr& right_column, + static void vector_const(const ColumnFloat64* left_column, const ColumnFloat64* right_column, ColumnString::MutablePtr& res, NullMap& null_map, const size_t size, GeoPoint& point, std::string& buf) { - double y = right_column->operator[](0).get(); + double y = right_column->get_element(0); for (int row = 0; row < size; ++row) { - auto cur_res = point.from_coord(right_column->operator[](row).get(), y); + auto cur_res = point.from_coord(right_column->get_element(row), y); loop_do(cur_res, res, null_map, row, point, buf); } } - static void vector_vector(const ColumnPtr& left_column, const ColumnPtr& right_column, + static void vector_vector(const ColumnFloat64* left_column, const ColumnFloat64* right_column, ColumnString::MutablePtr& res, NullMap& null_map, const size_t size, GeoPoint& point, std::string& buf) { for (int row = 0; row < size; ++row) { - auto cur_res = point.from_coord(left_column->operator[](row).get(), - right_column->operator[](row).get()); + auto cur_res = + point.from_coord(left_column->get_element(row), right_column->get_element(row)); loop_do(cur_res, res, null_map, row, point, buf); } } @@ -246,11 +249,15 @@ struct StDistanceSphere { DCHECK_EQ(arguments.size(), 4); auto return_type = block.get_data_type(result); - auto x_lng = block.get_by_position(arguments[0]).column->convert_to_full_column_if_const(); - auto x_lat = block.get_by_position(arguments[1]).column->convert_to_full_column_if_const(); - auto y_lng = block.get_by_position(arguments[2]).column->convert_to_full_column_if_const(); - auto y_lat = block.get_by_position(arguments[3]).column->convert_to_full_column_if_const(); - + const auto* x_lng = check_and_get_column( + block.get_by_position(arguments[0]).column->convert_to_full_column_if_const()); + const auto* x_lat = check_and_get_column( + block.get_by_position(arguments[1]).column->convert_to_full_column_if_const()); + const auto* y_lng = check_and_get_column( + block.get_by_position(arguments[2]).column->convert_to_full_column_if_const()); + const auto* y_lat = check_and_get_column( + block.get_by_position(arguments[3]).column->convert_to_full_column_if_const()); + CHECK(x_lng && x_lat && y_lng && y_lat); const auto size = x_lng->size(); auto res = ColumnFloat64::create(); res->reserve(size); @@ -258,10 +265,9 @@ struct StDistanceSphere { auto& null_map_data = null_map->get_data(); for (int row = 0; row < size; ++row) { double distance = 0; - if (!GeoPoint::ComputeDistance(x_lng->operator[](row).get(), - x_lat->operator[](row).get(), - y_lng->operator[](row).get(), - y_lat->operator[](row).get(), &distance)) { + if (!GeoPoint::ComputeDistance(x_lng->get_element(row), x_lat->get_element(row), + y_lng->get_element(row), y_lat->get_element(row), + &distance)) { null_map_data[row] = 1; res->insert_default(); continue; @@ -284,10 +290,15 @@ struct StAngleSphere { DCHECK_EQ(arguments.size(), 4); auto return_type = block.get_data_type(result); - auto x_lng = block.get_by_position(arguments[0]).column->convert_to_full_column_if_const(); - auto x_lat = block.get_by_position(arguments[1]).column->convert_to_full_column_if_const(); - auto y_lng = block.get_by_position(arguments[2]).column->convert_to_full_column_if_const(); - auto y_lat = block.get_by_position(arguments[3]).column->convert_to_full_column_if_const(); + const auto* x_lng = check_and_get_column( + block.get_by_position(arguments[0]).column->convert_to_full_column_if_const()); + const auto* x_lat = check_and_get_column( + block.get_by_position(arguments[1]).column->convert_to_full_column_if_const()); + const auto* y_lng = check_and_get_column( + block.get_by_position(arguments[2]).column->convert_to_full_column_if_const()); + const auto* y_lat = check_and_get_column( + block.get_by_position(arguments[3]).column->convert_to_full_column_if_const()); + CHECK(x_lng && x_lat && y_lng && y_lat); const auto size = x_lng->size(); @@ -298,10 +309,9 @@ struct StAngleSphere { for (int row = 0; row < size; ++row) { double angle = 0; - if (!GeoPoint::ComputeAngleSphere(x_lng->operator[](row).get(), - x_lat->operator[](row).get(), - y_lng->operator[](row).get(), - y_lat->operator[](row).get(), &angle)) { + if (!GeoPoint::ComputeAngleSphere(x_lng->get_element(row), x_lat->get_element(row), + y_lng->get_element(row), y_lat->get_element(row), + &angle)) { null_map_data[row] = 1; res->insert_default(); continue; diff --git a/be/src/vec/functions/math.cpp b/be/src/vec/functions/math.cpp index a3b54c8026db75e..af2e68ec9822c8d 100644 --- a/be/src/vec/functions/math.cpp +++ b/be/src/vec/functions/math.cpp @@ -37,6 +37,7 @@ #include "vec/functions/function_const.h" #include "vec/functions/function_math_log.h" #include "vec/functions/function_math_unary.h" +#include "vec/functions/function_math_unary_alway_nullable.h" #include "vec/functions/function_string.h" #include "vec/functions/function_totype.h" #include "vec/functions/function_unary_arithmetic.h" @@ -53,13 +54,19 @@ struct Log2Impl; namespace doris::vectorized { struct AcosName { static constexpr auto name = "acos"; + // https://dev.mysql.com/doc/refman/8.4/en/mathematical-functions.html#function_acos + static constexpr bool is_invalid_input(Float64 x) { return x < -1 || x > 1; } }; -using FunctionAcos = FunctionMathUnary>; +using FunctionAcos = + FunctionMathUnaryAlwayNullable>; struct AsinName { static constexpr auto name = "asin"; + // https://dev.mysql.com/doc/refman/8.4/en/mathematical-functions.html#function_asin + static constexpr bool is_invalid_input(Float64 x) { return x < -1 || x > 1; } }; -using FunctionAsin = FunctionMathUnary>; +using FunctionAsin = + FunctionMathUnaryAlwayNullable>; struct AtanName { static constexpr auto name = "atan"; @@ -242,8 +249,11 @@ using FunctionSin = FunctionMathUnary; struct SqrtName { static constexpr auto name = "sqrt"; + // https://dev.mysql.com/doc/refman/8.4/en/mathematical-functions.html#function_sqrt + static constexpr bool is_invalid_input(Float64 x) { return x < 0; } }; -using FunctionSqrt = FunctionMathUnary>; +using FunctionSqrt = + FunctionMathUnaryAlwayNullable>; struct CbrtName { static constexpr auto name = "cbrt"; diff --git a/be/src/vec/sink/writer/vfile_result_writer.cpp b/be/src/vec/sink/writer/vfile_result_writer.cpp index 16491311c17ae65..5161cf2928d4097 100644 --- a/be/src/vec/sink/writer/vfile_result_writer.cpp +++ b/be/src/vec/sink/writer/vfile_result_writer.cpp @@ -84,6 +84,11 @@ VFileResultWriter::VFileResultWriter( Status VFileResultWriter::open(RuntimeState* state, RuntimeProfile* profile) { _state = state; _init_profile(profile); + // check orc writer version + if (_file_opts->file_format == TFileFormatType::FORMAT_ORC && + _file_opts->orc_writer_version < 1) { + return Status::InternalError("orc writer version is less than 1."); + } // Delete existing files if (_file_opts->delete_existing_files) { RETURN_IF_ERROR(_delete_dir()); diff --git a/be/test/olap/cumulative_compaction_time_series_policy_test.cpp b/be/test/olap/cumulative_compaction_time_series_policy_test.cpp index 01963d591bedabc..3e88e424e43c0b9 100644 --- a/be/test/olap/cumulative_compaction_time_series_policy_test.cpp +++ b/be/test/olap/cumulative_compaction_time_series_policy_test.cpp @@ -404,7 +404,7 @@ TEST_F(TestTimeSeriesCumulativeCompactionPolicy, calc_cumulative_compaction_scor const uint32_t score = _tablet->calc_compaction_score(CompactionType::CUMULATIVE_COMPACTION, cumulative_compaction_policy); - EXPECT_EQ(0, score); + EXPECT_EQ(9, score); } TEST_F(TestTimeSeriesCumulativeCompactionPolicy, calc_cumulative_compaction_score_big_rowset) { diff --git a/be/test/olap/segcompaction_mow_test.cpp b/be/test/olap/segcompaction_mow_test.cpp index a0c180f7fbf8456..62a3232889dedee 100644 --- a/be/test/olap/segcompaction_mow_test.cpp +++ b/be/test/olap/segcompaction_mow_test.cpp @@ -47,7 +47,7 @@ using namespace ErrorCode; static const uint32_t MAX_PATH_LEN = 1024; static const uint32_t TABLET_ID = 12345; -static StorageEngine* s_engine; +static StorageEngine* s_engine = nullptr; static const std::string lTestDir = "./data_test/data/segcompaction_mow_test"; class SegCompactionMoWTest : public ::testing::TestWithParam { @@ -76,22 +76,29 @@ class SegCompactionMoWTest : public ::testing::TestWithParam { options.store_paths = paths; auto engine = std::make_unique(options); + Status s = engine->open(); + EXPECT_TRUE(s.ok()) << s.to_string(); s_engine = engine.get(); ExecEnv::GetInstance()->set_storage_engine(std::move(engine)); - Status s = s_engine->open(); + s = ThreadPoolBuilder("SegCompactionTaskThreadPool") + .set_min_threads(config::segcompaction_num_threads) + .set_max_threads(config::segcompaction_num_threads) + .build(&s_engine->_seg_compaction_thread_pool); EXPECT_TRUE(s.ok()) << s.to_string(); _data_dir = std::make_unique(*s_engine, lTestDir); static_cast(_data_dir->update_capacity()); EXPECT_TRUE(io::global_local_filesystem()->create_directory(lTestDir).ok()); - - s = s_engine->start_bg_threads(); - EXPECT_TRUE(s.ok()) << s.to_string(); } - void TearDown() { config::enable_segcompaction = false; } + void TearDown() { + config::enable_segcompaction = false; + ExecEnv* exec_env = doris::ExecEnv::GetInstance(); + s_engine = nullptr; + exec_env->set_storage_engine(nullptr); + } protected: OlapReaderStatistics _stats; diff --git a/be/test/olap/segcompaction_test.cpp b/be/test/olap/segcompaction_test.cpp index 468b24784af3c21..ba0d23acb02cef0 100644 --- a/be/test/olap/segcompaction_test.cpp +++ b/be/test/olap/segcompaction_test.cpp @@ -46,7 +46,7 @@ namespace doris { using namespace ErrorCode; static const uint32_t MAX_PATH_LEN = 1024; -static StorageEngine* l_engine; +static StorageEngine* l_engine = nullptr; static const std::string lTestDir = "./data_test/data/segcompaction_test"; class SegCompactionTest : public testing::Test { @@ -75,22 +75,30 @@ class SegCompactionTest : public testing::Test { options.store_paths = paths; auto engine = std::make_unique(options); + Status s = engine->open(); + EXPECT_TRUE(s.ok()) << s.to_string(); + l_engine = engine.get(); ExecEnv::GetInstance()->set_storage_engine(std::move(engine)); - Status s = l_engine->open(); + s = ThreadPoolBuilder("SegCompactionTaskThreadPool") + .set_min_threads(config::segcompaction_num_threads) + .set_max_threads(config::segcompaction_num_threads) + .build(&l_engine->_seg_compaction_thread_pool); EXPECT_TRUE(s.ok()) << s.to_string(); _data_dir = std::make_unique(*l_engine, lTestDir); static_cast(_data_dir->update_capacity()); EXPECT_TRUE(io::global_local_filesystem()->create_directory(lTestDir).ok()); - - s = l_engine->start_bg_threads(); - EXPECT_TRUE(s.ok()) << s.to_string(); } - void TearDown() { config::enable_segcompaction = false; } + void TearDown() { + config::enable_segcompaction = false; + ExecEnv* exec_env = doris::ExecEnv::GetInstance(); + l_engine = nullptr; + exec_env->set_storage_engine(nullptr); + } protected: OlapReaderStatistics _stats; diff --git a/be/test/vec/core/get_common_type_test.cpp b/be/test/vec/core/get_common_type_test.cpp index c8c60a3776cd2d2..86b7299176dba0d 100644 --- a/be/test/vec/core/get_common_type_test.cpp +++ b/be/test/vec/core/get_common_type_test.cpp @@ -22,6 +22,7 @@ #include "gtest/gtest_pred_impl.h" #include "vec/data_types/data_type.h" +#include "vec/data_types/data_type_jsonb.h" #include "vec/data_types/data_type_nothing.h" #include "vec/data_types/data_type_number.h" #include "vec/data_types/data_type_string.h" @@ -67,6 +68,8 @@ static DataTypePtr typeFromString(const std::string& str) { return std::make_shared(); } else if (str == "String") { return std::make_shared(); + } else if (str == "Jsonb") { + return std::make_shared(); } return nullptr; } @@ -141,12 +144,7 @@ INSTANTIATE_TEST_SUITE_P(data_type, LeastSuperTypeTest, {"Int8 Int32 Int64", "Int64"}, {"UInt8 UInt32 Int64", "Int64"}, {"Float32 Float64", "Float64"}, + {"Date Date", "Date"}, {"Float32 UInt16 Int32", "Float64"}, {"Float32 Int16 UInt32", "Float64"}, - {"Date Date", "Date"}, - {"DateTime DateTime", "DateTime"}, - {"String String String", "String"}, - {"Int8 String", nullptr}, - {"Int64 UInt64", nullptr}, - {"Float32 UInt64", nullptr}, - {"Float64 Int64", nullptr}})); + {"String String String", "String"}})); diff --git a/cloud/CMakeLists.txt b/cloud/CMakeLists.txt index bb697d791e7116f..7a273dd04e29c4b 100644 --- a/cloud/CMakeLists.txt +++ b/cloud/CMakeLists.txt @@ -435,7 +435,7 @@ endif() if (NOT EXISTS "${THIRDPARTY_SRC}/${FDB_LIB}") file(MAKE_DIRECTORY ${THIRDPARTY_SRC}) - execute_process(COMMAND "curl" "${FDB_LIB_URL}" + execute_process(COMMAND "curl --retry 10 --retry-delay 2 --retry-max-time 30" "${FDB_LIB_URL}" "-o" "${THIRDPARTY_SRC}/${FDB_LIB}" "-k" RESULTS_VARIABLE DOWNLOAD_RET) if (NOT ${DOWNLOAD_RET} STREQUAL "0") diff --git a/cloud/script/start.sh b/cloud/script/start.sh index 2e444ac1778e6d4..582c80c2e6fa4c4 100644 --- a/cloud/script/start.sh +++ b/cloud/script/start.sh @@ -82,7 +82,7 @@ if ldd "${bin}" | grep -Ei 'libfdb_c.*not found' &>/dev/null; then exit 1 fi patchelf --set-rpath "${lib_path}" "${bin}" - ldd "${bin}" + # ldd "${bin}" fi chmod 550 "${DORIS_HOME}/lib/doris_cloud" @@ -91,7 +91,7 @@ if [[ -z "${JAVA_HOME}" ]]; then echo "The JAVA_HOME environment variable is not defined correctly" echo "This environment variable is needed to run this program" echo "NB: JAVA_HOME should point to a JDK not a JRE" - echo "You can set it in be.conf" + echo "You can set it in doris_cloud.conf" exit 1 fi @@ -122,7 +122,10 @@ fi echo "LIBHDFS3_CONF=${LIBHDFS3_CONF}" -export JEMALLOC_CONF="percpu_arena:percpu,background_thread:true,metadata_thp:auto,muzzy_decay_ms:5000,dirty_decay_ms:5000,oversize_threshold:0,prof:false,lg_prof_interval:-1" +# to enable dump jeprof heap stats prodigally, change `prof:false` to `prof:true` +# to control the dump interval change `lg_prof_interval` to a specific value, it is pow/exponent of 2 in size of bytes, default 34 means 2 ** 34 = 16GB +# to control the dump path, change `prof_prefix` to a specific path, e.g. /doris_cloud/log/ms_, by default it dumps at the path where the start command called +export JEMALLOC_CONF="percpu_arena:percpu,background_thread:true,metadata_thp:auto,muzzy_decay_ms:5000,dirty_decay_ms:5000,oversize_threshold:0,prof_prefix:ms_,prof:false,lg_prof_interval:34" if [[ "${RUN_VERSION}" -eq 1 ]]; then "${bin}" --version @@ -131,14 +134,22 @@ fi mkdir -p "${DORIS_HOME}/log" echo "starts ${process} with args: $*" +out_file=${DORIS_HOME}/log/${process}.out if [[ "${RUN_DAEMON}" -eq 1 ]]; then - date >>"${DORIS_HOME}/log/${process}.out" - nohup "${bin}" "$@" >>"${DORIS_HOME}/log/${process}.out" 2>&1 & - # wait for log flush - sleep 1.5 - tail -n10 "${DORIS_HOME}/log/${process}.out" | grep 'working directory' -B1 -A10 - echo "please check process log for more details" - echo "" + # append 10 blank lines to ensure the following tail -n10 works correctly + printf "\n\n\n\n\n\n\n\n\n\n" >>"${out_file}" + echo "$(date +'%F %T') try to start ${process}" >>"${out_file}" + nohup "${bin}" "$@" >>"${out_file}" 2>&1 & + echo "wait and check ${process} start successfully" + sleep 3 + tail -n10 "${out_file}" | grep 'successfully started brpc' + ret=$? + if [[ ${ret} -ne 0 ]]; then + echo "${process} may not start successfully please check process log for more details" + exit 1 + fi + echo "${process} start successfully" + exit 0 elif [[ "${RUN_CONSOLE}" -eq 1 ]]; then export DORIS_LOG_TO_STDERR=1 date diff --git a/cloud/script/stop.sh b/cloud/script/stop.sh index b40b4061c398dc5..48f01c545ae840e 100644 --- a/cloud/script/stop.sh +++ b/cloud/script/stop.sh @@ -32,4 +32,17 @@ fi pid=$(cat "${DORIS_HOME}/bin/${process}.pid") kill -2 "${pid}" +cnt=0 +while true; do + cnt=$((cnt + 1)) + echo "waiting ${pid} to quit, ${cnt} seconds elapsed" + msg=$(ps "${pid}") + ret=$? + if [[ ${ret} -ne 0 ]]; then + echo "${pid} has quit" + break + fi + echo "${msg}" + sleep 1 +done rm -f "${DORIS_HOME}/bin/${process}.pid" diff --git a/cloud/src/common/config.h b/cloud/src/common/config.h index d401caa4ad975f8..b1db41a6eb734eb 100644 --- a/cloud/src/common/config.h +++ b/cloud/src/common/config.h @@ -55,7 +55,7 @@ CONF_Bool(enable_file_logger, "true"); // recycler config CONF_mInt64(recycle_interval_seconds, "3600"); -CONF_mInt64(retention_seconds, "259200"); // 72h +CONF_mInt64(retention_seconds, "259200"); // 72h, global retention time CONF_Int32(recycle_concurrency, "16"); CONF_Int32(recycle_job_lease_expired_ms, "60000"); CONF_mInt64(compacted_rowset_retention_seconds, "10800"); // 3h @@ -77,7 +77,9 @@ CONF_mInt32(scan_instances_interval_seconds, "60"); // 1min CONF_mInt32(check_object_interval_seconds, "43200"); // 12hours CONF_mInt64(check_recycle_task_interval_seconds, "600"); // 10min -CONF_mInt64(recycle_task_threshold_seconds, "10800"); // 3h +CONF_mInt64(recycler_sleep_before_scheduling_seconds, "60"); +// log a warning if a recycle task takes longer than this duration +CONF_mInt64(recycle_task_threshold_seconds, "10800"); // 3h CONF_String(test_s3_ak, ""); CONF_String(test_s3_sk, ""); diff --git a/cloud/src/main.cpp b/cloud/src/main.cpp index 9356a3546d03a95..74e6a8daaf161c5 100644 --- a/cloud/src/main.cpp +++ b/cloud/src/main.cpp @@ -161,13 +161,13 @@ DECLARE_int64(socket_max_unwritten_bytes); int main(int argc, char** argv) { if (argc > 1) { if (auto ret = args.parse(argc - 1, argv + 1); !ret.empty()) { - std::cerr << ret << std::endl; + std::cerr << "parse arguments error: " << ret << std::endl; help(); return -1; } } - if (argc < 2 || args.get(ARG_HELP)) { + if (args.get(ARG_HELP)) { help(); return 0; } @@ -177,21 +177,16 @@ int main(int argc, char** argv) { return 0; } - // FIXME(gavin): do we need to enable running both MS and recycler within - // single process - if (!(args.get(ARG_META_SERVICE) ^ args.get(ARG_RECYCLER))) { - std::cerr << "only one of --meta-service and --recycler must be specified" << std::endl; - return 1; - } - - // There may be more roles to play + // There may be more roles to play in the future, if there are multi roles specified, + // use meta_service as the process name std::string process_name = args.get(ARG_META_SERVICE) ? "meta_service" : args.get(ARG_RECYCLER) ? "recycler" - : ""; - if (process_name.empty()) { - std::cerr << "failed to determine prcess name with given args" << std::endl; - return 1; - } + : "meta_service"; + + using namespace std::chrono; + + auto start = steady_clock::now(); + auto end = start; auto pid_file_fd_holder = gen_pidfile("doris_cloud"); if (pid_file_fd_holder == nullptr) { @@ -215,11 +210,19 @@ int main(int argc, char** argv) { } // We can invoke glog from now on - std::string msg; + LOG(INFO) << "try to start doris_cloud"; LOG(INFO) << build_info(); std::cout << build_info() << std::endl; + if (!args.get(ARG_META_SERVICE) && !args.get(ARG_RECYCLER)) { + std::get<0>(args.args()[ARG_META_SERVICE]) = true; + std::get<0>(args.args()[ARG_RECYCLER]) = true; + LOG(INFO) << "meta_service and recycler are both not specified, " + "run doris_cloud as meta_service and recycler by default"; + std::cout << "run doris_cloud as meta_service and recycler by default" << std::endl; + } + brpc::Server server; brpc::FLAGS_max_body_size = config::brpc_max_body_size; brpc::FLAGS_socket_max_unwritten_bytes = config::brpc_socket_max_unwritten_bytes; @@ -238,19 +241,22 @@ int main(int argc, char** argv) { return 1; } LOG(INFO) << "begin to init txn kv"; + auto start_init_kv = steady_clock::now(); int ret = txn_kv->init(); if (ret != 0) { LOG(WARNING) << "failed to init txnkv, ret=" << ret; return 1; } - LOG(INFO) << "successfully init txn kv"; + end = steady_clock::now(); + LOG(INFO) << "successfully init txn kv, elapsed milliseconds: " + << duration_cast(end - start_init_kv).count(); if (init_global_encryption_key_info_map(txn_kv.get()) != 0) { LOG(WARNING) << "failed to init global encryption key map"; return -1; } - std::unique_ptr meta_server; + std::unique_ptr meta_server; // meta-service std::unique_ptr recycler; std::thread periodiccally_log_thread; std::mutex periodiccally_log_thread_lock; @@ -269,7 +275,8 @@ int main(int argc, char** argv) { msg = "meta-service started"; LOG(INFO) << msg; std::cout << msg << std::endl; - } else if (args.get(ARG_RECYCLER)) { + } + if (args.get(ARG_RECYCLER)) { recycler = std::make_unique(txn_kv); int ret = recycler->start(&server); if (ret != 0) { @@ -284,15 +291,12 @@ int main(int argc, char** argv) { auto periodiccally_log = [&]() { while (periodiccally_log_thread_run) { std::unique_lock lck {periodiccally_log_thread_lock}; - periodiccally_log_thread_cv.wait_for( - lck, std::chrono::milliseconds(config::periodically_log_ms)); + periodiccally_log_thread_cv.wait_for(lck, + milliseconds(config::periodically_log_ms)); LOG(INFO) << "Periodically log for recycler"; } }; periodiccally_log_thread = std::thread {periodiccally_log}; - } else { - std::cerr << "cloud starts without doing anything and exits" << std::endl; - return -1; } // start service brpc::ServerOptions options; @@ -309,7 +313,11 @@ int main(int argc, char** argv) { << ", errmsg=" << strerror_r(errno, buf, 64) << ", port=" << port; return -1; } - LOG(INFO) << "successfully started brpc listening on port=" << port; + end = steady_clock::now(); + msg = "successfully started brpc listening on port=" + std::to_string(port) + + " time_elapsed_ms=" + std::to_string(duration_cast(end - start).count()); + LOG(INFO) << msg; + std::cout << msg << std::endl; server.RunUntilAskedToQuit(); // Wait for signals server.ClearServices(); @@ -326,7 +334,7 @@ int main(int argc, char** argv) { periodiccally_log_thread_run = false; // immediately notify the log thread to quickly exit in case it block the // whole procedure - periodiccally_log_thread_cv.notify_one(); + periodiccally_log_thread_cv.notify_all(); } periodiccally_log_thread.join(); } diff --git a/cloud/src/meta-service/meta_service_txn.cpp b/cloud/src/meta-service/meta_service_txn.cpp index 3cd1bd798bbc841..5f2638c8d1fa28a 100644 --- a/cloud/src/meta-service/meta_service_txn.cpp +++ b/cloud/src/meta-service/meta_service_txn.cpp @@ -479,8 +479,8 @@ void MetaServiceImpl::precommit_txn(::google::protobuf::RpcController* controlle return; } - LOG(INFO) << "xxx put running_key=" << hex(running_key) << " txn_id=" << txn_id; txn->put(running_key, running_val); + LOG(INFO) << "xxx put running_key=" << hex(running_key) << " txn_id=" << txn_id; err = txn->commit(); if (err != TxnErrorCode::TXN_OK) { @@ -569,8 +569,6 @@ void put_routine_load_progress(MetaServiceCode& code, std::string& msg, new_statistic_info->set_task_execution_time_ms(commit_attachment.task_execution_time_ms()); } - LOG(INFO) << "routine load new progress: " << new_progress_info.ShortDebugString(); - if (!new_progress_info.SerializeToString(&new_progress_val)) { code = MetaServiceCode::PROTOBUF_SERIALIZE_ERR; ss << "failed to serialize new progress val, txn_id=" << txn_id; @@ -579,6 +577,8 @@ void put_routine_load_progress(MetaServiceCode& code, std::string& msg, } txn->put(rl_progress_key, new_progress_val); + LOG(INFO) << "put rl_progress_key key=" << hex(rl_progress_key) + << " routine load new progress: " << new_progress_info.ShortDebugString(); } void MetaServiceImpl::get_rl_task_commit_attach(::google::protobuf::RpcController* controller, @@ -689,6 +689,7 @@ void MetaServiceImpl::reset_rl_progress(::google::protobuf::RpcController* contr if (request->partition_to_offset().size() == 0) { txn->remove(rl_progress_key); + LOG(INFO) << "remove rl_progress_key key=" << hex(rl_progress_key); } if (request->partition_to_offset().size() > 0) { @@ -738,6 +739,7 @@ void MetaServiceImpl::reset_rl_progress(::google::protobuf::RpcController* contr return; } txn->put(rl_progress_key, new_progress_val); + LOG(INFO) << "put rl_progress_key key=" << hex(rl_progress_key); } err = txn->commit(); @@ -892,6 +894,7 @@ void update_tablet_stats(const StatsTabletKeyInfo& info, const TabletStats& stat stats_pb.set_num_segments(stats_pb.num_segments() + stats.num_segs); stats_pb.SerializeToString(&val); txn->put(key, val); + LOG(INFO) << "put stats_tablet_key key=" << hex(key); } } @@ -2370,6 +2373,7 @@ void commit_txn_with_sub_txn(const CommitTxnRequest* request, CommitTxnResponse* stats_pb.set_num_segments(stats_pb.num_segments() + stats.num_segs); stats_pb.SerializeToString(&val); txn->put(key, val); + LOG(INFO) << "put stats_tablet_key, key=" << hex(key); }; } for (auto& [tablet_id, stats] : tablet_stats) { @@ -2405,8 +2409,9 @@ void commit_txn_with_sub_txn(const CommitTxnRequest* request, CommitTxnResponse* return; } txn->put(recycle_key, recycle_val); + LOG(INFO) << "xxx commit_txn put recycle_txn_key key=" << hex(recycle_key) + << " txn_id=" << txn_id; - LOG(INFO) << "xxx commit_txn put recycle_key key=" << hex(recycle_key) << " txn_id=" << txn_id; LOG(INFO) << "commit_txn put_size=" << txn->put_bytes() << " del_size=" << txn->delete_bytes() << " num_put_keys=" << txn->num_put_keys() << " num_del_keys=" << txn->num_del_keys() << " txn_size=" << txn->approximate_bytes() << " txn_id=" << txn_id; @@ -2541,7 +2546,7 @@ static void _abort_txn(const std::string& instance_id, const AbortTxnRequest* re //1. abort txn by txn id //2. abort txn by label and db_id if (txn_id > 0) { - VLOG_DEBUG << "abort_txn by txn_id"; + VLOG_DEBUG << "abort_txn by txn_id, txn_id=" << txn_id; //abort txn by txn id // Get db id with txn id @@ -2610,7 +2615,7 @@ static void _abort_txn(const std::string& instance_id, const AbortTxnRequest* re return; } } else { - VLOG_DEBUG << "abort_txn by db_id and txn label"; + VLOG_DEBUG << "abort_txn db_id and label, db_id=" << db_id << " label=" << label; //abort txn by label. std::string label_key = txn_label_key({instance_id, db_id, label}); std::string label_val; @@ -2726,7 +2731,7 @@ static void _abort_txn(const std::string& instance_id, const AbortTxnRequest* re return; } txn->put(recycle_key, recycle_val); - LOG(INFO) << "xxx put recycle_key=" << hex(recycle_key) + LOG(INFO) << "put recycle_txn_key=" << hex(recycle_key) << " txn_id=" << return_txn_info.txn_id(); } diff --git a/cloud/src/recycler/checker.cpp b/cloud/src/recycler/checker.cpp index 49421f97ca0aeb1..c3e9f69ed9d6bc5 100644 --- a/cloud/src/recycler/checker.cpp +++ b/cloud/src/recycler/checker.cpp @@ -79,6 +79,8 @@ int Checker::start() { // launch instance scanner auto scanner_func = [this]() { + std::this_thread::sleep_for( + std::chrono::seconds(config::recycler_sleep_before_scheduling_seconds)); while (!stopped()) { std::vector instances; get_all_instances(txn_kv_.get(), instances); diff --git a/cloud/src/recycler/hdfs_accessor.cpp b/cloud/src/recycler/hdfs_accessor.cpp index e5038735f5735f6..97a4670d2bfc6ba 100644 --- a/cloud/src/recycler/hdfs_accessor.cpp +++ b/cloud/src/recycler/hdfs_accessor.cpp @@ -354,8 +354,27 @@ int HdfsAccessor::init() { } int HdfsAccessor::delete_prefix(const std::string& path_prefix, int64_t expiration_time) { - LOG_INFO("delete prefix").tag("uri", to_uri(path_prefix)); // Audit log - return 0; + auto uri = to_uri(path_prefix); + LOG(INFO) << "delete prefix, uri=" << uri; + std::unique_ptr list_iter; + int ret = list_all(&list_iter); + if (ret != 0) { + LOG(WARNING) << "delete prefix, failed to list" << uri; + return ret; + } + size_t num_listed = 0, num_deleted = 0; + for (auto file = list_iter->next(); file; file = list_iter->next()) { + ++num_listed; + if (file->path.find(path_prefix) != 0) continue; + if (int del_ret = delete_file(file->path); del_ret != 0) { + ret = del_ret; + break; + } + ++num_deleted; + } + LOG(INFO) << "delete prefix " << (ret != 0 ? "failed" : "succ") << " ret=" << ret + << " uri=" << uri << " num_listed=" << num_listed << " num_deleted=" << num_deleted; + return ret; } int HdfsAccessor::delete_directory_impl(const std::string& dir_path) { diff --git a/cloud/src/recycler/recycler.cpp b/cloud/src/recycler/recycler.cpp index 919b50358c77f52..76d4a7ca767a55e 100644 --- a/cloud/src/recycler/recycler.cpp +++ b/cloud/src/recycler/recycler.cpp @@ -189,6 +189,11 @@ Recycler::~Recycler() { } void Recycler::instance_scanner_callback() { + // sleep 60 seconds before scheduling for the launch procedure to complete: + // some bad hdfs connection may cause some log to stdout stderr + // which may pollute .out file and affect the script to check success + std::this_thread::sleep_for( + std::chrono::seconds(config::recycler_sleep_before_scheduling_seconds)); while (!stopped()) { std::vector instances; get_all_instances(txn_kv_.get(), instances); @@ -1219,7 +1224,7 @@ int InstanceRecycler::recycle_tablets(int64_t table_id, int64_t index_id, int64_ return {std::string_view(), range_move}; } ++num_recycled; - LOG_INFO("k is {}, is empty {}", k, k.empty()); + LOG(INFO) << "recycle_tablets scan, key=" << (k.empty() ? "(empty)" : hex(k)); return {k, range_move}; }); } else { @@ -1694,6 +1699,10 @@ int InstanceRecycler::recycle_rowsets() { LOG_WARNING("malformed recycle rowset").tag("key", hex(k)); return -1; } + + VLOG_DEBUG << "recycle rowset scan, key=" << hex(k) << " num_scanned=" << num_scanned + << " num_expired=" << num_expired << " expiration=" << calc_expiration(rowset) + << " RecycleRowsetPB=" << rowset.ShortDebugString(); int64_t current_time = ::time(nullptr); if (current_time < calc_expiration(rowset)) { // not expired return 0; @@ -1745,8 +1754,8 @@ int InstanceRecycler::recycle_rowsets() { << rowset_meta->start_version() << '-' << rowset_meta->end_version() << "] txn_id=" << rowset_meta->txn_id() << " type=" << RecycleRowsetPB_Type_Name(rowset.type()) - << " rowset_meta_size=" << v.size() << " creation_time" - << rowset_meta->creation_time(); + << " rowset_meta_size=" << v.size() + << " creation_time=" << rowset_meta->creation_time(); if (rowset.type() == RecycleRowsetPB::PREPARE) { // unable to calculate file path, can only be deleted by rowset id prefix num_prepare += 1; @@ -1910,6 +1919,10 @@ int InstanceRecycler::recycle_tmp_rowsets() { // duration or timeout always < `retention_time` in practice. int64_t expiration = rowset.txn_expiration() > 0 ? rowset.txn_expiration() : rowset.creation_time(); + VLOG_DEBUG << "recycle tmp rowset scan, key=" << hex(k) << " num_scanned=" << num_scanned + << " num_expired=" << num_expired << " expiration=" << expiration + << " txn_expiration=" << rowset.txn_expiration() + << " rowset_creation_time=" << rowset.creation_time(); if (current_time < expiration + config::retention_seconds) { // not expired return 0; @@ -2150,7 +2163,7 @@ int InstanceRecycler::recycle_expired_txn_label() { recycle_txn_key(recycle_txn_key_info0, &begin_recycle_txn_key); recycle_txn_key(recycle_txn_key_info1, &end_recycle_txn_key); - LOG_INFO("begin to recycle expire txn").tag("instance_id", instance_id_); + LOG_INFO("begin to recycle expired txn").tag("instance_id", instance_id_); int64_t start_time = duration_cast(steady_clock::now().time_since_epoch()).count(); register_recycle_task(task_name, start_time); diff --git a/cloud/src/recycler/recycler_service.cpp b/cloud/src/recycler/recycler_service.cpp index 3c1a5b2ab48d95b..08e937a4106ba55 100644 --- a/cloud/src/recycler/recycler_service.cpp +++ b/cloud/src/recycler/recycler_service.cpp @@ -448,7 +448,7 @@ void RecyclerServiceImpl::http(::google::protobuf::RpcController* controller, } status_code = 404; - msg = "not found"; + msg = "http path " + uri.path() + " not found, it may be not implemented"; response_body = msg; } diff --git a/cloud/test/recycler_test.cpp b/cloud/test/recycler_test.cpp index d767c1bd8b73937..146873548398bf0 100644 --- a/cloud/test/recycler_test.cpp +++ b/cloud/test/recycler_test.cpp @@ -64,6 +64,7 @@ int main(int argc, char** argv) { using namespace std::chrono; current_time = duration_cast(system_clock::now().time_since_epoch()).count(); + config::recycler_sleep_before_scheduling_seconds = 0; // we dont have to wait in UT ::testing::InitGoogleTest(&argc, argv); auto s3_producer_pool = std::make_shared(config::recycle_pool_parallelism); diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index 81e2c602c6e1726..9b61a734246828b 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -2042,7 +2042,7 @@ public class Config extends ConfigBase { * Decrease this value if FE's memory is small */ @ConfField(mutable = false, masterOnly = false) - public static long max_external_file_cache_num = 100000; + public static long max_external_file_cache_num = 10000; /** * Max cache num of external table's schema diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 index 42092b896822c2d..2343f208642d6e0 100644 --- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 +++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 @@ -554,6 +554,7 @@ VAULT: 'VAULT'; VERBOSE: 'VERBOSE'; VERSION: 'VERSION'; VIEW: 'VIEW'; +VIEWS: 'VIEWS'; WARM: 'WARM'; WARNINGS: 'WARNINGS'; WEEK: 'WEEK'; diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 index b46b55a96e0da8f..087af9d717de135 100644 --- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 +++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 @@ -38,7 +38,7 @@ statement | CALL name=multipartIdentifier LEFT_PAREN (expression (COMMA expression)*)? RIGHT_PAREN #callProcedure | (ALTER | CREATE (OR REPLACE)? | REPLACE) (PROCEDURE | PROC) name=multipartIdentifier LEFT_PAREN .*? RIGHT_PAREN .*? #createProcedure | DROP (PROCEDURE | PROC) (IF EXISTS)? name=multipartIdentifier #dropProcedure - | SHOW PROCEDURE STATUS (LIKE pattern=valueExpression | whereClause)? #showProcedureStatus + | SHOW (PROCEDURE | FUNCTION) STATUS (LIKE pattern=valueExpression | whereClause)? #showProcedureStatus | SHOW CREATE PROCEDURE name=multipartIdentifier #showCreateProcedure // FIXME: like should be wildWhere? FRONTEND should not contain FROM backendid | ADMIN? SHOW type=(FRONTEND | BACKEND) CONFIG (LIKE pattern=valueExpression)? (FROM backendId=INTEGER_VALUE)? #showConfig @@ -49,7 +49,7 @@ statementBase | supportedDmlStatement #supportedDmlStatementAlias | supportedCreateStatement #supportedCreateStatementAlias | supportedAlterStatement #supportedAlterStatementAlias - | materailizedViewStatement #materailizedViewStatementAlias + | materializedViewStatement #materializedViewStatementAlias | constraintStatement #constraintStatementAlias | supportedDropStatement #supportedDropStatementAlias | unsupportedStatement #unsupported @@ -75,10 +75,11 @@ unsupportedStatement | unsupportedCleanStatement | unsupportedRefreshStatement | unsupportedLoadStatement + | unsupportedShowStatement | unsupportedOtherStatement ; -materailizedViewStatement +materializedViewStatement : CREATE MATERIALIZED VIEW (IF NOT EXISTS)? mvName=multipartIdentifier (LEFT_PAREN cols=simpleColumnDefs RIGHT_PAREN)? buildMode? (REFRESH refreshMethod? refreshTrigger?)? @@ -202,6 +203,121 @@ lockTable (READ (LOCAL)? | (LOW_PRIORITY)? WRITE) ; +unsupportedShowStatement + : SHOW SQL_BLOCK_RULE (FOR ruleName=identifier)? #showSqlBlockRule + | SHOW ROW POLICY (FOR (userIdentify | (ROLE role=identifier)))? #showRowPolicy + | SHOW STORAGE POLICY (USING (FOR policy=identifierOrText)?)? #showStoragePolicy + | SHOW STAGES #showStages + | SHOW STORAGE VAULT #showStorageVault + | SHOW CREATE REPOSITORY FOR identifier #showCreateRepository + | SHOW WHITELIST #showWhitelist + | SHOW (GLOBAL | SESSION | LOCAL)? VARIABLES wildWhere? #showVariables + | SHOW OPEN TABLES ((FROM | IN) database=multipartIdentifier)? wildWhere? #showOpenTables + | SHOW TABLE STATUS ((FROM | IN) database=multipartIdentifier)? wildWhere? #showTableStatus + | SHOW FULL? TABLES ((FROM | IN) database=multipartIdentifier)? wildWhere? #showTables + | SHOW FULL? VIEWS ((FROM | IN) database=multipartIdentifier)? wildWhere? #showViews + | SHOW TABLE tableId=INTEGER_VALUE #showTableId + | SHOW FULL? PROCESSLIST #showProcessList + | SHOW (GLOBAL | SESSION | LOCAL)? STATUS wildWhere? #showStatus + | SHOW FULL? TRIGGERS ((FROM | IN) database=multipartIdentifier)? wildWhere? #showTriggers + | SHOW EVENTS ((FROM | IN) database=multipartIdentifier)? wildWhere? #showEvents + | SHOW PLUGINS #showPlugins + | SHOW STORAGE? ENGINES #showStorageEngines + | SHOW AUTHORS #showAuthors + | SHOW BRIEF? CREATE TABLE name=multipartIdentifier #showCreateTable + | SHOW CREATE VIEW name=multipartIdentifier #showCreateView + | SHOW CREATE MATERIALIZED VIEW name=multipartIdentifier #showMaterializedView + | SHOW CREATE (DATABASE | SCHEMA) name=multipartIdentifier #showCreateDatabase + | SHOW CREATE CATALOG name=identifier #showCreateCatalog + | SHOW CREATE (GLOBAL | SESSION | LOCAL)? FUNCTION functionIdentifier + LEFT_PAREN functionArguments? RIGHT_PAREN + ((FROM | IN) database=multipartIdentifier)? #showCreateFunction + | SHOW (DATABASES | SCHEMAS) (FROM catalog=identifier)? wildWhere? #showDatabases + | SHOW DATABASE databaseId=INTEGER_VALUE #showDatabaseId + | SHOW DATA TYPES #showDataTypes + | SHOW CATALOGS wildWhere? #showCatalogs + | SHOW CATALOG name=identifier #showCatalog + | SHOW DYNAMIC PARTITION TABLES ((FROM | IN) database=multipartIdentifier)? #showDynamicPartition + | SHOW FULL? (COLUMNS | FIELDS) (FROM | IN) tableName=multipartIdentifier + ((FROM | IN) database=multipartIdentifier)? wildWhere? #showColumns + | SHOW COLLATION wildWhere? #showCollation + | SHOW ((CHAR SET) | CHARSET) wildWhere? #showCharset + | SHOW PROC path=STRING_LITERAL #showProc + | SHOW COUNT LEFT_PAREN ASTERISK RIGHT_PAREN (WARNINGS | ERRORS) #showWaringErrorCount + | SHOW (WARNINGS | ERRORS) limitClause? #showWaringErrors + | SHOW LOAD WARNINGS ((((FROM | IN) database=multipartIdentifier)? + wildWhere? limitClause?) | (ON url=STRING_LITERAL)) #showLoadWarings + | SHOW STREAM? LOAD ((FROM | IN) database=multipartIdentifier)? wildWhere? + sortClause? limitClause? #showLoad + | SHOW EXPORT ((FROM | IN) database=multipartIdentifier)? wildWhere? + sortClause? limitClause? #showExport + | SHOW DELETE ((FROM | IN) database=multipartIdentifier)? #showDelete + | SHOW ALTER TABLE (ROLLUP | (MATERIALIZED VIEW) | COLUMN) + ((FROM | IN) database=multipartIdentifier)? wildWhere? + sortClause? limitClause? #showAlterTable + | SHOW DATA SKEW FROM baseTableRef #showDataSkew + | SHOW DATA (FROM tableName=multipartIdentifier)? sortClause? propertyClause? #showData + | SHOW TEMPORARY? PARTITIONS FROM tableName=multipartIdentifier + wildWhere? sortClause? limitClause? #showPartitions + | SHOW PARTITION partitionId=INTEGER_VALUE #showPartitionId + | SHOW TABLET tabletId=INTEGER_VALUE #showTabletId + | SHOW TABLETS BELONG + tabletIds+=INTEGER_VALUE (COMMA tabletIds+=INTEGER_VALUE)* #showTabletBelong + | SHOW TABLETS FROM tableName=multipartIdentifier partitionSpec? + wildWhere? sortClause? limitClause? #showTabletsFromTable + | SHOW PROPERTY (FOR user=identifierOrText)? wildWhere? #showUserProperties + | SHOW ALL PROPERTIES wildWhere? #showAllProperties + | SHOW BACKUP ((FROM | IN) database=multipartIdentifier)? wildWhere? #showBackup + | SHOW BRIEF? RESTORE ((FROM | IN) database=multipartIdentifier)? wildWhere? #showRestore + | SHOW BROKER #showBroker + | SHOW RESOURCES wildWhere? sortClause? limitClause? #showResources + | SHOW WORKLOAD GROUPS wildWhere? #showWorkloadGroups + | SHOW BACKENDS #showBackends + | SHOW TRASH (ON backend=STRING_LITERAL)? #showTrash + | SHOW FRONTENDS name=identifier? #showFrontends + | SHOW REPOSITORIES #showRepositories + | SHOW SNAPSHOT ON repo=identifier wildWhere? #showSnapshot + | SHOW ALL? GRANTS #showGrants + | SHOW GRANTS FOR userIdentify #showGrantsForUser + | SHOW ROLES #showRoles + | SHOW PRIVILEGES #showPrivileges + | SHOW FULL? BUILTIN? FUNCTIONS + ((FROM | IN) database=multipartIdentifier)? wildWhere? #showFunctions + | SHOW GLOBAL FULL? FUNCTIONS wildWhere? #showGlobalFunctions + | SHOW TYPECAST ((FROM | IN) database=multipartIdentifier)? #showTypeCast + | SHOW FILE ((FROM | IN) database=multipartIdentifier)? #showSmallFiles + | SHOW (KEY | KEYS | INDEX | INDEXES) + (FROM |IN) tableName=multipartIdentifier + ((FROM | IN) database=multipartIdentifier)? #showIndex + | SHOW VIEW + (FROM |IN) tableName=multipartIdentifier + ((FROM | IN) database=multipartIdentifier)? #showView + | SHOW TRANSACTION ((FROM | IN) database=multipartIdentifier)? wildWhere? #showTransaction + | SHOW QUERY PROFILE queryIdPath=STRING_LITERAL #showQueryProfile + | SHOW LOAD PROFILE loadIdPath=STRING_LITERAL #showLoadProfile + | SHOW CACHE HOTSPOT tablePath=STRING_LITERAL #showCacheHotSpot + | SHOW ENCRYPTKEYS ((FROM | IN) database=multipartIdentifier)? wildWhere? #showEncryptKeys + | SHOW SYNC JOB ((FROM | IN) database=multipartIdentifier)? #showSyncJob + | SHOW TABLE CREATION ((FROM | IN) database=multipartIdentifier)? wildWhere? #showTableCreation + | SHOW LAST INSERT #showLastInsert + | SHOW CREATE MATERIALIZED VIEW mvName=identifier + ON tableName=multipartIdentifier #showCreateMaterializedView + | SHOW CATALOG RECYCLE BIN wildWhere? #showCatalogRecycleBin + | SHOW QUERY STATS ((FOR database=identifier) + | (FROM tableName=multipartIdentifier (ALL VERBOSE?)?))? #showQueryStats + | SHOW BUILD INDEX ((FROM | IN) database=multipartIdentifier)? + wildWhere? sortClause? limitClause? #showBuildIndex + | SHOW CLUSTERS #showClusters + | SHOW CONVERT_LSC ((FROM | IN) database=multipartIdentifier)? #showConvertLsc + | SHOW REPLICA STATUS FROM baseTableRef wildWhere? #showReplicaStatus + | SHOW REPLICA DISTRIBUTION FROM baseTableRef #showREplicaDistribution + | SHOW TABLET STORAGE FORMAT VERBOSE? #showTabletStorageFormat + | SHOW TABLET DIAGNOSIS tabletId=INTEGER_VALUE #showDiagnoseTablet + | SHOW COPY ((FROM | IN) database=multipartIdentifier)? + whereClause? sortClause? limitClause? #showCopy + | SHOW WARM UP JOB wildWhere? #showWarmUpJob + ; + unsupportedLoadStatement : LOAD mysqlDataDesc (PROPERTIES LEFT_PAREN properties=propertyItemList RIGHT_PAREN)? @@ -432,6 +548,7 @@ unsupportedAlterStatement properties=propertyClause #alterStoragePlicy | ALTER USER (IF EXISTS)? grantUserIdentify passwordOption (COMMENT STRING_LITERAL)? #alterUser + | ALTER ROLE role=identifier commentSpec #alterRole | ALTER REPOSITORY name=identifier properties=propertyClause? #alterRepository ; @@ -562,6 +679,18 @@ unsupportedStatsStatement | DROP EXPIRED STATS #dropExpiredStats | DROP ANALYZE JOB INTEGER_VALUE #dropAanalyzeJob | KILL ANALYZE jobId=INTEGER_VALUE #killAnalyzeJob + | SHOW TABLE STATS tableName=multipartIdentifier + partitionSpec? columnList=identifierList? #showTableStats + | SHOW TABLE STATS tableId=INTEGER_VALUE #showTableStats + | SHOW INDEX STATS tableName=multipartIdentifier indexId=identifier #showIndexStats + | SHOW COLUMN CACHED? STATS tableName=multipartIdentifier + columnList=identifierList? partitionSpec? #showColumnStats + | SHOW COLUMN HISTOGRAM tableName=multipartIdentifier + columnList=identifierList #showColumnHistogramStats + | SHOW AUTO? ANALYZE tableName=multipartIdentifier? wildWhere? #showAnalyze + | SHOW ANALYZE jobId=INTEGER_VALUE wildWhere? #showAnalyzeFromJobId + | SHOW AUTO JOBS tableName=multipartIdentifier? wildWhere? #showAutoAnalyzeJobs + | SHOW ANALYZE TASK STATUS jobId=INTEGER_VALUE #showAnalyzeTask ; analyzeProperties @@ -1952,6 +2081,7 @@ nonReserved | VERBOSE | VERSION | VIEW + | VIEWS | WARM | WARNINGS | WEEK diff --git a/fe/fe-core/src/main/cup/sql_parser.cup b/fe/fe-core/src/main/cup/sql_parser.cup index 921fdd590ed6070..29a05856ff3b8e5 100644 --- a/fe/fe-core/src/main/cup/sql_parser.cup +++ b/fe/fe-core/src/main/cup/sql_parser.cup @@ -718,7 +718,7 @@ nonterminal StatementBase stmt, show_stmt, show_param, help_stmt, load_stmt, use_stmt, use_cloud_cluster_stmt, kill_stmt, drop_stmt, recover_stmt, grant_stmt, revoke_stmt, create_stmt, set_stmt, sync_stmt, cancel_stmt, cancel_param, delete_stmt, switch_stmt, transaction_stmt, unsupported_stmt, export_stmt, admin_stmt, truncate_stmt, import_columns_stmt, import_delete_on_stmt, import_sequence_stmt, import_where_stmt, install_plugin_stmt, uninstall_plugin_stmt, - import_preceding_filter_stmt, unlock_tables_stmt, lock_tables_stmt, refresh_stmt, clean_stmt, analyze_stmt, show_mtmv_stmt, kill_analysis_job_stmt, insert_overwrite_stmt, copy_stmt, + import_preceding_filter_stmt, unlock_tables_stmt, lock_tables_stmt, refresh_stmt, clean_stmt, analyze_stmt, kill_analysis_job_stmt, insert_overwrite_stmt, copy_stmt, warm_up_stmt; nonterminal FromClause opt_using_clause; @@ -1272,8 +1272,6 @@ stmt ::= {: RESULT = stmt; :} | analyze_stmt:stmt {: RESULT = stmt; :} - | show_mtmv_stmt : stmt - {: RESULT = stmt; :} | warm_up_stmt:stmt {: RESULT = stmt; :} | /* empty: query only has comments */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java b/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java index 95ad5ae824bebcc..b4700ec32d41a7a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java @@ -44,6 +44,7 @@ import org.apache.doris.catalog.Column; import org.apache.doris.catalog.DataProperty; import org.apache.doris.catalog.Database; +import org.apache.doris.catalog.DatabaseIf; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.MTMV; import org.apache.doris.catalog.MysqlTable; @@ -53,6 +54,7 @@ import org.apache.doris.catalog.PartitionInfo; import org.apache.doris.catalog.ReplicaAllocation; import org.apache.doris.catalog.Table; +import org.apache.doris.catalog.TableIf; import org.apache.doris.catalog.TableIf.TableType; import org.apache.doris.catalog.View; import org.apache.doris.cloud.alter.CloudSchemaChangeHandler; @@ -65,6 +67,7 @@ import org.apache.doris.common.util.MetaLockUtils; import org.apache.doris.common.util.PropertyAnalyzer; import org.apache.doris.common.util.PropertyAnalyzer.RewriteProperty; +import org.apache.doris.datasource.ExternalTable; import org.apache.doris.nereids.trees.plans.commands.info.TableNameInfo; import org.apache.doris.persist.AlterMTMV; import org.apache.doris.persist.AlterViewInfo; @@ -72,6 +75,7 @@ import org.apache.doris.persist.ModifyCommentOperationLog; import org.apache.doris.persist.ModifyPartitionInfo; import org.apache.doris.persist.ModifyTableEngineOperationLog; +import org.apache.doris.persist.ModifyTablePropertyOperationLog; import org.apache.doris.persist.ReplaceTableOperationLog; import org.apache.doris.policy.StoragePolicy; import org.apache.doris.qe.ConnectContext; @@ -303,6 +307,24 @@ private boolean processAlterOlapTable(AlterTableStmt stmt, OlapTable olapTable, return needProcessOutsideTableLock; } + private void setExternalTableAutoAnalyzePolicy(ExternalTable table, List alterClauses) { + Preconditions.checkState(alterClauses.size() == 1); + AlterClause alterClause = alterClauses.get(0); + Preconditions.checkState(alterClause instanceof ModifyTablePropertiesClause); + Map properties = alterClause.getProperties(); + Preconditions.checkState(properties.size() == 1); + Preconditions.checkState(properties.containsKey(PropertyAnalyzer.PROPERTIES_AUTO_ANALYZE_POLICY)); + String value = properties.get(PropertyAnalyzer.PROPERTIES_AUTO_ANALYZE_POLICY); + Preconditions.checkState(PropertyAnalyzer.ENABLE_AUTO_ANALYZE_POLICY.equalsIgnoreCase(value) + || PropertyAnalyzer.DISABLE_AUTO_ANALYZE_POLICY.equalsIgnoreCase(value) + || PropertyAnalyzer.USE_CATALOG_AUTO_ANALYZE_POLICY.equalsIgnoreCase(value)); + value = value.equalsIgnoreCase(PropertyAnalyzer.USE_CATALOG_AUTO_ANALYZE_POLICY) ? null : value; + table.getCatalog().setAutoAnalyzePolicy(table.getDatabase().getFullName(), table.getName(), value); + ModifyTablePropertyOperationLog info = new ModifyTablePropertyOperationLog(table.getCatalog().getName(), + table.getDatabase().getFullName(), table.getName(), properties); + Env.getCurrentEnv().getEditLog().logModifyTableProperties(info); + } + private boolean needChangeMTMVState(List alterClauses) { for (AlterClause alterClause : alterClauses) { if (alterClause.needChangeMTMVState()) { @@ -467,31 +489,45 @@ private void processModifyEngineInternal(Database db, Table externalTable, public void processAlterTable(AlterTableStmt stmt) throws UserException { TableName dbTableName = stmt.getTbl(); + String ctlName = dbTableName.getCtl(); String dbName = dbTableName.getDb(); String tableName = dbTableName.getTbl(); - Database db = Env.getCurrentInternalCatalog().getDbOrDdlException(dbName); - Table table = db.getTableOrDdlException(tableName); + DatabaseIf dbIf = Env.getCurrentEnv().getCatalogMgr() + .getCatalogOrException(ctlName, catalog -> new DdlException("Unknown catalog " + catalog)) + .getDbOrDdlException(dbName); + TableIf tableIf = dbIf.getTableOrDdlException(tableName); List alterClauses = Lists.newArrayList(); // some operations will take long time to process, need to be done outside the table lock boolean needProcessOutsideTableLock = false; - switch (table.getType()) { + switch (tableIf.getType()) { case MATERIALIZED_VIEW: case OLAP: - OlapTable olapTable = (OlapTable) table; - needProcessOutsideTableLock = processAlterOlapTable(stmt, olapTable, alterClauses, db); + OlapTable olapTable = (OlapTable) tableIf; + needProcessOutsideTableLock = processAlterOlapTable(stmt, olapTable, alterClauses, (Database) dbIf); break; case ODBC: case JDBC: case HIVE: case MYSQL: case ELASTICSEARCH: - processAlterExternalTable(stmt, table, db); + processAlterExternalTable(stmt, (Table) tableIf, (Database) dbIf); + return; + case HMS_EXTERNAL_TABLE: + case JDBC_EXTERNAL_TABLE: + case ICEBERG_EXTERNAL_TABLE: + case PAIMON_EXTERNAL_TABLE: + case MAX_COMPUTE_EXTERNAL_TABLE: + case HUDI_EXTERNAL_TABLE: + case TRINO_CONNECTOR_EXTERNAL_TABLE: + alterClauses.addAll(stmt.getOps()); + setExternalTableAutoAnalyzePolicy((ExternalTable) tableIf, alterClauses); return; default: throw new DdlException("Do not support alter " - + table.getType().toString() + " table[" + tableName + "]"); + + tableIf.getType().toString() + " table[" + tableName + "]"); } + Database db = (Database) dbIf; // the following ops should done outside table lock. because it contain synchronized create operation if (needProcessOutsideTableLock) { Preconditions.checkState(alterClauses.size() == 1); @@ -514,7 +550,7 @@ public void processAlterTable(AlterTableStmt stmt) throws UserException { List partitionNames = clause.getPartitionNames(); ((SchemaChangeHandler) schemaChangeHandler).updatePartitionsProperties( db, tableName, partitionNames, properties); - OlapTable olapTable = (OlapTable) table; + OlapTable olapTable = (OlapTable) tableIf; olapTable.writeLockOrDdlException(); try { modifyPartitionsProperty(db, olapTable, partitionNames, properties, clause.isTempPartition()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java index 87d76d8de39e6d7..3051d37527f1075 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java @@ -2288,7 +2288,8 @@ public void updateTableProperties(Database db, String tableName, Map partitions = Lists.newArrayList(); @@ -2364,7 +2365,8 @@ public void updateTableProperties(Database db, String tableName, Map TITLE_NAMES = new ImmutableList.Builder() .add("Item").add("Info").add("Suggestion") .build(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DropUserStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DropUserStmt.java index e9b3e6fe794a2c0..f9bb218744fc85a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DropUserStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DropUserStmt.java @@ -30,7 +30,7 @@ // drop user cmy@['domain']; // drop user cmy <==> drop user cmy@'%' // drop user cmy@'192.168.1.%' -public class DropUserStmt extends DdlStmt { +public class DropUserStmt extends DdlStmt implements NotFallbackInParser { private boolean ifExists; private UserIdentity userIdent; diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ExportStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ExportStmt.java index 232a61f1edc0195..a9ce85b2d3e078f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ExportStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ExportStmt.java @@ -60,7 +60,7 @@ // [PROPERTIES("key"="value")] // WITH BROKER 'broker_name' [( $broker_attrs)] @Getter -public class ExportStmt extends StatementBase { +public class ExportStmt extends StatementBase implements NotFallbackInParser { public static final String PARALLELISM = "parallelism"; public static final String LABEL = "label"; public static final String DATA_CONSISTENCY = "data_consistency"; diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java index 4041fa4873b5be2..a1d89e12ec43b08 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java @@ -645,11 +645,7 @@ private String paramsToSql() { && (fnName.getFunction().equalsIgnoreCase("aes_decrypt") || fnName.getFunction().equalsIgnoreCase("aes_encrypt") || fnName.getFunction().equalsIgnoreCase("sm4_decrypt") - || fnName.getFunction().equalsIgnoreCase("sm4_encrypt") - || fnName.getFunction().equalsIgnoreCase("aes_decrypt_v2") - || fnName.getFunction().equalsIgnoreCase("aes_encrypt_v2") - || fnName.getFunction().equalsIgnoreCase("sm4_decrypt_v2") - || fnName.getFunction().equalsIgnoreCase("sm4_encrypt_v2"))) { + || fnName.getFunction().equalsIgnoreCase("sm4_encrypt"))) { sb.append("\'***\'"); } else if (orderByElements.size() > 0 && i == len - orderByElements.size()) { sb.append("ORDER BY "); @@ -734,22 +730,13 @@ private String paramsToDigest() { if (fnName.getFunction().equalsIgnoreCase("aes_decrypt") || fnName.getFunction().equalsIgnoreCase("aes_encrypt") || fnName.getFunction().equalsIgnoreCase("sm4_decrypt") - || fnName.getFunction().equalsIgnoreCase("sm4_encrypt") - || fnName.getFunction().equalsIgnoreCase("aes_decrypt_v2") - || fnName.getFunction().equalsIgnoreCase("aes_encrypt_v2") - || fnName.getFunction().equalsIgnoreCase("sm4_decrypt_v2") - || fnName.getFunction().equalsIgnoreCase("sm4_encrypt_v2")) { + || fnName.getFunction().equalsIgnoreCase("sm4_encrypt")) { len = len - 1; } for (int i = 0; i < len; ++i) { if (i == 1 && (fnName.getFunction().equalsIgnoreCase("aes_decrypt") || fnName.getFunction().equalsIgnoreCase("aes_encrypt") - || fnName.getFunction().equalsIgnoreCase("sm4_decrypt") - || fnName.getFunction().equalsIgnoreCase("sm4_encrypt") - || fnName.getFunction().equalsIgnoreCase("aes_decrypt_v2") - || fnName.getFunction().equalsIgnoreCase("aes_encrypt_v2") - || fnName.getFunction().equalsIgnoreCase("sm4_decrypt_v2") - || fnName.getFunction().equalsIgnoreCase("sm4_encrypt_v2"))) { + || fnName.getFunction().equalsIgnoreCase("sm4_decrypt"))) { result.add("\'***\'"); } else { result.add(children.get(i).toDigest()); @@ -1152,13 +1139,8 @@ private void analyzeBuiltinAggFunction(Analyzer analyzer) throws AnalysisExcepti if ((fnName.getFunction().equalsIgnoreCase("aes_decrypt") || fnName.getFunction().equalsIgnoreCase("aes_encrypt") || fnName.getFunction().equalsIgnoreCase("sm4_decrypt") - || fnName.getFunction().equalsIgnoreCase("sm4_encrypt") - || fnName.getFunction().equalsIgnoreCase("aes_decrypt_v2") - || fnName.getFunction().equalsIgnoreCase("aes_encrypt_v2") - || fnName.getFunction().equalsIgnoreCase("sm4_decrypt_v2") - || fnName.getFunction().equalsIgnoreCase("sm4_encrypt_v2")) + || fnName.getFunction().equalsIgnoreCase("sm4_encrypt")) && (children.size() == 2 || children.size() == 3)) { - String blockEncryptionMode = ""; Set aesModes = new HashSet<>(Arrays.asList( "AES_128_ECB", "AES_192_ECB", @@ -1192,43 +1174,20 @@ private void analyzeBuiltinAggFunction(Analyzer analyzer) throws AnalysisExcepti "SM4_128_OFB", "SM4_128_CTR")); + String blockEncryptionMode = ""; if (ConnectContext.get() != null) { blockEncryptionMode = ConnectContext.get().getSessionVariable().getBlockEncryptionMode(); if (fnName.getFunction().equalsIgnoreCase("aes_decrypt") - || fnName.getFunction().equalsIgnoreCase("aes_encrypt") - || fnName.getFunction().equalsIgnoreCase("aes_decrypt_v2") - || fnName.getFunction().equalsIgnoreCase("aes_encrypt_v2")) { + || fnName.getFunction().equalsIgnoreCase("aes_encrypt")) { if (StringUtils.isAllBlank(blockEncryptionMode)) { blockEncryptionMode = "AES_128_ECB"; } if (!aesModes.contains(blockEncryptionMode.toUpperCase())) { throw new AnalysisException("session variable block_encryption_mode is invalid with aes"); } - if (children.size() == 2) { - boolean isECB = blockEncryptionMode.equalsIgnoreCase("AES_128_ECB") - || blockEncryptionMode.equalsIgnoreCase("AES_192_ECB") - || blockEncryptionMode.equalsIgnoreCase("AES_256_ECB"); - if (fnName.getFunction().equalsIgnoreCase("aes_decrypt_v2")) { - if (!isECB) { - throw new AnalysisException( - "Incorrect parameter count in the call to native function 'aes_decrypt'"); - } - } else if (fnName.getFunction().equalsIgnoreCase("aes_encrypt_v2")) { - if (!isECB) { - throw new AnalysisException( - "Incorrect parameter count in the call to native function 'aes_encrypt'"); - } - } else { - // if there are only 2 params, we need set encryption mode to AES_128_ECB - // this keeps the behavior consistent with old doris ver. - blockEncryptionMode = "AES_128_ECB"; - } - } } if (fnName.getFunction().equalsIgnoreCase("sm4_decrypt") - || fnName.getFunction().equalsIgnoreCase("sm4_encrypt") - || fnName.getFunction().equalsIgnoreCase("sm4_decrypt_v2") - || fnName.getFunction().equalsIgnoreCase("sm4_encrypt_v2")) { + || fnName.getFunction().equalsIgnoreCase("sm4_encrypt")) { if (StringUtils.isAllBlank(blockEncryptionMode)) { blockEncryptionMode = "SM4_128_ECB"; } @@ -1236,36 +1195,12 @@ private void analyzeBuiltinAggFunction(Analyzer analyzer) throws AnalysisExcepti throw new AnalysisException( "session variable block_encryption_mode is invalid with sm4"); } - if (children.size() == 2) { - if (fnName.getFunction().equalsIgnoreCase("sm4_decrypt_v2")) { - throw new AnalysisException( - "Incorrect parameter count in the call to native function 'sm4_decrypt'"); - } else if (fnName.getFunction().equalsIgnoreCase("sm4_encrypt_v2")) { - throw new AnalysisException( - "Incorrect parameter count in the call to native function 'sm4_encrypt'"); - } else { - // if there are only 2 params, we need add an empty string as the third param - // and set encryption mode to SM4_128_ECB - // this keeps the behavior consistent with old doris ver. - children.add(new StringLiteral("")); - blockEncryptionMode = "SM4_128_ECB"; - } - } } + } else { + throw new AnalysisException("cannot get session variable `block_encryption_mode`, " + + "please explicitly specify by using 4-args function"); } - if (!blockEncryptionMode.equals(children.get(children.size() - 1).toString())) { - children.add(new StringLiteral(blockEncryptionMode)); - } - - if (fnName.getFunction().equalsIgnoreCase("aes_decrypt_v2")) { - fnName = FunctionName.createBuiltinName("aes_decrypt"); - } else if (fnName.getFunction().equalsIgnoreCase("aes_encrypt_v2")) { - fnName = FunctionName.createBuiltinName("aes_encrypt"); - } else if (fnName.getFunction().equalsIgnoreCase("sm4_decrypt_v2")) { - fnName = FunctionName.createBuiltinName("sm4_decrypt"); - } else if (fnName.getFunction().equalsIgnoreCase("sm4_encrypt_v2")) { - fnName = FunctionName.createBuiltinName("sm4_encrypt"); - } + children.add(new StringLiteral(blockEncryptionMode)); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/GroupByClause.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/GroupByClause.java index f6305e611da442e..9b5055944674b25 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/GroupByClause.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/GroupByClause.java @@ -192,6 +192,11 @@ public void analyze(Analyzer analyzer) throws AnalysisException { "GROUP BY expression must not contain aggregate functions: " + groupingExpr.toSql()); } + if (groupingExpr.contains(GroupingFunctionCallExpr.class)) { + throw new AnalysisException( + "GROUP BY expression must not contain grouping scalar functions: " + + groupingExpr.toSql()); + } if (groupingExpr.contains(AnalyticExpr.class)) { // reference the original expr in the error msg throw new AnalysisException( diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/InsertOverwriteTableStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/InsertOverwriteTableStmt.java index bef4a38bc3eb190..a0fca62e2848be5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/InsertOverwriteTableStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/InsertOverwriteTableStmt.java @@ -34,7 +34,7 @@ import java.util.ArrayList; import java.util.List; -public class InsertOverwriteTableStmt extends DdlStmt { +public class InsertOverwriteTableStmt extends DdlStmt implements NotFallbackInParser { private final InsertTarget target; diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ModifyTablePropertiesClause.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ModifyTablePropertiesClause.java index c7ea5c2eccad608..2d3bc4ae05f2ccf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ModifyTablePropertiesClause.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ModifyTablePropertiesClause.java @@ -29,6 +29,7 @@ import org.apache.doris.common.util.DynamicPartitionUtil; import org.apache.doris.common.util.PrintableMap; import org.apache.doris.common.util.PropertyAnalyzer; +import org.apache.doris.datasource.InternalCatalog; import com.google.common.base.Strings; @@ -350,6 +351,19 @@ public void analyze(Analyzer analyzer) throws AnalysisException { // do nothing, will be analyzed when creating alter job } else if (properties.containsKey(PropertyAnalyzer.PROPERTIES_ROW_STORE_COLUMNS)) { // do nothing, will be analyzed when creating alter job + } else if (properties.containsKey(PropertyAnalyzer.PROPERTIES_AUTO_ANALYZE_POLICY)) { + String analyzePolicy = properties.getOrDefault(PropertyAnalyzer.PROPERTIES_AUTO_ANALYZE_POLICY, ""); + if (analyzePolicy != null + && !analyzePolicy.equals(PropertyAnalyzer.ENABLE_AUTO_ANALYZE_POLICY) + && !analyzePolicy.equals(PropertyAnalyzer.DISABLE_AUTO_ANALYZE_POLICY) + && !analyzePolicy.equals(PropertyAnalyzer.USE_CATALOG_AUTO_ANALYZE_POLICY)) { + throw new AnalysisException( + "Table auto analyze policy only support for " + PropertyAnalyzer.ENABLE_AUTO_ANALYZE_POLICY + + " or " + PropertyAnalyzer.DISABLE_AUTO_ANALYZE_POLICY + + " or " + PropertyAnalyzer.USE_CATALOG_AUTO_ANALYZE_POLICY); + } + this.needTableStable = false; + this.opType = AlterOpType.MODIFY_TABLE_PROPERTY_SYNC; } else { throw new AnalysisException("Unknown table property: " + properties.keySet()); } @@ -358,6 +372,10 @@ public void analyze(Analyzer analyzer) throws AnalysisException { private void analyzeForMTMV() throws AnalysisException { if (tableName != null) { + // Skip external catalog. + if (!(InternalCatalog.INTERNAL_CATALOG_NAME.equals(tableName.getCtl()))) { + return; + } Table table = Env.getCurrentInternalCatalog().getDbOrAnalysisException(tableName.getDb()) .getTableOrAnalysisException(tableName.getTbl()); if (!(table instanceof MTMV)) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/OutFileClause.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/OutFileClause.java index 6debdca789f125b..ef65b4058537653 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/OutFileClause.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/OutFileClause.java @@ -884,6 +884,7 @@ public TResultFileSinkOptions toSinkOptions() { if (isOrcFormat()) { sinkOptions.setOrcSchema(serializeOrcSchema()); sinkOptions.setOrcCompressionType(orcCompressionType); + sinkOptions.setOrcWriterVersion(1); } return sinkOptions; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SchemaTableType.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/SchemaTableType.java index 755ca3278ed8ba4..7fcb61e3d2d4da7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SchemaTableType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SchemaTableType.java @@ -84,7 +84,9 @@ public enum SchemaTableType { SCH_WORKLOAD_GROUP_RESOURCE_USAGE("WORKLOAD_GROUP_RESOURCE_USAGE", "WORKLOAD_GROUP_RESOURCE_USAGE", TSchemaTableType.SCH_WORKLOAD_GROUP_RESOURCE_USAGE), SCH_TABLE_PROPERTIES("TABLE_PROPERTIES", "TABLE_PROPERTIES", - TSchemaTableType.SCH_TABLE_PROPERTIES); + TSchemaTableType.SCH_TABLE_PROPERTIES), + SCH_CATALOG_META_CACHE_STATISTICS("CATALOG_META_CACHE_STATISTICS", "CATALOG_META_CACHE_STATISTICS", + TSchemaTableType.SCH_CATALOG_META_CACHE_STATISTICS); private static final String dbName = "INFORMATION_SCHEMA"; private static SelectList fullSelectLists; diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java index b7b778ae9012eab..d9f8e00c292bf07 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java @@ -80,7 +80,7 @@ * clauses. */ @Deprecated -public class SelectStmt extends QueryStmt { +public class SelectStmt extends QueryStmt implements NotFallbackInParser { private static final Logger LOG = LogManager.getLogger(SelectStmt.class); public static final String DEFAULT_VALUE = "__DEFAULT_VALUE__"; private UUID id = UUID.randomUUID(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SetDefaultStorageVaultStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/SetDefaultStorageVaultStmt.java index 26687942d0298ba..3c3fc6f1863645f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SetDefaultStorageVaultStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SetDefaultStorageVaultStmt.java @@ -29,7 +29,7 @@ import org.apache.doris.qe.ConnectContext; // SET vault_name DEFAULT STORAGE VAULT -public class SetDefaultStorageVaultStmt extends DdlStmt { +public class SetDefaultStorageVaultStmt extends DdlStmt implements NotFallbackInParser { public final String vaultName; public SetDefaultStorageVaultStmt(String vaultName) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SetOperationStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/SetOperationStmt.java index 3c4f8a76fd549df..2746da4d8fd5952 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SetOperationStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SetOperationStmt.java @@ -53,7 +53,7 @@ * and we need to mark the slots of resolved exprs as materialized. */ @Deprecated -public class SetOperationStmt extends QueryStmt { +public class SetOperationStmt extends QueryStmt implements NotFallbackInParser { private static final Logger LOG = LogManager.getLogger(SetOperationStmt.class); public enum Operation { diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SetUserPropertyStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/SetUserPropertyStmt.java index 7342fe7a79c2a21..04247fb227f4308 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SetUserPropertyStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SetUserPropertyStmt.java @@ -27,7 +27,7 @@ import java.util.List; -public class SetUserPropertyStmt extends DdlStmt { +public class SetUserPropertyStmt extends DdlStmt implements NotFallbackInParser { private String user; private final List propertyList; diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowAlterStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowAlterStmt.java index 466d958ab19e354..8040271005ad83e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowAlterStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowAlterStmt.java @@ -48,7 +48,7 @@ * SHOW ALTER TABLE [COLUMN | ROLLUP] [FROM dbName] [WHERE TableName="xxx"] * [ORDER BY CreateTime DESC] [LIMIT [offset,]rows] */ -public class ShowAlterStmt extends ShowStmt { +public class ShowAlterStmt extends ShowStmt implements NotFallbackInParser { private static final Logger LOG = LogManager.getLogger(ShowAlterStmt.class); public static enum AlterType { diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowAnalyzeStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowAnalyzeStmt.java index f015cdb05552159..56e16bba76a0d86 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowAnalyzeStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowAnalyzeStmt.java @@ -45,7 +45,7 @@ * [ORDER BY ...] * [LIMIT limit]; */ -public class ShowAnalyzeStmt extends ShowStmt { +public class ShowAnalyzeStmt extends ShowStmt implements NotFallbackInParser { private static final String STATE_NAME = "state"; private static final ImmutableList TITLE_NAMES = new ImmutableList.Builder() .add("job_id") diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowAnalyzeTaskStatus.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowAnalyzeTaskStatus.java index 364e3d0a4f39186..ff52cf0ccbaa406 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowAnalyzeTaskStatus.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowAnalyzeTaskStatus.java @@ -26,7 +26,7 @@ /** * SHOW ANALYZE TASK STATUS [JOB_ID] */ -public class ShowAnalyzeTaskStatus extends ShowStmt { +public class ShowAnalyzeTaskStatus extends ShowStmt implements NotFallbackInParser { private static final ShowResultSetMetaData ROW_META_DATA = ShowResultSetMetaData.builder() diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowAuthorStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowAuthorStmt.java index 477efd694d274f1..f82f6022d4166f2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowAuthorStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowAuthorStmt.java @@ -22,7 +22,7 @@ import org.apache.doris.qe.ShowResultSetMetaData; // Used to test show executor. -public class ShowAuthorStmt extends ShowStmt { +public class ShowAuthorStmt extends ShowStmt implements NotFallbackInParser { private static final ShowResultSetMetaData META_DATA = ShowResultSetMetaData.builder() .addColumn(new Column("Name", ScalarType.createVarchar(30))) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowAutoAnalyzeJobsStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowAutoAnalyzeJobsStmt.java index 560387fa5bc11cb..9b07796df784d41 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowAutoAnalyzeJobsStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowAutoAnalyzeJobsStmt.java @@ -43,7 +43,7 @@ * [PRIORITY = ["HIGH"|"MID"|"LOW"]] * ] */ -public class ShowAutoAnalyzeJobsStmt extends ShowStmt { +public class ShowAutoAnalyzeJobsStmt extends ShowStmt implements NotFallbackInParser { private static final String PRIORITY = "priority"; private static final ImmutableList TITLE_NAMES = new ImmutableList.Builder() .add("catalog_name") diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowBackendsStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowBackendsStmt.java index 46009bd1c097985..8e1e589477d2dc0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowBackendsStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowBackendsStmt.java @@ -28,7 +28,7 @@ import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.ShowResultSetMetaData; -public class ShowBackendsStmt extends ShowStmt { +public class ShowBackendsStmt extends ShowStmt implements NotFallbackInParser { public ShowBackendsStmt() { } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowBackupStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowBackupStmt.java index 3c06a1da29eecd8..a76857a8f8bc583 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowBackupStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowBackupStmt.java @@ -37,7 +37,7 @@ import java.util.function.Predicate; -public class ShowBackupStmt extends ShowStmt { +public class ShowBackupStmt extends ShowStmt implements NotFallbackInParser { public static final ImmutableList TITLE_NAMES = new ImmutableList.Builder() .add("JobId").add("SnapshotName").add("DbName").add("State").add("BackupObjs").add("CreateTime") .add("SnapshotFinishedTime").add("UploadFinishedTime").add("FinishedTime").add("UnfinishedTasks") diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowBrokerStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowBrokerStmt.java index 2ef68cd149e7d2b..9f2b4d4ad5f2b25 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowBrokerStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowBrokerStmt.java @@ -28,7 +28,7 @@ import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.ShowResultSetMetaData; -public class ShowBrokerStmt extends ShowStmt { +public class ShowBrokerStmt extends ShowStmt implements NotFallbackInParser { public ShowBrokerStmt() { } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowBuildIndexStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowBuildIndexStmt.java index 5b02e306aa9deb6..944365fb0d48dfe 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowBuildIndexStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowBuildIndexStmt.java @@ -45,7 +45,7 @@ // syntax: // SHOW BUILD INDEX [FROM db] [WHERE ] // [ORDER BY [DESC|ASC] [NULLS LAST | NULLS FIRST]]] [ LIMIT 1, 100] -public class ShowBuildIndexStmt extends ShowStmt { +public class ShowBuildIndexStmt extends ShowStmt implements NotFallbackInParser { private static final Logger LOG = LogManager.getLogger(ShowBuildIndexStmt.class); private String dbName; diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowCacheHotSpotStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowCacheHotSpotStmt.java index 23b72938e966749..847b015825dc1c1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowCacheHotSpotStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowCacheHotSpotStmt.java @@ -40,7 +40,7 @@ import java.util.Arrays; import java.util.List; -public class ShowCacheHotSpotStmt extends ShowStmt { +public class ShowCacheHotSpotStmt extends ShowStmt implements NotFallbackInParser { public static final ShowResultSetMetaData[] RESULT_SET_META_DATAS = { ShowResultSetMetaData.builder() .addColumn(new Column("cluster_id", ScalarType.createType(PrimitiveType.BIGINT))) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowCatalogRecycleBinStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowCatalogRecycleBinStmt.java index f15c3657240d878..3a4a33e4d628fd9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowCatalogRecycleBinStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowCatalogRecycleBinStmt.java @@ -36,7 +36,7 @@ import java.util.function.Predicate; -public class ShowCatalogRecycleBinStmt extends ShowStmt { +public class ShowCatalogRecycleBinStmt extends ShowStmt implements NotFallbackInParser { public static final ImmutableList TITLE_NAMES = new ImmutableList.Builder() .add("Type").add("Name").add("DbId").add("TableId").add("PartitionId").add("DropTime") .add("DataSize").add("RemoteDataSize").build(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowCatalogStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowCatalogStmt.java index 191c5ba9a40ed56..f1770859002c7dc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowCatalogStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowCatalogStmt.java @@ -25,7 +25,7 @@ /** * Statement for show all catalog or desc the specific catalog. */ -public class ShowCatalogStmt extends ShowStmt { +public class ShowCatalogStmt extends ShowStmt implements NotFallbackInParser { private static final ShowResultSetMetaData META_DATA_ALL = ShowResultSetMetaData.builder().addColumn(new Column("CatalogId", ScalarType.BIGINT)) .addColumn(new Column("CatalogName", ScalarType.createVarchar(64))) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowCharsetStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowCharsetStmt.java index 07e78fdf9cec8b8..50725f690445a11 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowCharsetStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowCharsetStmt.java @@ -21,7 +21,7 @@ import org.apache.doris.catalog.ScalarType; import org.apache.doris.qe.ShowResultSetMetaData; -public class ShowCharsetStmt extends ShowStmt { +public class ShowCharsetStmt extends ShowStmt implements NotFallbackInParser { private static final ShowResultSetMetaData META_DATA = ShowResultSetMetaData.builder() .addColumn(new Column("Charset", ScalarType.createVarchar(20))) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowCloudWarmUpStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowCloudWarmUpStmt.java index d34c46d54a5d316..e91e9b7d6fe251d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowCloudWarmUpStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowCloudWarmUpStmt.java @@ -28,7 +28,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -public class ShowCloudWarmUpStmt extends ShowStmt { +public class ShowCloudWarmUpStmt extends ShowStmt implements NotFallbackInParser { private static final Logger LOG = LogManager.getLogger(ShowCloudWarmUpStmt.class); private Expr whereClause; private boolean showAllJobs = false; diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowClusterStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowClusterStmt.java index 50bd36b18827677..acb6d789f45e591 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowClusterStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowClusterStmt.java @@ -33,7 +33,7 @@ import com.google.common.collect.ImmutableList; -public class ShowClusterStmt extends ShowStmt { +public class ShowClusterStmt extends ShowStmt implements NotFallbackInParser { public static final ImmutableList TITLE_NAMES = new ImmutableList.Builder() .add("cluster").add("is_current").add("users").build(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowCollationStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowCollationStmt.java index df7e1f6a1637f35..6fda6ab9220c644 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowCollationStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowCollationStmt.java @@ -22,7 +22,7 @@ import org.apache.doris.catalog.ScalarType; import org.apache.doris.qe.ShowResultSetMetaData; -public class ShowCollationStmt extends ShowStmt { +public class ShowCollationStmt extends ShowStmt implements NotFallbackInParser { private static final ShowResultSetMetaData META_DATA = ShowResultSetMetaData.builder() .addColumn(new Column("Collation", ScalarType.createVarchar(20))) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowColumnHistStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowColumnHistStmt.java index 88eb244589621bf..ae5010ffe0d72fb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowColumnHistStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowColumnHistStmt.java @@ -45,7 +45,7 @@ import java.util.Set; import java.util.stream.Collectors; -public class ShowColumnHistStmt extends ShowStmt { +public class ShowColumnHistStmt extends ShowStmt implements NotFallbackInParser { private static final ImmutableList TITLE_NAMES = new ImmutableList.Builder() diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowColumnStatsStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowColumnStatsStmt.java index 833ca5e27a9784c..354b57bc55c7d95 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowColumnStatsStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowColumnStatsStmt.java @@ -50,7 +50,7 @@ import java.util.Set; import java.util.stream.Collectors; -public class ShowColumnStatsStmt extends ShowStmt { +public class ShowColumnStatsStmt extends ShowStmt implements NotFallbackInParser { private static final ImmutableList TABLE_COLUMN_TITLE_NAMES = new ImmutableList.Builder() diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowColumnStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowColumnStmt.java index 9af269104cc9933..cf9c4a27660ec71 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowColumnStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowColumnStmt.java @@ -32,7 +32,7 @@ import com.google.common.collect.Lists; // SHOW COLUMNS -public class ShowColumnStmt extends ShowStmt { +public class ShowColumnStmt extends ShowStmt implements NotFallbackInParser { private static final ShowResultSetMetaData META_DATA = ShowResultSetMetaData.builder() .addColumn(new Column("Field", ScalarType.createVarchar(20))) .addColumn(new Column("Type", ScalarType.createVarchar(20))) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowConvertLSCStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowConvertLSCStmt.java index bab2e3b34b88477..cedf3e2d2466e0a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowConvertLSCStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowConvertLSCStmt.java @@ -27,7 +27,7 @@ import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.ShowResultSetMetaData; -public class ShowConvertLSCStmt extends ShowStmt { +public class ShowConvertLSCStmt extends ShowStmt implements NotFallbackInParser { private final String dbName; diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowCopyStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowCopyStmt.java index 0d4ba6a909e42cc..50988fd6a47493c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowCopyStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowCopyStmt.java @@ -38,7 +38,7 @@ // // syntax: // SHOW COPY [FROM db] [LIKE mask] -public class ShowCopyStmt extends ShowLoadStmt { +public class ShowCopyStmt extends ShowLoadStmt implements NotFallbackInParser { private static final Logger LOG = LogManager.getLogger(ShowCopyStmt.class); public ShowCopyStmt(String db, Expr labelExpr, List orderByElements, LimitElement limitElement) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowCreateCatalogStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowCreateCatalogStmt.java index 45208394eaf17bd..f324b0701a80299 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowCreateCatalogStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowCreateCatalogStmt.java @@ -32,7 +32,7 @@ // Show create catalog statement // Syntax: // SHOW CREATE CATALOG catalogName -public class ShowCreateCatalogStmt extends ShowStmt { +public class ShowCreateCatalogStmt extends ShowStmt implements NotFallbackInParser { private static final ShowResultSetMetaData META_DATA = ShowResultSetMetaData.builder() .addColumn(new Column("Catalog", ScalarType.createVarchar(20))) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowCreateDbStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowCreateDbStmt.java index b709be8d7e95c2d..99cd1b3343823f7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowCreateDbStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowCreateDbStmt.java @@ -33,7 +33,7 @@ // Show create database statement // Syntax: // SHOW CREATE DATABASE db -public class ShowCreateDbStmt extends ShowStmt { +public class ShowCreateDbStmt extends ShowStmt implements NotFallbackInParser { private static final ShowResultSetMetaData META_DATA = ShowResultSetMetaData.builder() .addColumn(new Column("Database", ScalarType.createVarchar(20))) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowCreateFunctionStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowCreateFunctionStmt.java index e1ff16ba8ddc8f7..b65f9318e58d4e9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowCreateFunctionStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowCreateFunctionStmt.java @@ -33,7 +33,7 @@ import com.google.common.base.Strings; -public class ShowCreateFunctionStmt extends ShowStmt { +public class ShowCreateFunctionStmt extends ShowStmt implements NotFallbackInParser { private static final ShowResultSetMetaData META_DATA = ShowResultSetMetaData.builder() .addColumn(new Column("Function Signature", ScalarType.createVarchar(256))) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowCreateMTMVStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowCreateMTMVStmt.java index fb8e69e779a70e9..215217f63bd2929 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowCreateMTMVStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowCreateMTMVStmt.java @@ -30,7 +30,7 @@ import org.apache.doris.qe.ShowResultSetMetaData; // SHOW CREATE Materialized View statement. -public class ShowCreateMTMVStmt extends ShowStmt { +public class ShowCreateMTMVStmt extends ShowStmt implements NotFallbackInParser { private static final ShowResultSetMetaData META_DATA = ShowResultSetMetaData.builder() .addColumn(new Column("Materialized View", ScalarType.createVarchar(20))) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowCreateMaterializedViewStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowCreateMaterializedViewStmt.java index f0c48520a5c64c8..ad5469809cf41bd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowCreateMaterializedViewStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowCreateMaterializedViewStmt.java @@ -36,7 +36,7 @@ **/ @AllArgsConstructor @Getter -public class ShowCreateMaterializedViewStmt extends ShowStmt { +public class ShowCreateMaterializedViewStmt extends ShowStmt implements NotFallbackInParser { private static final ShowResultSetMetaData META_DATA = ShowResultSetMetaData.builder() diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowCreateRepositoryStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowCreateRepositoryStmt.java index 9de7dd0e9eed727..efd436ea2406392 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowCreateRepositoryStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowCreateRepositoryStmt.java @@ -28,7 +28,7 @@ import org.apache.doris.qe.ShowResultSetMetaData; // SHOW CREATE REPOSITORY statement -public class ShowCreateRepositoryStmt extends ShowStmt { +public class ShowCreateRepositoryStmt extends ShowStmt implements NotFallbackInParser { private static final ShowResultSetMetaData META_DATA = ShowResultSetMetaData.builder() diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowCreateTableStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowCreateTableStmt.java index 1a8ec38080ff84e..9b60417c721b0fa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowCreateTableStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowCreateTableStmt.java @@ -31,7 +31,7 @@ import org.apache.doris.qe.ShowResultSetMetaData; // SHOW CREATE TABLE statement. -public class ShowCreateTableStmt extends ShowStmt { +public class ShowCreateTableStmt extends ShowStmt implements NotFallbackInParser { private static final ShowResultSetMetaData META_DATA = ShowResultSetMetaData.builder() .addColumn(new Column("Table", ScalarType.createVarchar(20))) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowDataSkewStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowDataSkewStmt.java index c1ccc521c3bf949..6db5aeaa9da264c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowDataSkewStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowDataSkewStmt.java @@ -31,7 +31,7 @@ import com.google.common.collect.ImmutableList; // show data skew from tbl [partition(p1, p2, ...)] -public class ShowDataSkewStmt extends ShowStmt { +public class ShowDataSkewStmt extends ShowStmt implements NotFallbackInParser { public static final ImmutableList TITLE_NAMES = new ImmutableList.Builder() .add("PartitionName").add("BucketIdx").add("AvgRowCount").add("AvgDataSize") .add("Graph").add("Percent") diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowDataStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowDataStmt.java index 84ce67283ac487b..26b77e0b3d5d260 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowDataStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowDataStmt.java @@ -56,7 +56,7 @@ import java.util.TreeMap; import java.util.TreeSet; -public class ShowDataStmt extends ShowStmt { +public class ShowDataStmt extends ShowStmt implements NotFallbackInParser { private static final ShowResultSetMetaData SHOW_DATABASE_DATA_META_DATA = ShowResultSetMetaData.builder() .addColumn(new Column("DbId", ScalarType.createVarchar(20))) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowDataTypesStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowDataTypesStmt.java index 0ee95dd817e9f7b..31bc8f3f598b966 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowDataTypesStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowDataTypesStmt.java @@ -31,7 +31,7 @@ import java.util.Comparator; import java.util.List; -public class ShowDataTypesStmt extends ShowStmt { +public class ShowDataTypesStmt extends ShowStmt implements NotFallbackInParser { private static final ShowResultSetMetaData META_DATA = ShowResultSetMetaData.builder() diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowDbIdStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowDbIdStmt.java index 967714e0fa25461..6e800274817e8a8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowDbIdStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowDbIdStmt.java @@ -28,7 +28,7 @@ import org.apache.doris.qe.ShowResultSetMetaData; // SHOW DATABASE ID -public class ShowDbIdStmt extends ShowStmt { +public class ShowDbIdStmt extends ShowStmt implements NotFallbackInParser { private long dbId; public ShowDbIdStmt(long dbId) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowDbStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowDbStmt.java index c9606bc5bd224af..aba12fbd0c6c6d7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowDbStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowDbStmt.java @@ -29,7 +29,7 @@ import com.google.common.collect.Lists; // Show database statement. -public class ShowDbStmt extends ShowStmt { +public class ShowDbStmt extends ShowStmt implements NotFallbackInParser { private static final TableName TABLE_NAME = new TableName(InternalCatalog.INTERNAL_CATALOG_NAME, InfoSchemaDb.DATABASE_NAME, "schemata"); private static final String DB_COL = "Database"; diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowDeleteStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowDeleteStmt.java index 85df05718998b81..dcf36d116094c8f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowDeleteStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowDeleteStmt.java @@ -28,7 +28,7 @@ import com.google.common.base.Strings; -public class ShowDeleteStmt extends ShowStmt { +public class ShowDeleteStmt extends ShowStmt implements NotFallbackInParser { private String dbName; diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowDynamicPartitionStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowDynamicPartitionStmt.java index d8b0c1779214d71..ef1d731d77b0788 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowDynamicPartitionStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowDynamicPartitionStmt.java @@ -26,7 +26,7 @@ import com.google.common.base.Strings; -public class ShowDynamicPartitionStmt extends ShowStmt { +public class ShowDynamicPartitionStmt extends ShowStmt implements NotFallbackInParser { private String db; private static final ShowResultSetMetaData SHOW_DYNAMIC_PARTITION_META_DATA = ShowResultSetMetaData.builder() diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowEncryptKeysStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowEncryptKeysStmt.java index 83358209c235397..44ebc7b2b70deaf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowEncryptKeysStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowEncryptKeysStmt.java @@ -30,7 +30,7 @@ import com.google.common.base.Strings; -public class ShowEncryptKeysStmt extends ShowStmt { +public class ShowEncryptKeysStmt extends ShowStmt implements NotFallbackInParser { private static final ShowResultSetMetaData META_DATA = ShowResultSetMetaData.builder() .addColumn(new Column("EncryptKey Name", ScalarType.createVarchar(20))) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowEnginesStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowEnginesStmt.java index f6ff28a671c93e2..86ac4af67fb0018 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowEnginesStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowEnginesStmt.java @@ -21,7 +21,7 @@ import org.apache.doris.catalog.ScalarType; import org.apache.doris.qe.ShowResultSetMetaData; -public class ShowEnginesStmt extends ShowStmt { +public class ShowEnginesStmt extends ShowStmt implements NotFallbackInParser { private static final ShowResultSetMetaData META_DATA = ShowResultSetMetaData.builder() .addColumn(new Column("Engine", ScalarType.createVarchar(64))) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowEventsStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowEventsStmt.java index 4e47ac79dec5f07..97ac779cd287292 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowEventsStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowEventsStmt.java @@ -22,7 +22,7 @@ import org.apache.doris.qe.ShowResultSetMetaData; // Show Events statement -public class ShowEventsStmt extends ShowStmt { +public class ShowEventsStmt extends ShowStmt implements NotFallbackInParser { private static final ShowResultSetMetaData META_DATA = ShowResultSetMetaData.builder() .addColumn(new Column("Db", ScalarType.createVarchar(20))) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowExportStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowExportStmt.java index 14b5d05537c298e..757eaf83bacf36f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowExportStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowExportStmt.java @@ -40,7 +40,7 @@ // // syntax: // SHOW EXPORT [FROM db] [where ...] -public class ShowExportStmt extends ShowStmt { +public class ShowExportStmt extends ShowStmt implements NotFallbackInParser { private static final Logger LOG = LogManager.getLogger(ShowExportStmt.class); private String dbName; diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowFrontendsStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowFrontendsStmt.java index b3b5808d945d921..d8b5c454ff9d081 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowFrontendsStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowFrontendsStmt.java @@ -30,7 +30,7 @@ import com.google.common.collect.ImmutableList; -public class ShowFrontendsStmt extends ShowStmt { +public class ShowFrontendsStmt extends ShowStmt implements NotFallbackInParser { private String detail; public ShowFrontendsStmt() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowFunctionsStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowFunctionsStmt.java index 5abd18f74631f5c..a994d0f6463c4fa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowFunctionsStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowFunctionsStmt.java @@ -32,7 +32,7 @@ import com.google.common.base.Strings; -public class ShowFunctionsStmt extends ShowStmt { +public class ShowFunctionsStmt extends ShowStmt implements NotFallbackInParser { private static final ShowResultSetMetaData META_DATA = ShowResultSetMetaData.builder() .addColumn(new Column("Signature", ScalarType.createVarchar(256))) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowGrantsStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowGrantsStmt.java index f78d4ca59fa6824..0439544ae0e3a15 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowGrantsStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowGrantsStmt.java @@ -43,7 +43,7 @@ // // SHOW GRANTS; // SHOW GRANTS FOR user@'xxx' -public class ShowGrantsStmt extends ShowStmt { +public class ShowGrantsStmt extends ShowStmt implements NotFallbackInParser { private static final ShowResultSetMetaData META_DATA; diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowIndexStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowIndexStmt.java index b35f2e77287b5eb..43f02f4426d579c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowIndexStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowIndexStmt.java @@ -30,7 +30,7 @@ import com.google.common.base.Strings; -public class ShowIndexStmt extends ShowStmt { +public class ShowIndexStmt extends ShowStmt implements NotFallbackInParser { private static final ShowResultSetMetaData META_DATA = ShowResultSetMetaData.builder() .addColumn(new Column("Table", ScalarType.createVarchar(64))) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowLastInsertStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowLastInsertStmt.java index 9ed0ebc6cd7f779..4ba3521d51cbd3f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowLastInsertStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowLastInsertStmt.java @@ -22,7 +22,7 @@ import org.apache.doris.qe.ShowResultSetMetaData; // SHOW LAST INSERT -public class ShowLastInsertStmt extends ShowStmt { +public class ShowLastInsertStmt extends ShowStmt implements NotFallbackInParser { private static final ShowResultSetMetaData META_DATA = ShowResultSetMetaData.builder() .addColumn(new Column("TransactionId", ScalarType.createVarchar(128))) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowLoadProfileStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowLoadProfileStmt.java index 490a521132119a0..78437cec7968988 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowLoadProfileStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowLoadProfileStmt.java @@ -24,7 +24,7 @@ // deprecated stmt, use will be guided to a specific url to get profile from // web browser -public class ShowLoadProfileStmt extends ShowStmt { +public class ShowLoadProfileStmt extends ShowStmt implements NotFallbackInParser { private String loadIdPath; public ShowLoadProfileStmt(String path) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowLoadStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowLoadStmt.java index b08ef917ab3aa28..760f70481fcf9ce 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowLoadStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowLoadStmt.java @@ -42,7 +42,7 @@ // // syntax: // SHOW LOAD [FROM db] [LIKE mask] -public class ShowLoadStmt extends ShowStmt { +public class ShowLoadStmt extends ShowStmt implements NotFallbackInParser { private static final Logger LOG = LogManager.getLogger(ShowLoadStmt.class); private String dbName; diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowLoadWarningsStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowLoadWarningsStmt.java index 624fc0a402b02ee..f034c32a496626d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowLoadWarningsStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowLoadWarningsStmt.java @@ -33,7 +33,7 @@ import java.net.URL; // SHOW LOAD WARNINGS statement used to get error detail of src data. -public class ShowLoadWarningsStmt extends ShowStmt { +public class ShowLoadWarningsStmt extends ShowStmt implements NotFallbackInParser { private static final Logger LOG = LogManager.getLogger(ShowLoadWarningsStmt.class); private static final ShowResultSetMetaData META_DATA = diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowOpenTableStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowOpenTableStmt.java index 7c7dd8a9253a7f5..3c2ae9f37950457 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowOpenTableStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowOpenTableStmt.java @@ -22,7 +22,7 @@ import org.apache.doris.qe.ShowResultSetMetaData; // SHOW OPEN TABLES -public class ShowOpenTableStmt extends ShowStmt { +public class ShowOpenTableStmt extends ShowStmt implements NotFallbackInParser { private static final ShowResultSetMetaData META_DATA = ShowResultSetMetaData.builder() .addColumn(new Column("Database", ScalarType.createVarchar(64))) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowPartitionIdStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowPartitionIdStmt.java index 5aca99da3375397..793a886ff31e9ba 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowPartitionIdStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowPartitionIdStmt.java @@ -28,7 +28,7 @@ import org.apache.doris.qe.ShowResultSetMetaData; // SHOW PARTITION ID -public class ShowPartitionIdStmt extends ShowStmt { +public class ShowPartitionIdStmt extends ShowStmt implements NotFallbackInParser { private long partitionId; public ShowPartitionIdStmt(long partitionId) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowPartitionsStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowPartitionsStmt.java index 1c61018bc53ef4e..0be41ef60fa7bae 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowPartitionsStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowPartitionsStmt.java @@ -54,7 +54,7 @@ import java.util.Map; import java.util.Objects; -public class ShowPartitionsStmt extends ShowStmt { +public class ShowPartitionsStmt extends ShowStmt implements NotFallbackInParser { private static final Logger LOG = LogManager.getLogger(ShowPartitionsStmt.class); private static final String FILTER_PARTITION_ID = "PartitionId"; diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowPluginsStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowPluginsStmt.java index 249e49da28dbaf9..dc04e77d2ac7f8d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowPluginsStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowPluginsStmt.java @@ -29,7 +29,7 @@ // Show plugins statement. // TODO(zhaochun): only for support MySQL -public class ShowPluginsStmt extends ShowStmt { +public class ShowPluginsStmt extends ShowStmt implements NotFallbackInParser { private static final ShowResultSetMetaData META_DATA = ShowResultSetMetaData.builder() .addColumn(new Column("Name", ScalarType.createVarchar(64))) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowPolicyStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowPolicyStmt.java index df413ee61e465c3..4c471fbcc42c3bd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowPolicyStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowPolicyStmt.java @@ -36,7 +36,7 @@ * syntax: * SHOW ROW POLICY [FOR user｜ROLE role] **/ -public class ShowPolicyStmt extends ShowStmt { +public class ShowPolicyStmt extends ShowStmt implements NotFallbackInParser { @Getter private final PolicyTypeEnum type; diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowPrivilegesStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowPrivilegesStmt.java index 31f2d0edeab7f31..a053c9e327f130b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowPrivilegesStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowPrivilegesStmt.java @@ -21,7 +21,7 @@ import org.apache.doris.catalog.ScalarType; import org.apache.doris.qe.ShowResultSetMetaData; -public class ShowPrivilegesStmt extends ShowStmt { +public class ShowPrivilegesStmt extends ShowStmt implements NotFallbackInParser { private static final ShowResultSetMetaData META_DATA; static { diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowProcStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowProcStmt.java index bc7c70cdc792574..0d154db62326f70 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowProcStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowProcStmt.java @@ -31,7 +31,7 @@ import org.apache.doris.qe.ShowResultSetMetaData; // SHOW PROC statement. Used to show proc information, only admin can use. -public class ShowProcStmt extends ShowStmt { +public class ShowProcStmt extends ShowStmt implements NotFallbackInParser { private String path; private ProcNodeInterface node; diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowProcedureStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowProcedureStmt.java index db7684591fd7522..960f54a2a8c2d3c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowProcedureStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowProcedureStmt.java @@ -22,7 +22,7 @@ import org.apache.doris.qe.ShowResultSetMetaData; // SHOW PROCEDURE STATUS -public class ShowProcedureStmt extends ShowStmt { +public class ShowProcedureStmt extends ShowStmt implements NotFallbackInParser { private static final ShowResultSetMetaData META_DATA = ShowResultSetMetaData.builder() .addColumn(new Column("Db", ScalarType.createVarchar(64))) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowProcesslistStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowProcesslistStmt.java index e602e85f499105d..a4fd6a183c0678c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowProcesslistStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowProcesslistStmt.java @@ -25,7 +25,7 @@ // SHOW PROCESSLIST statement. // Used to show connection belong to this user. -public class ShowProcesslistStmt extends ShowStmt { +public class ShowProcesslistStmt extends ShowStmt implements NotFallbackInParser { private static final ShowResultSetMetaData META_DATA = ShowResultSetMetaData.builder() .addColumn(new Column("CurrentConnected", ScalarType.createVarchar(16))) .addColumn(new Column("Id", ScalarType.createType(PrimitiveType.BIGINT))) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowQueryProfileStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowQueryProfileStmt.java index 39d077438264bb5..ad664652d533432 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowQueryProfileStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowQueryProfileStmt.java @@ -24,7 +24,7 @@ // deprecated stmt, use will be guided to a specific url to get profile from // web browser -public class ShowQueryProfileStmt extends ShowStmt { +public class ShowQueryProfileStmt extends ShowStmt implements NotFallbackInParser { private String queryIdPath; public ShowQueryProfileStmt(String queryIdPath) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowQueryStatsStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowQueryStatsStmt.java index fee9809f96e36f2..723a0ef8629a6c9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowQueryStatsStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowQueryStatsStmt.java @@ -39,7 +39,7 @@ import java.util.List; import java.util.Map; -public class ShowQueryStatsStmt extends ShowStmt { +public class ShowQueryStatsStmt extends ShowStmt implements NotFallbackInParser { private static final ShowResultSetMetaData SHOW_QUERY_STATS_CATALOG_META_DATA = ShowResultSetMetaData.builder() .addColumn(new Column("Database", ScalarType.createVarchar(20))) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowReplicaDistributionStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowReplicaDistributionStmt.java index 58d2ac1052f6538..24ef8eddfc9a31e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowReplicaDistributionStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowReplicaDistributionStmt.java @@ -32,7 +32,7 @@ import com.google.common.collect.ImmutableList; // show replica distribution from tbl [partition(p1, p2, ...)] -public class ShowReplicaDistributionStmt extends ShowStmt { +public class ShowReplicaDistributionStmt extends ShowStmt implements NotFallbackInParser { public static final ImmutableList TITLE_NAMES = new ImmutableList.Builder() .add("BackendId").add("ReplicaNum").add("ReplicaSize") .add("NumGraph").add("NumPercent") diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowReplicaStatusStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowReplicaStatusStmt.java index 2a1f96fed96224a..44215ed115008ff 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowReplicaStatusStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowReplicaStatusStmt.java @@ -36,7 +36,7 @@ import java.util.List; -public class ShowReplicaStatusStmt extends ShowStmt { +public class ShowReplicaStatusStmt extends ShowStmt implements NotFallbackInParser { public static final ImmutableList TITLE_NAMES = new ImmutableList.Builder() .add("TabletId").add("ReplicaId").add("BackendId").add("Version").add("LastFailedVersion") .add("LastSuccessVersion").add("CommittedVersion").add("SchemaHash").add("VersionNum") diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowRepositoriesStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowRepositoriesStmt.java index 6e5166a5c870b89..6dc6e3bf0bbed87 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowRepositoriesStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowRepositoriesStmt.java @@ -29,7 +29,7 @@ import com.google.common.collect.ImmutableList; -public class ShowRepositoriesStmt extends ShowStmt { +public class ShowRepositoriesStmt extends ShowStmt implements NotFallbackInParser { public static final ImmutableList TITLE_NAMES = new ImmutableList.Builder() .add("RepoId").add("RepoName").add("CreateTime").add("IsReadOnly").add("Location") .add("Broker").add("Type").add("ErrMsg") diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowResourcesStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowResourcesStmt.java index 97393334679c219..9289d9036277825 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowResourcesStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowResourcesStmt.java @@ -40,7 +40,7 @@ // // syntax: // SHOW RESOURCES [LIKE mask] -public class ShowResourcesStmt extends ShowStmt { +public class ShowResourcesStmt extends ShowStmt implements NotFallbackInParser { private static final Logger LOG = LogManager.getLogger(ShowResourcesStmt.class); private String pattern; diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowRestoreStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowRestoreStmt.java index 26016be7814dea3..2ffa8607fbd86ea 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowRestoreStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowRestoreStmt.java @@ -38,7 +38,7 @@ import java.util.List; import java.util.function.Predicate; -public class ShowRestoreStmt extends ShowStmt { +public class ShowRestoreStmt extends ShowStmt implements NotFallbackInParser { public static final ImmutableList TITLE_NAMES = new ImmutableList.Builder() .add("JobId").add("Label").add("Timestamp").add("DbName").add("State") .add("AllowLoad").add("ReplicationNum").add("ReplicaAllocation").add("ReserveReplica") diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowRolesStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowRolesStmt.java index bbb66b274438ee6..1f3f19a6d29202a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowRolesStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowRolesStmt.java @@ -27,7 +27,7 @@ import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.ShowResultSetMetaData; -public class ShowRolesStmt extends ShowStmt { +public class ShowRolesStmt extends ShowStmt implements NotFallbackInParser { private static final ShowResultSetMetaData META_DATA; static { diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowRollupStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowRollupStmt.java index 2684f3bcea8d4e9..f7f65800b583264 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowRollupStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowRollupStmt.java @@ -31,7 +31,7 @@ // // Syntax: // SHOW ROLLUP { FROM | IN } table [ FROM db ] -public class ShowRollupStmt extends ShowStmt { +public class ShowRollupStmt extends ShowStmt implements NotFallbackInParser { private static final ShowResultSetMetaData META_DATA = ShowResultSetMetaData.builder() .addColumn(new Column("Table", ScalarType.createVarchar(20))) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowSmallFilesStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowSmallFilesStmt.java index ad8c57b7a477346..5a227f96eb4ca38 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowSmallFilesStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowSmallFilesStmt.java @@ -30,7 +30,7 @@ import com.google.common.base.Strings; -public class ShowSmallFilesStmt extends ShowStmt { +public class ShowSmallFilesStmt extends ShowStmt implements NotFallbackInParser { private static final ShowResultSetMetaData META_DATA = ShowResultSetMetaData.builder() .addColumn(new Column("Id", ScalarType.createVarchar(32))) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowSnapshotStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowSnapshotStmt.java index 83465a346005876..6a736499d8725db 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowSnapshotStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowSnapshotStmt.java @@ -32,7 +32,7 @@ import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; -public class ShowSnapshotStmt extends ShowStmt { +public class ShowSnapshotStmt extends ShowStmt implements NotFallbackInParser { public enum SnapshotType { REMOTE, LOCAL diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowSqlBlockRuleStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowSqlBlockRuleStmt.java index c487e1c7978b30a..ccf2b1ee6bf3df5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowSqlBlockRuleStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowSqlBlockRuleStmt.java @@ -36,7 +36,7 @@ show sql_block_rule show sql_block_rule for rule_name */ -public class ShowSqlBlockRuleStmt extends ShowStmt { +public class ShowSqlBlockRuleStmt extends ShowStmt implements NotFallbackInParser { private static final ShowResultSetMetaData META_DATA = ShowResultSetMetaData.builder() diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowStageStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowStageStmt.java index 09626911c1e328c..eac5b4aec83f3e6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowStageStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowStageStmt.java @@ -30,7 +30,7 @@ // // syntax: // SHOW STAGES -public class ShowStageStmt extends ShowStmt { +public class ShowStageStmt extends ShowStmt implements NotFallbackInParser { private static final Logger LOG = LogManager.getLogger(ShowStageStmt.class); private static final String NAME_COL = "StageName"; diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowStatusStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowStatusStmt.java index a6276a88d4ee301..abce39aec6f0ea7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowStatusStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowStatusStmt.java @@ -23,7 +23,7 @@ // Show Status statement // TODO(zhaochun): Add status information. -public class ShowStatusStmt extends ShowStmt { +public class ShowStatusStmt extends ShowStmt implements NotFallbackInParser { private static final ShowResultSetMetaData META_DATA = ShowResultSetMetaData.builder() .addColumn(new Column("Variable_name", ScalarType.createVarchar(20))) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowStmt.java index 9be50ee9dbd1f2b..92ed01acdfc81c8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowStmt.java @@ -20,7 +20,7 @@ import org.apache.doris.common.AnalysisException; import org.apache.doris.qe.ShowResultSetMetaData; -public abstract class ShowStmt extends StatementBase { +public abstract class ShowStmt extends StatementBase implements NotFallbackInParser { public abstract ShowResultSetMetaData getMetaData(); public SelectStmt toSelectStmt(Analyzer analyzer) throws AnalysisException { diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowStoragePolicyUsingStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowStoragePolicyUsingStmt.java index 56598a194eec129..b6a8e3671248330 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowStoragePolicyUsingStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowStoragePolicyUsingStmt.java @@ -34,7 +34,7 @@ * syntax: * SHOW STORAGE POLICY USING [for policy_name] **/ -public class ShowStoragePolicyUsingStmt extends ShowStmt { +public class ShowStoragePolicyUsingStmt extends ShowStmt implements NotFallbackInParser { public static final ShowResultSetMetaData RESULT_META_DATA = ShowResultSetMetaData.builder() diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowStorageVaultStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowStorageVaultStmt.java index 1cd1d06a1499d1e..f6124c4d20184d2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowStorageVaultStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowStorageVaultStmt.java @@ -31,7 +31,7 @@ * syntax: * SHOW STORAGE VAULT **/ -public class ShowStorageVaultStmt extends ShowStmt { +public class ShowStorageVaultStmt extends ShowStmt implements NotFallbackInParser { private final String stmt = "SHOW STORAGE VAULT"; diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowStreamLoadStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowStreamLoadStmt.java index ae133aa2a23e9e3..92b67e6cc3683c6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowStreamLoadStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowStreamLoadStmt.java @@ -41,7 +41,7 @@ // // syntax: // SHOW STREAM LOAD [FROM db] [LIKE mask] -public class ShowStreamLoadStmt extends ShowStmt { +public class ShowStreamLoadStmt extends ShowStmt implements NotFallbackInParser { private static final Logger LOG = LogManager.getLogger(ShowStreamLoadStmt.class); public enum StreamLoadState { diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowSyncJobStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowSyncJobStmt.java index f0671f8afe0619e..17a31fa1e16a6e6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowSyncJobStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowSyncJobStmt.java @@ -37,7 +37,7 @@ // // syntax: // SHOW SYNC JOB [FROM db] -public class ShowSyncJobStmt extends ShowStmt { +public class ShowSyncJobStmt extends ShowStmt implements NotFallbackInParser { private static final Logger LOG = LogManager.getLogger(ShowSyncJobStmt.class); public static final ImmutableList TITLE_NAMES = new ImmutableList.Builder() diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTableCreationStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTableCreationStmt.java index d6bd9d0ea647ca6..500265d45c65265 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTableCreationStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTableCreationStmt.java @@ -32,7 +32,7 @@ * Syntax: * SHOW TABLE CREATION [FROM db] [LIKE mask] */ -public class ShowTableCreationStmt extends ShowStmt { +public class ShowTableCreationStmt extends ShowStmt implements NotFallbackInParser { private static final ShowResultSetMetaData META_DATA = ShowResultSetMetaData.builder() diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTableIdStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTableIdStmt.java index fc07f1c672219c2..d58304896457502 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTableIdStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTableIdStmt.java @@ -29,7 +29,7 @@ import org.apache.doris.qe.ShowResultSetMetaData; // SHOW TABLE ID -public class ShowTableIdStmt extends ShowStmt { +public class ShowTableIdStmt extends ShowStmt implements NotFallbackInParser { private long tableId; public ShowTableIdStmt(long tableId) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTableStatsStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTableStatsStmt.java index 6070d76103b7400..54c8f652e125fab 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTableStatsStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTableStatsStmt.java @@ -49,7 +49,7 @@ import java.util.Map; import java.util.Set; -public class ShowTableStatsStmt extends ShowStmt { +public class ShowTableStatsStmt extends ShowStmt implements NotFallbackInParser { private static final ImmutableList TABLE_TITLE_NAMES = new ImmutableList.Builder() @@ -61,6 +61,7 @@ public class ShowTableStatsStmt extends ShowStmt { .add("trigger") .add("new_partition") .add("user_inject") + .add("enable_auto_analyze") .build(); private static final ImmutableList PARTITION_TITLE_NAMES = @@ -197,12 +198,12 @@ public long getTableId() { return tableId; } - public ShowResultSet constructResultSet(TableStatsMeta tableStatistic) { + public ShowResultSet constructResultSet(TableStatsMeta tableStatistic, TableIf table) { if (indexName != null) { return constructIndexResultSet(tableStatistic); } if (partitionNames == null) { - return constructTableResultSet(tableStatistic); + return constructTableResultSet(tableStatistic, table); } if (columnNames == null) { return constructPartitionResultSet(tableStatistic); @@ -215,7 +216,7 @@ public ShowResultSet constructEmptyResultSet() { return new ShowResultSet(getMetaData(), new ArrayList<>()); } - public ShowResultSet constructResultSet(long rowCount) { + public ShowResultSet constructResultSet(TableIf table) { List> result = Lists.newArrayList(); if (partitionNames != null) { // For partition, return empty result if table stats not exist. @@ -224,17 +225,18 @@ public ShowResultSet constructResultSet(long rowCount) { List row = Lists.newArrayList(); row.add(""); row.add(""); - row.add(String.valueOf(rowCount)); + row.add(String.valueOf(table.getCachedRowCount())); row.add(""); row.add(""); row.add(""); row.add(""); row.add(""); + row.add(String.valueOf(table.autoAnalyzeEnabled())); result.add(row); return new ShowResultSet(getMetaData(), result); } - public ShowResultSet constructTableResultSet(TableStatsMeta tableStatistic) { + public ShowResultSet constructTableResultSet(TableStatsMeta tableStatistic, TableIf table) { DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); if (tableStatistic == null) { return new ShowResultSet(getMetaData(), new ArrayList<>()); @@ -253,6 +255,7 @@ public ShowResultSet constructTableResultSet(TableStatsMeta tableStatistic) { row.add(tableStatistic.jobType.toString()); row.add(String.valueOf(tableStatistic.partitionChanged.get())); row.add(String.valueOf(tableStatistic.userInjected)); + row.add(table == null ? "N/A" : String.valueOf(table.autoAnalyzeEnabled())); result.add(row); return new ShowResultSet(getMetaData(), result); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTableStatusStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTableStatusStmt.java index 20e8790826832bc..296eb628da2df57 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTableStatusStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTableStatusStmt.java @@ -34,7 +34,7 @@ import com.google.common.collect.Lists; // SHOW TABLE STATUS -public class ShowTableStatusStmt extends ShowStmt { +public class ShowTableStatusStmt extends ShowStmt implements NotFallbackInParser { private static final ShowResultSetMetaData META_DATA = ShowResultSetMetaData.builder() .addColumn(new Column("Name", ScalarType.createVarchar(64))) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTableStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTableStmt.java index f3e12f1aa92e4de..92c5f34435f8463 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTableStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTableStmt.java @@ -33,7 +33,7 @@ import org.apache.logging.log4j.Logger; // SHOW TABLES -public class ShowTableStmt extends ShowStmt { +public class ShowTableStmt extends ShowStmt implements NotFallbackInParser { private static final Logger LOG = LogManager.getLogger(ShowTableStmt.class); private static final String NAME_COL_PREFIX = "Tables_in_"; private static final String TYPE_COL = "Table_type"; diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTabletStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTabletStmt.java index 71ba4e2be13a843..646d6a806393afa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTabletStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTabletStmt.java @@ -37,7 +37,7 @@ import java.util.ArrayList; import java.util.List; -public class ShowTabletStmt extends ShowStmt { +public class ShowTabletStmt extends ShowStmt implements NotFallbackInParser { private TableName dbTableName; private String dbName; private String tableName; diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTabletStorageFormatStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTabletStorageFormatStmt.java index 9d0f3b88e6c3b26..cf76ee504ee6f0a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTabletStorageFormatStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTabletStorageFormatStmt.java @@ -27,7 +27,7 @@ import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.ShowResultSetMetaData; -public class ShowTabletStorageFormatStmt extends ShowStmt { +public class ShowTabletStorageFormatStmt extends ShowStmt implements NotFallbackInParser { private boolean verbose; public ShowTabletStorageFormatStmt(boolean verbose) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTabletsBelongStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTabletsBelongStmt.java index 3819541fea9a72c..81d9d6d50171f51 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTabletsBelongStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTabletsBelongStmt.java @@ -36,7 +36,7 @@ * syntax: * SHOW TABLETS BELONG tablet_ids */ -public class ShowTabletsBelongStmt extends ShowStmt { +public class ShowTabletsBelongStmt extends ShowStmt implements NotFallbackInParser { private List tabletIds; private static final ImmutableList TITLE_NAMES = new ImmutableList.Builder() diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTransactionStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTransactionStmt.java index 3d5d03bdf8f768c..cd7d2ca5b016fb0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTransactionStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTransactionStmt.java @@ -37,7 +37,7 @@ // syntax: // SHOW TRANSACTION WHERE id=123 -public class ShowTransactionStmt extends ShowStmt { +public class ShowTransactionStmt extends ShowStmt implements NotFallbackInParser { private static final Logger LOG = LogManager.getLogger(ShowTransactionStmt.class); private String dbName; diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTrashDiskStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTrashDiskStmt.java index f5fad57d0f544ee..7fb15eae6638c36 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTrashDiskStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTrashDiskStmt.java @@ -32,7 +32,7 @@ import com.google.common.collect.ImmutableMap; -public class ShowTrashDiskStmt extends ShowStmt { +public class ShowTrashDiskStmt extends ShowStmt implements NotFallbackInParser { private Backend backend; diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTrashStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTrashStmt.java index 3071a657c53a4c6..e8e301193bcd8ce 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTrashStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTrashStmt.java @@ -34,7 +34,7 @@ import java.util.List; -public class ShowTrashStmt extends ShowStmt { +public class ShowTrashStmt extends ShowStmt implements NotFallbackInParser { private List backends = Lists.newArrayList(); public ShowTrashStmt() throws AnalysisException { diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTriggersStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTriggersStmt.java index e903d2387ee075f..de9cdf76816a221 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTriggersStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTriggersStmt.java @@ -21,7 +21,7 @@ import org.apache.doris.catalog.ScalarType; import org.apache.doris.qe.ShowResultSetMetaData; -public class ShowTriggersStmt extends ShowStmt { +public class ShowTriggersStmt extends ShowStmt implements NotFallbackInParser { private static final ShowResultSetMetaData META_DATA = ShowResultSetMetaData.builder() .addColumn(new Column("Trigger", ScalarType.createVarchar(64))) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTypeCastStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTypeCastStmt.java index c88c39a03db25ae..8ddf1a3d07f49ca 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTypeCastStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTypeCastStmt.java @@ -27,7 +27,7 @@ import com.google.common.base.Strings; -public class ShowTypeCastStmt extends ShowStmt { +public class ShowTypeCastStmt extends ShowStmt implements NotFallbackInParser { private static final ShowResultSetMetaData META_DATA = ShowResultSetMetaData.builder() .addColumn(new Column("Origin Type", ScalarType.createVarchar(32))) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowUserPropertyStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowUserPropertyStmt.java index 7521e7497bcb15f..76a9b9a9b4fa512 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowUserPropertyStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowUserPropertyStmt.java @@ -46,7 +46,7 @@ // Show Property Stmt // syntax: // SHOW [ALL] PROPERTY [FOR user] [LIKE key pattern] -public class ShowUserPropertyStmt extends ShowStmt { +public class ShowUserPropertyStmt extends ShowStmt implements NotFallbackInParser { private static final Logger LOG = LogManager.getLogger(ShowUserPropertyStmt.class); private String user; diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowVariablesStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowVariablesStmt.java index efe36311ca1a3e0..f398f021dc2c69b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowVariablesStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowVariablesStmt.java @@ -28,7 +28,7 @@ import org.apache.logging.log4j.Logger; // Show variables statement. -public class ShowVariablesStmt extends ShowStmt { +public class ShowVariablesStmt extends ShowStmt implements NotFallbackInParser { private static final Logger LOG = LogManager.getLogger(ShowVariablesStmt.class); private static final String NAME_COL = "Variable_name"; private static final String VALUE_COL = "Value"; diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowViewStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowViewStmt.java index 67c77664cccdaba..f8cf26ed1bd14b1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowViewStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowViewStmt.java @@ -43,7 +43,7 @@ // // Syntax: // SHOW VIEW { FROM | IN } table [ FROM db ] -public class ShowViewStmt extends ShowStmt { +public class ShowViewStmt extends ShowStmt implements NotFallbackInParser { private static final ShowResultSetMetaData META_DATA = ShowResultSetMetaData.builder() .addColumn(new Column("View", ScalarType.createVarchar(30))) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowWarningStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowWarningStmt.java index 7de4525ac2d0075..3018e9e56fd8cf4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowWarningStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowWarningStmt.java @@ -22,7 +22,7 @@ import org.apache.doris.qe.ShowResultSetMetaData; // Show Warning stmt -public class ShowWarningStmt extends ShowStmt { +public class ShowWarningStmt extends ShowStmt implements NotFallbackInParser { private static final ShowResultSetMetaData META_DATA = ShowResultSetMetaData.builder() .addColumn(new Column("Level", ScalarType.createVarchar(20))) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowWhiteListStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowWhiteListStmt.java index 43609d235cf8a89..42682a5780856b3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowWhiteListStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowWhiteListStmt.java @@ -21,7 +21,7 @@ import org.apache.doris.catalog.ScalarType; import org.apache.doris.qe.ShowResultSetMetaData; -public class ShowWhiteListStmt extends ShowStmt { +public class ShowWhiteListStmt extends ShowStmt implements NotFallbackInParser { private static final ShowResultSetMetaData META_DATA = ShowResultSetMetaData.builder() .addColumn(new Column("user_name", ScalarType.createVarchar(20))) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowWorkloadGroupsStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowWorkloadGroupsStmt.java index c802411be81913e..bb097e25a7fa77b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowWorkloadGroupsStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowWorkloadGroupsStmt.java @@ -25,7 +25,7 @@ import org.apache.doris.qe.ShowResultSetMetaData; import org.apache.doris.resource.workloadgroup.WorkloadGroupMgr; -public class ShowWorkloadGroupsStmt extends ShowStmt { +public class ShowWorkloadGroupsStmt extends ShowStmt implements NotFallbackInParser { private String pattern; // TODO: not supported yet diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/TransactionStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/TransactionStmt.java index 60c8d040f7281d8..a1ad0ed2ed7a743 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/TransactionStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/TransactionStmt.java @@ -20,7 +20,7 @@ import org.apache.doris.common.AnalysisException; import org.apache.doris.common.UserException; -public class TransactionStmt extends StatementBase { +public class TransactionStmt extends StatementBase implements NotFallbackInParser { @Override public RedirectStatus getRedirectStatus() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java index 383a8bbc4fee3cb..bcb0864b64f2e20 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java @@ -22,9 +22,7 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.Abs; import org.apache.doris.nereids.trees.expressions.functions.scalar.Acos; import org.apache.doris.nereids.trees.expressions.functions.scalar.AesDecrypt; -import org.apache.doris.nereids.trees.expressions.functions.scalar.AesDecryptV2; import org.apache.doris.nereids.trees.expressions.functions.scalar.AesEncrypt; -import org.apache.doris.nereids.trees.expressions.functions.scalar.AesEncryptV2; import org.apache.doris.nereids.trees.expressions.functions.scalar.AppendTrailingCharIfAbsent; import org.apache.doris.nereids.trees.expressions.functions.scalar.Array; import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayApply; @@ -370,9 +368,7 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.Sm3; import org.apache.doris.nereids.trees.expressions.functions.scalar.Sm3sum; import org.apache.doris.nereids.trees.expressions.functions.scalar.Sm4Decrypt; -import org.apache.doris.nereids.trees.expressions.functions.scalar.Sm4DecryptV2; import org.apache.doris.nereids.trees.expressions.functions.scalar.Sm4Encrypt; -import org.apache.doris.nereids.trees.expressions.functions.scalar.Sm4EncryptV2; import org.apache.doris.nereids.trees.expressions.functions.scalar.Space; import org.apache.doris.nereids.trees.expressions.functions.scalar.SplitByChar; import org.apache.doris.nereids.trees.expressions.functions.scalar.SplitByRegexp; @@ -479,9 +475,7 @@ public class BuiltinScalarFunctions implements FunctionHelper { scalar(Abs.class, "abs"), scalar(Acos.class, "acos"), scalar(AesDecrypt.class, "aes_decrypt"), - scalar(AesDecryptV2.class, "aes_decrypt_v2"), scalar(AesEncrypt.class, "aes_encrypt"), - scalar(AesEncryptV2.class, "aes_encrypt_v2"), scalar(AppendTrailingCharIfAbsent.class, "append_trailing_char_if_absent"), scalar(Array.class, "array"), scalar(ArrayApply.class, "array_apply"), @@ -849,9 +843,7 @@ public class BuiltinScalarFunctions implements FunctionHelper { scalar(Sm3.class, "sm3"), scalar(Sm3sum.class, "sm3sum"), scalar(Sm4Decrypt.class, "sm4_decrypt"), - scalar(Sm4DecryptV2.class, "sm4_decrypt_v2"), scalar(Sm4Encrypt.class, "sm4_encrypt"), - scalar(Sm4EncryptV2.class, "sm4_encrypt_v2"), scalar(Space.class, "space"), scalar(SplitByChar.class, "split_by_char"), scalar(SplitByRegexp.class, "split_by_regexp"), diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java index 839072556413849..23ba86e53cd8fb8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @@ -131,6 +131,7 @@ import org.apache.doris.cooldown.CooldownConfHandler; import org.apache.doris.datasource.CatalogIf; import org.apache.doris.datasource.CatalogMgr; +import org.apache.doris.datasource.ExternalCatalog; import org.apache.doris.datasource.ExternalMetaCacheMgr; import org.apache.doris.datasource.ExternalMetaIdMgr; import org.apache.doris.datasource.InternalCatalog; @@ -1679,10 +1680,15 @@ private void transferToMaster() { */ void advanceNextId() { long currentId = idGenerator.getBatchEndId(); - long currentNanos = System.nanoTime(); + long currentMill = System.currentTimeMillis(); long nextId = currentId + 1; - if (nextId < currentNanos) { - nextId = currentNanos; + // Reserve ~1 trillion for use in case of bugs or frequent reboots (~2 billion reboots) + if ((1L << 63) - nextId < (1L << 40)) { + LOG.warn("nextId is too large: {}, it may be a bug and consider backup and migration", nextId); + } else { + // Keep compatible with previous impl, the previous impl may result in extreme large nextId, + // and guess there are no more than 1L<<32 (~4e9) ids used since last reboot + nextId = (currentId + 1) < currentMill ? currentMill : currentId + (1L << 32); } // ATTN: Because MetaIdGenerator has guaranteed that each id it returns must have @@ -5286,7 +5292,8 @@ public void modifyTableProperties(Database db, OlapTable table, Map properties = info.getProperties(); + // Handle HMSExternalTable set auto analyze policy. + if (ctlName != null && !(InternalCatalog.INTERNAL_CATALOG_NAME.equalsIgnoreCase(ctlName))) { + setExternalTableAutoAnalyze(properties, info); + return; + } + Database db = getInternalCatalog().getDbOrMetaException(dbId); OlapTable olapTable = (OlapTable) db.getTableOrMetaException(tableId, TableType.OLAP); olapTable.writeLock(); @@ -5356,6 +5370,33 @@ public void replayModifyTableProperty(short opCode, ModifyTablePropertyOperation } } + private void setExternalTableAutoAnalyze(Map properties, ModifyTablePropertyOperationLog info) { + if (properties.size() != 1) { + LOG.warn("External table property should contain exactly 1 entry."); + return; + } + if (!properties.containsKey(PropertyAnalyzer.PROPERTIES_AUTO_ANALYZE_POLICY)) { + LOG.warn("External table property should only contain auto_analyze_policy"); + return; + } + String value = properties.get(PropertyAnalyzer.PROPERTIES_AUTO_ANALYZE_POLICY); + if (!PropertyAnalyzer.ENABLE_AUTO_ANALYZE_POLICY.equalsIgnoreCase(value) + && !PropertyAnalyzer.DISABLE_AUTO_ANALYZE_POLICY.equalsIgnoreCase(value) + && !PropertyAnalyzer.USE_CATALOG_AUTO_ANALYZE_POLICY.equalsIgnoreCase(value)) { + LOG.warn("External table property should be 'enable', 'disable' or 'base_on_catalog'"); + return; + } + try { + CatalogIf catalog = Env.getCurrentEnv().getCatalogMgr() + .getCatalogOrException(info.getCtlName(), + ctlName -> new DdlException("Unknown catalog " + ctlName)); + value = value.equalsIgnoreCase(PropertyAnalyzer.USE_CATALOG_AUTO_ANALYZE_POLICY) ? null : value; + ((ExternalCatalog) catalog).setAutoAnalyzePolicy(info.getDbName(), info.getTableName(), value); + } catch (Exception e) { + LOG.warn("Failed to replay external table set property.", e); + } + } + public void modifyDefaultDistributionBucketNum(Database db, OlapTable olapTable, ModifyDistributionClause modifyDistributionClause) throws DdlException { diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java index 78d97e8e48bc90c..533c24daa0e7bc4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -54,6 +54,8 @@ import org.apache.doris.mtmv.MTMVRelatedTableIf; import org.apache.doris.mtmv.MTMVSnapshotIf; import org.apache.doris.mtmv.MTMVVersionSnapshot; +import org.apache.doris.nereids.hint.Hint; +import org.apache.doris.nereids.hint.UseMvHint; import org.apache.doris.persist.gson.GsonPostProcessable; import org.apache.doris.persist.gson.GsonUtils; import org.apache.doris.qe.ConnectContext; @@ -565,6 +567,99 @@ public Map getVisibleIndexIdToMeta() { return visibleMVs; } + public Long getBestMvIdWithHint(List orderedMvs) { + Optional useMvHint = getUseMvHint("USE_MV"); + Optional noUseMvHint = getUseMvHint("NO_USE_MV"); + if (useMvHint.isPresent() && noUseMvHint.isPresent()) { + if (noUseMvHint.get().getNoUseMVName(this.name).contains(useMvHint.get().getUseMvName(this.name))) { + String errorMsg = "conflict mv exist in use_mv and no_use_mv in the same time" + + useMvHint.get().getUseMvName(this.name); + useMvHint.get().setStatus(Hint.HintStatus.SYNTAX_ERROR); + useMvHint.get().setErrorMessage(errorMsg); + noUseMvHint.get().setStatus(Hint.HintStatus.SYNTAX_ERROR); + noUseMvHint.get().setErrorMessage(errorMsg); + } + return getMvIdWithUseMvHint(useMvHint.get(), orderedMvs); + } else if (useMvHint.isPresent()) { + return getMvIdWithUseMvHint(useMvHint.get(), orderedMvs); + } else if (noUseMvHint.isPresent()) { + return getMvIdWithNoUseMvHint(noUseMvHint.get(), orderedMvs); + } + return orderedMvs.get(0); + } + + private Long getMvIdWithUseMvHint(UseMvHint useMvHint, List orderedMvs) { + if (useMvHint.isAllMv()) { + useMvHint.setStatus(Hint.HintStatus.SYNTAX_ERROR); + useMvHint.setErrorMessage("use_mv hint should only have one mv in one table: " + + this.name); + return orderedMvs.get(0); + } else { + String mvName = useMvHint.getUseMvName(this.name); + if (mvName != null) { + if (mvName.equals("`*`")) { + useMvHint.setStatus(Hint.HintStatus.SYNTAX_ERROR); + useMvHint.setErrorMessage("use_mv hint should only have one mv in one table: " + + this.name); + return orderedMvs.get(0); + } + Long choosedIndexId = indexNameToId.get(mvName); + if (orderedMvs.contains(choosedIndexId)) { + useMvHint.setStatus(Hint.HintStatus.SUCCESS); + return choosedIndexId; + } else { + useMvHint.setStatus(Hint.HintStatus.SYNTAX_ERROR); + useMvHint.setErrorMessage("do not have mv: " + mvName + " in table: " + this.name); + } + } + } + return orderedMvs.get(0); + } + + private Long getMvIdWithNoUseMvHint(UseMvHint noUseMvHint, List orderedMvs) { + if (noUseMvHint.isAllMv()) { + noUseMvHint.setStatus(Hint.HintStatus.SUCCESS); + return getBaseIndex().getId(); + } else { + List mvNames = noUseMvHint.getNoUseMVName(this.name); + Set forbiddenIndexIds = Sets.newHashSet(); + for (int i = 0; i < mvNames.size(); i++) { + if (mvNames.get(i).equals("`*`")) { + noUseMvHint.setStatus(Hint.HintStatus.SUCCESS); + return getBaseIndex().getId(); + } + if (hasMaterializedIndex(mvNames.get(i))) { + Long forbiddenIndexId = indexNameToId.get(mvNames.get(i)); + forbiddenIndexIds.add(forbiddenIndexId); + } else { + noUseMvHint.setStatus(Hint.HintStatus.SYNTAX_ERROR); + noUseMvHint.setErrorMessage("do not have mv: " + mvNames.get(i) + " in table: " + this.name); + break; + } + } + for (int i = 0; i < orderedMvs.size(); i++) { + if (forbiddenIndexIds.contains(orderedMvs.get(i))) { + noUseMvHint.setStatus(Hint.HintStatus.SUCCESS); + } else { + return orderedMvs.get(i); + } + } + } + return orderedMvs.get(0); + } + + private Optional getUseMvHint(String useMvName) { + for (Hint hint : ConnectContext.get().getStatementContext().getHints()) { + if (hint.isSyntaxError()) { + continue; + } + if (hint.getHintName().equalsIgnoreCase(useMvName)) { + return Optional.of((UseMvHint) hint); + } + } + return Optional.empty(); + } + public List getVisibleIndex() { Optional partition = idToPartition.values().stream().findFirst(); if (!partition.isPresent()) { @@ -3158,4 +3253,19 @@ public boolean isShadowIndex(long indexId) { return false; } } + + @Override + public boolean autoAnalyzeEnabled() { + if (tableProperty == null) { + return super.autoAnalyzeEnabled(); + } + Map properties = tableProperty.getProperties(); + if (properties == null || !properties.containsKey(PropertyAnalyzer.PROPERTIES_AUTO_ANALYZE_POLICY) + || properties.get(PropertyAnalyzer.PROPERTIES_AUTO_ANALYZE_POLICY) + .equalsIgnoreCase(PropertyAnalyzer.USE_CATALOG_AUTO_ANALYZE_POLICY)) { + return super.autoAnalyzeEnabled(); + } + return properties.get(PropertyAnalyzer.PROPERTIES_AUTO_ANALYZE_POLICY) + .equalsIgnoreCase(PropertyAnalyzer.ENABLE_AUTO_ANALYZE_POLICY); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/SchemaTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/SchemaTable.java index 683430548a6408d..174b2ca7d550a4e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/SchemaTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/SchemaTable.java @@ -557,6 +557,14 @@ public class SchemaTable extends Table { .column("REMOTE_SCAN_BYTES_PER_SECOND", ScalarType.createType(PrimitiveType.BIGINT)) .build()) ) + .put("catalog_meta_cache_statistics", + new SchemaTable(SystemIdGenerator.getNextId(), "catalog_meta_cache_statistics", TableType.SCHEMA, + builder().column("CATALOG_NAME", ScalarType.createStringType()) + .column("CACHE_NAME", ScalarType.createStringType()) + .column("METRIC_NAME", ScalarType.createStringType()) + .column("METRIC_VALUE", ScalarType.createStringType()) + .build()) + ) .build(); private boolean fetchAllFe = false; diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java index 906a710f369c621..e9392deae4a6b08 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java @@ -634,4 +634,9 @@ public Set> getColumnIndexPairs(Set columns) { public long getCachedRowCount() { return getRowCount(); } + + @Override + public boolean autoAnalyzeEnabled() { + return true; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java index cb9d780d55a80e4..83c07c485a1d2bd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java @@ -561,4 +561,6 @@ default Set getDistributionColumnNames() { default boolean isPartitionedTable() { return false; } + + boolean autoAnalyzeEnabled(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java index 7864cc7dee991c7..fdb60357f40ba07 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java @@ -123,6 +123,9 @@ public class TableProperty implements Writable, GsonPostProcessable { private long timeSeriesCompactionLevelThreshold = PropertyAnalyzer.TIME_SERIES_COMPACTION_LEVEL_THRESHOLD_DEFAULT_VALUE; + private String autoAnalyzePolicy = PropertyAnalyzer.ENABLE_AUTO_ANALYZE_POLICY; + + private DataSortInfo dataSortInfo = new DataSortInfo(); public TableProperty(Map properties) { @@ -162,6 +165,7 @@ public TableProperty buildProperty(short opCode) { buildTimeSeriesCompactionEmptyRowsetsThreshold(); buildTimeSeriesCompactionLevelThreshold(); buildTTLSeconds(); + buildAutoAnalyzeProperty(); break; default: break; @@ -235,6 +239,12 @@ public TableProperty buildDisableAutoCompaction() { return this; } + public TableProperty buildAutoAnalyzeProperty() { + autoAnalyzePolicy = properties.getOrDefault(PropertyAnalyzer.PROPERTIES_AUTO_ANALYZE_POLICY, + PropertyAnalyzer.ENABLE_AUTO_ANALYZE_POLICY); + return this; + } + public boolean disableAutoCompaction() { return disableAutoCompaction; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/alter/CloudSchemaChangeHandler.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/alter/CloudSchemaChangeHandler.java index e48b8a745a3e0c0..fa3bb96014c558c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/alter/CloudSchemaChangeHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/alter/CloudSchemaChangeHandler.java @@ -100,7 +100,8 @@ public void updateTableProperties(Database db, String tableName, Map target throws UserException, TException { super(db, table, targetColumnNames, queryId, groupCommit); } - - @Override - protected void selectBackends(ConnectContext ctx) throws DdlException { - try { - backend = Env.getCurrentEnv().getGroupCommitManager() - .selectBackendForGroupCommit(this.table.getId(), ctx, true); - } catch (LoadException e) { - throw new DdlException("No suitable backend"); - } - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/Daemon.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/Daemon.java index 472285b476497f2..16ac3259318f608 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/Daemon.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/Daemon.java @@ -121,7 +121,7 @@ public void run() { try { Thread.sleep(intervalMs); } catch (InterruptedException e) { - LOG.error("InterruptedException: ", e); + LOG.info("InterruptedException: ", e); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java index 6000831172e421d..8441c9307ca5ad3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java @@ -219,6 +219,11 @@ public class PropertyAnalyzer { public static final boolean PROPERTIES_ENABLE_MOW_LIGHT_DELETE_DEFAULT_VALUE = Config.enable_mow_light_delete; + public static final String PROPERTIES_AUTO_ANALYZE_POLICY = "auto_analyze_policy"; + public static final String ENABLE_AUTO_ANALYZE_POLICY = "enable"; + public static final String DISABLE_AUTO_ANALYZE_POLICY = "disable"; + public static final String USE_CATALOG_AUTO_ANALYZE_POLICY = "base_on_catalog"; + // compaction policy public static final String SIZE_BASED_COMPACTION_POLICY = "size_based"; public static final String TIME_SERIES_COMPACTION_POLICY = "time_series"; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java index 0d854a9ecdc2917..37e44c63569da22 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java @@ -33,6 +33,7 @@ import org.apache.doris.cluster.ClusterNamespace; import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; +import org.apache.doris.common.Pair; import org.apache.doris.common.UserException; import org.apache.doris.common.Version; import org.apache.doris.common.io.Text; @@ -127,6 +128,9 @@ public abstract class ExternalCatalog protected Map> idToDb = Maps.newConcurrentMap(); @SerializedName(value = "lastUpdateTime") protected long lastUpdateTime; + // to tableAutoAnalyzePolicy + @SerializedName(value = "taap") + protected Map, String> tableAutoAnalyzePolicy = Maps.newHashMap(); // db name does not contains "default_cluster" protected Map dbNameToId = Maps.newConcurrentMap(); private boolean objectCreated = false; @@ -723,6 +727,9 @@ public void gsonPostProcess() throws IOException { this.propLock = new byte[0]; this.initialized = false; setDefaultPropsIfMissing(true); + if (tableAutoAnalyzePolicy == null) { + tableAutoAnalyzePolicy = Maps.newHashMap(); + } } public void addDatabaseForTest(ExternalDatabase db) { @@ -884,4 +891,13 @@ public void truncateTable(TruncateTableStmt stmt) throws DdlException { public String getQualifiedName(String dbName) { return String.join(".", name, dbName); } + + public void setAutoAnalyzePolicy(String dbName, String tableName, String policy) { + Pair key = Pair.of(dbName, tableName); + if (policy == null) { + tableAutoAnalyzePolicy.remove(key); + } else { + tableAutoAnalyzePolicy.put(key, policy); + } + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java index 513fc951672f9fb..8759f9f5ef2f04c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java @@ -36,6 +36,7 @@ import com.github.benmanes.caffeine.cache.CacheLoader; import com.github.benmanes.caffeine.cache.RemovalListener; +import com.github.benmanes.caffeine.cache.stats.CacheStats; import com.google.common.collect.Maps; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -296,4 +297,15 @@ public MetaCache buildMetaCache(String name, maxSize, namesCacheLoader, metaObjCacheLoader, removalListener); return metaCache; } + + public static Map getCacheStats(CacheStats cacheStats, long estimatedSize) { + Map stats = Maps.newHashMap(); + stats.put("hit_ratio", String.valueOf(cacheStats.hitRate())); + stats.put("hit_count", String.valueOf(cacheStats.hitCount())); + stats.put("read_count", String.valueOf(cacheStats.hitCount() + cacheStats.missCount())); + stats.put("eviction_count", String.valueOf(cacheStats.evictionCount())); + stats.put("average_load_penalty", String.valueOf(cacheStats.averageLoadPenalty())); + stats.put("estimated_size", String.valueOf(estimatedSize)); + return stats; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java index 2d5689a54687c16..eedbe4e20da3126 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java @@ -27,6 +27,7 @@ import org.apache.doris.common.Pair; import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; +import org.apache.doris.common.util.PropertyAnalyzer; import org.apache.doris.common.util.Util; import org.apache.doris.persist.gson.GsonPostProcessable; import org.apache.doris.persist.gson.GsonUtils; @@ -282,6 +283,16 @@ public List getColumns() { return getFullSchema(); } + @Override + public boolean autoAnalyzeEnabled() { + makeSureInitialized(); + String policy = catalog.getTableAutoAnalyzePolicy().get(Pair.of(dbName, name)); + if (policy == null) { + return catalog.enableAutoAnalyze(); + } + return policy.equalsIgnoreCase(PropertyAnalyzer.ENABLE_AUTO_ANALYZE_POLICY); + } + @Override public Optional getColumnStatistic(String colName) { return Optional.empty(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java index a9f2da13b4093b8..8217f1c3a367df7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java @@ -53,6 +53,7 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData; @@ -741,18 +742,17 @@ public Map getAndCopyPartitionItems() { return res; } - private HiveMetaStoreCache.HivePartitionValues getHivePartitionValues() { - HiveMetaStoreCache cache = Env.getCurrentEnv().getExtMetaCacheMgr() - .getMetaStoreCache((HMSExternalCatalog) getCatalog()); - return cache.getPartitionValues( - getDbName(), getName(), getPartitionColumnTypes()); - } - @Override public MTMVSnapshotIf getPartitionSnapshot(String partitionName, MTMVRefreshContext context) throws AnalysisException { - long partitionLastModifyTime = getPartitionLastModifyTime(partitionName); - return new MTMVTimestampSnapshot(partitionLastModifyTime); + HiveMetaStoreCache cache = Env.getCurrentEnv().getExtMetaCacheMgr() + .getMetaStoreCache((HMSExternalCatalog) getCatalog()); + HiveMetaStoreCache.HivePartitionValues hivePartitionValues = cache.getPartitionValues( + getDbName(), getName(), getPartitionColumnTypes()); + Long partitionId = getPartitionIdByNameOrAnalysisException(partitionName, hivePartitionValues); + HivePartition hivePartition = getHivePartitionByIdOrAnalysisException(partitionId, + hivePartitionValues, cache); + return new MTMVTimestampSnapshot(hivePartition.getLastModifiedTime()); } @Override @@ -760,45 +760,50 @@ public MTMVSnapshotIf getTableSnapshot(MTMVRefreshContext context) throws Analys if (getPartitionType() == PartitionType.UNPARTITIONED) { return new MTMVMaxTimestampSnapshot(getName(), getLastDdlTime()); } - String partitionName = ""; + Long maxPartitionId = 0L; long maxVersionTime = 0L; long visibleVersionTime; - for (Entry entry : getAndCopyPartitionItems().entrySet()) { - visibleVersionTime = getPartitionLastModifyTime(entry.getKey()); + HiveMetaStoreCache cache = Env.getCurrentEnv().getExtMetaCacheMgr() + .getMetaStoreCache((HMSExternalCatalog) getCatalog()); + HiveMetaStoreCache.HivePartitionValues hivePartitionValues = cache.getPartitionValues( + getDbName(), getName(), getPartitionColumnTypes()); + BiMap idToName = hivePartitionValues.getPartitionNameToIdMap().inverse(); + if (MapUtils.isEmpty(idToName)) { + throw new AnalysisException("partitions is empty for : " + getName()); + } + for (Long partitionId : idToName.keySet()) { + visibleVersionTime = getHivePartitionByIdOrAnalysisException(partitionId, hivePartitionValues, + cache).getLastModifiedTime(); if (visibleVersionTime > maxVersionTime) { maxVersionTime = visibleVersionTime; - partitionName = entry.getKey(); + maxPartitionId = partitionId; } } - return new MTMVMaxTimestampSnapshot(partitionName, maxVersionTime); + return new MTMVMaxTimestampSnapshot(idToName.get(maxPartitionId), maxVersionTime); } - private long getPartitionLastModifyTime(String partitionName) throws AnalysisException { - return getPartitionByName(partitionName).getLastModifiedTime(); - } - - private HivePartition getPartitionByName(String partitionName) throws AnalysisException { - PartitionItem item = getAndCopyPartitionItems().get(partitionName); - List> partitionValuesList = transferPartitionItemToPartitionValues(item); - List partitions = getPartitionsByPartitionValues(partitionValuesList); - if (partitions.size() != 1) { - throw new AnalysisException("partition not normal, size: " + partitions.size()); + private Long getPartitionIdByNameOrAnalysisException(String partitionName, + HiveMetaStoreCache.HivePartitionValues hivePartitionValues) + throws AnalysisException { + Long partitionId = hivePartitionValues.getPartitionNameToIdMap().get(partitionName); + if (partitionId == null) { + throw new AnalysisException("can not find partition: " + partitionName); } - return partitions.get(0); + return partitionId; } - private List getPartitionsByPartitionValues(List> partitionValuesList) { - HiveMetaStoreCache cache = Env.getCurrentEnv().getExtMetaCacheMgr() - .getMetaStoreCache((HMSExternalCatalog) getCatalog()); - return cache.getAllPartitionsWithCache(getDbName(), getName(), - partitionValuesList); - } - - private List> transferPartitionItemToPartitionValues(PartitionItem item) { - List> partitionValuesList = Lists.newArrayListWithCapacity(1); - partitionValuesList.add( - ((ListPartitionItem) item).getItems().get(0).getPartitionValuesAsStringListForHive()); - return partitionValuesList; + private HivePartition getHivePartitionByIdOrAnalysisException(Long partitionId, + HiveMetaStoreCache.HivePartitionValues hivePartitionValues, + HiveMetaStoreCache cache) throws AnalysisException { + List partitionValues = hivePartitionValues.getPartitionValuesMap().get(partitionId); + if (CollectionUtils.isEmpty(partitionValues)) { + throw new AnalysisException("can not find partitionValues: " + partitionId); + } + HivePartition partition = cache.getHivePartition(getDbName(), getName(), partitionValues); + if (partition == null) { + throw new AnalysisException("can not find partition: " + partitionId); + } + return partition; } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java index b87c14afbc8a428..3483ca155ba6ae7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java @@ -36,6 +36,7 @@ import org.apache.doris.common.util.LocationPath; import org.apache.doris.common.util.Util; import org.apache.doris.datasource.CacheException; +import org.apache.doris.datasource.ExternalMetaCacheMgr; import org.apache.doris.datasource.hive.AcidInfo.DeleteDeltaInfo; import org.apache.doris.datasource.property.PropertyConverter; import org.apache.doris.fs.FileSystemCache; @@ -141,7 +142,7 @@ private void init() { OptionalLong.of(28800L), OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60L), Config.max_hive_partition_table_cache_num, - false, + true, null); partitionValuesCache = partitionValuesCacheFactory.buildCache(key -> loadPartitionValues(key), null, refreshExecutor); @@ -150,7 +151,7 @@ private void init() { OptionalLong.of(28800L), OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60L), Config.max_hive_partition_cache_num, - false, + true, null); partitionCache = partitionCacheFactory.buildCache(new CacheLoader() { @Override @@ -183,7 +184,7 @@ public void setNewFileCache() { ? fileMetaCacheTtlSecond : 28800L), OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60L), Config.max_external_file_cache_num, - false, + true, null); CacheLoader loader = new CacheBulkLoader() { @@ -521,6 +522,10 @@ public List getFilesByPartitions(List partitions, return fileLists; } + public HivePartition getHivePartition(String dbName, String name, List partitionValues) { + return partitionCache.get(new PartitionCacheKey(dbName, name, partitionValues)); + } + public List getAllPartitionsWithCache(String dbName, String name, List> partitionValuesList) { return getAllPartitions(dbName, name, partitionValuesList, true); @@ -1131,4 +1136,19 @@ public HivePartitionValues copy() { return copy; } } + + /** + * get cache stats + * @return metric value>> + */ + public Map> getStats() { + Map> res = Maps.newHashMap(); + res.put("hive_partition_values_cache", ExternalMetaCacheMgr.getCacheStats(partitionValuesCache.stats(), + partitionCache.estimatedSize())); + res.put("hive_partition_cache", + ExternalMetaCacheMgr.getCacheStats(partitionCache.stats(), partitionCache.estimatedSize())); + res.put("hive_file_cache", + ExternalMetaCacheMgr.getCacheStats(fileCacheRef.get().stats(), fileCacheRef.get().estimatedSize())); + return res; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiCachedPartitionProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiCachedPartitionProcessor.java index 2372b88e0db32e6..62094b21c2ba1d1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiCachedPartitionProcessor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiCachedPartitionProcessor.java @@ -21,6 +21,7 @@ import org.apache.doris.common.Config; import org.apache.doris.common.util.Util; import org.apache.doris.datasource.CacheException; +import org.apache.doris.datasource.ExternalMetaCacheMgr; import org.apache.doris.datasource.TablePartitionValues; import org.apache.doris.datasource.TablePartitionValues.TablePartitionKey; import org.apache.doris.datasource.hive.HMSExternalCatalog; @@ -28,6 +29,7 @@ import com.github.benmanes.caffeine.cache.LoadingCache; import com.google.common.base.Preconditions; +import com.google.common.collect.Maps; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.timeline.HoodieInstant; import org.apache.hudi.common.table.timeline.HoodieTimeline; @@ -37,6 +39,7 @@ import java.util.Arrays; import java.util.List; +import java.util.Map; import java.util.OptionalLong; import java.util.concurrent.Executor; import java.util.concurrent.ExecutorService; @@ -55,7 +58,7 @@ public HudiCachedPartitionProcessor(long catalogId, ExecutorService executor) { OptionalLong.of(28800L), OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60), Config.max_external_table_cache_num, - false, + true, null); this.partitionCache = partitionCacheFactory.buildCache(key -> new TablePartitionValues(), null, executor); } @@ -167,4 +170,11 @@ public TablePartitionValues getPartitionValues(HMSExternalTable table, HoodieTab throw new CacheException("Failed to get hudi partitions: " + Util.getRootCauseMessage(e), e); } } + + public Map> getCacheStats() { + Map> res = Maps.newHashMap(); + res.put("hudi_partition_cache", ExternalMetaCacheMgr.getCacheStats(partitionCache.stats(), + partitionCache.estimatedSize())); + return res; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataCache.java index 13bd9650978f81b..a35c73dc296db59 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataCache.java @@ -22,6 +22,7 @@ import org.apache.doris.common.Config; import org.apache.doris.common.UserException; import org.apache.doris.datasource.CatalogIf; +import org.apache.doris.datasource.ExternalMetaCacheMgr; import org.apache.doris.datasource.hive.HMSExternalCatalog; import org.apache.doris.datasource.hive.HiveMetaStoreClientHelper; import org.apache.doris.datasource.property.constants.HMSProperties; @@ -31,6 +32,7 @@ import com.github.benmanes.caffeine.cache.LoadingCache; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import org.apache.hadoop.conf.Configuration; import org.apache.iceberg.ManifestFiles; import org.apache.iceberg.SerializableTable; @@ -58,7 +60,7 @@ public IcebergMetadataCache(ExecutorService executor) { OptionalLong.of(28800L), OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60), Config.max_external_table_cache_num, - false, + true, null); this.snapshotListCache = snapshotListCacheFactory.buildCache(key -> loadSnapshots(key), null, executor); @@ -66,7 +68,7 @@ public IcebergMetadataCache(ExecutorService executor) { OptionalLong.of(28800L), OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60), Config.max_external_table_cache_num, - false, + true, null); this.tableCache = tableCacheFactory.buildCache(key -> loadTable(key), null, executor); } @@ -246,4 +248,13 @@ public int hashCode() { return Objects.hash(catalog.getId(), dbName, tableName); } } + + public Map> getCacheStats() { + Map> res = Maps.newHashMap(); + res.put("iceberg_snapshot_cache", ExternalMetaCacheMgr.getCacheStats(snapshotListCache.stats(), + snapshotListCache.estimatedSize())); + res.put("iceberg_table_cache", ExternalMetaCacheMgr.getCacheStats(tableCache.stats(), + tableCache.estimatedSize())); + return res; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/LoadAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/LoadAction.java index c9681eb784981f7..1f1c5d965c25da8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/LoadAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/LoadAction.java @@ -434,7 +434,7 @@ private TNetworkAddress selectLocalRedirectBackend(boolean groupCommit, HttpServ throw new LoadException(SystemInfoService.NO_BACKEND_LOAD_AVAILABLE_MSG + ", policy: " + policy); } if (groupCommit) { - backend = selectBackendForGroupCommit("", request, tableId, false); + backend = selectBackendForGroupCommit("", request, tableId); } else { backend = Env.getCurrentSystemInfo().getBackend(backendIds.get(0)); } @@ -449,7 +449,7 @@ private TNetworkAddress selectCloudRedirectBackend(String clusterName, HttpServl throws LoadException { Backend backend = null; if (groupCommit) { - backend = selectBackendForGroupCommit(clusterName, req, tableId, true); + backend = selectBackendForGroupCommit(clusterName, req, tableId); } else { backend = StreamLoadHandler.selectBackend(clusterName); } @@ -665,8 +665,7 @@ private String getAllHeaders(HttpServletRequest request) { return headers.toString(); } - private Backend selectBackendForGroupCommit(String clusterName, HttpServletRequest req, long tableId, - boolean isCloud) + private Backend selectBackendForGroupCommit(String clusterName, HttpServletRequest req, long tableId) throws LoadException { ConnectContext ctx = new ConnectContext(); ctx.setEnv(Env.getCurrentEnv()); @@ -676,14 +675,14 @@ private Backend selectBackendForGroupCommit(String clusterName, HttpServletReque // TMasterOpRequest(FrontendService.thrift) ctx.setQualifiedUser(Auth.ADMIN_USER); ctx.setThreadLocalInfo(); - if (isCloud) { + if (Config.isCloudMode()) { ctx.setCloudCluster(clusterName); } Backend backend = null; try { backend = Env.getCurrentEnv().getGroupCommitManager() - .selectBackendForGroupCommit(tableId, ctx, isCloud); + .selectBackendForGroupCommit(tableId, ctx); } catch (DdlException e) { throw new LoadException(e.getMessage(), e); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/GroupCommitManager.java b/fe/fe-core/src/main/java/org/apache/doris/load/GroupCommitManager.java index e4dd77e3e206557..717cf0c61035e10 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/GroupCommitManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/GroupCommitManager.java @@ -186,14 +186,14 @@ private long getWalQueueSize(Backend backend, PGetWalQueueSizeRequest request) { return size; } - public Backend selectBackendForGroupCommit(long tableId, ConnectContext context, boolean isCloud) + public Backend selectBackendForGroupCommit(long tableId, ConnectContext context) throws LoadException, DdlException { // If a group commit request is sent to the follower FE, we will send this request to the master FE. master FE // can select a BE and return this BE id to follower FE. if (!Env.getCurrentEnv().isMaster()) { try { long backendId = new MasterOpExecutor(context) - .getGroupCommitLoadBeId(tableId, context.getCloudCluster(), isCloud); + .getGroupCommitLoadBeId(tableId, context.getCloudCluster()); return Env.getCurrentSystemInfo().getBackend(backendId); } catch (Exception e) { throw new LoadException(e.getMessage()); @@ -201,11 +201,11 @@ public Backend selectBackendForGroupCommit(long tableId, ConnectContext context, } else { // Master FE will select BE by itself. return Env.getCurrentSystemInfo() - .getBackend(selectBackendForGroupCommitInternal(tableId, context.getCloudCluster(), isCloud)); + .getBackend(selectBackendForGroupCommitInternal(tableId, context.getCloudCluster())); } } - public long selectBackendForGroupCommitInternal(long tableId, String cluster, boolean isCloud) + public long selectBackendForGroupCommitInternal(long tableId, String cluster) throws LoadException, DdlException { // Understanding Group Commit and Backend Selection Logic // @@ -237,7 +237,7 @@ public long selectBackendForGroupCommitInternal(long tableId, String cluster, bo // a BE is chosen at random. This BE is then recorded along with the mapping of table A and its load level. // This approach ensures that group commits can effectively batch data together // while managing the load on each BE efficiently. - return isCloud ? selectBackendForCloudGroupCommitInternal(tableId, cluster) + return Config.isCloudMode() ? selectBackendForCloudGroupCommitInternal(tableId, cluster) : selectBackendForLocalGroupCommitInternal(tableId); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/Role.java b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/Role.java index 8354c655e2a7f3e..0054579062fcc50 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/Role.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/Role.java @@ -662,14 +662,15 @@ private void grantPrivs(ResourcePattern resourcePattern, PrivBitSet privs) throw break; case CLUSTER: cloudClusterPrivTable.addEntry(entry, false, false); - LOG.info("cloud cluster add list {}", cloudClusterPrivTable); + LOG.info("cloud cluster priv table after add {}", cloudClusterPrivTable); break; case STAGE: cloudStagePrivTable.addEntry(entry, false, false); - LOG.info("cloud stage add list {}", cloudStagePrivTable); + LOG.info("cloud stage priv table after add {}", cloudStagePrivTable); break; case STORAGE_VAULT: storageVaultPrivTable.addEntry(entry, false, false); + LOG.info("cloud storage vault priv table after add {}", storageVaultPrivTable); break; default: throw new DdlException("Unknown resource type: " + resourcePattern.getResourceType() + " name=" @@ -1166,18 +1167,26 @@ private void rebuildPrivTables() { workloadGroupPrivTable = new WorkloadGroupPrivTable(); cloudClusterPrivTable = new ResourcePrivTable(); cloudStagePrivTable = new ResourcePrivTable(); + storageVaultPrivTable = new ResourcePrivTable(); for (Entry entry : tblPatternToPrivs.entrySet()) { try { grantPrivs(entry.getKey(), entry.getValue().copy()); } catch (DdlException e) { - LOG.warn("grant failed,", e); + LOG.warn("grant tblPatternToPrivs failed,", e); } } for (Entry entry : resourcePatternToPrivs.entrySet()) { try { grantPrivs(entry.getKey(), entry.getValue().copy()); } catch (DdlException e) { - LOG.warn("grant failed,", e); + LOG.warn("grant resourcePatternToPrivs failed,", e); + } + } + for (Entry entry : storageVaultPatternToPrivs.entrySet()) { + try { + grantPrivs(entry.getKey(), entry.getValue().copy()); + } catch (DdlException e) { + LOG.warn("grant storageVaultPatternToPrivs failed,", e); } } for (Entry entry : clusterPatternToPrivs.entrySet()) { @@ -1204,7 +1213,7 @@ private void rebuildPrivTables() { try { grantPrivs(entry.getKey(), entry.getValue().copy()); } catch (DdlException e) { - LOG.warn("grant failed,", e); + LOG.warn("grant workloadGroupPatternToPrivs failed,", e); } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/hint/UseMvHint.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/hint/UseMvHint.java new file mode 100644 index 000000000000000..5e37bdc27603bc8 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/hint/UseMvHint.java @@ -0,0 +1,144 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.hint; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * rule hint. + */ +public class UseMvHint extends Hint { + + private final boolean isUseMv; + + private final boolean isAllMv; + + private final List parameters; + + private final Map useMvTableColumnMap; + + private final Map> noUseMvTableColumnMap; + + /** + * constructor of use mv hint + * @param hintName use mv + * @param parameters original parameters + * @param isUseMv use_mv hint or no_use_mv hint + * @param isAllMv should all mv be controlled + */ + public UseMvHint(String hintName, List parameters, boolean isUseMv, boolean isAllMv) { + super(hintName); + this.isUseMv = isUseMv; + this.isAllMv = isAllMv; + this.parameters = parameters; + this.useMvTableColumnMap = initUseMvTableColumnMap(parameters); + this.noUseMvTableColumnMap = initNoUseMvTableColumnMap(parameters); + } + + private Map initUseMvTableColumnMap(List parameters) { + Map tempUseMvTableColumnMap = new HashMap<>(); + if (!isUseMv) { + return tempUseMvTableColumnMap; + } + if (parameters.size() % 2 == 1) { + this.setStatus(HintStatus.SYNTAX_ERROR); + this.setErrorMessage("parameter of use_mv hint must be in pairs"); + return tempUseMvTableColumnMap; + } + for (int i = 0; i < parameters.size(); i += 2) { + String tableName = parameters.get(i); + String columnName = parameters.get(i + 1); + if (tempUseMvTableColumnMap.containsKey(tableName)) { + this.setStatus(HintStatus.SYNTAX_ERROR); + this.setErrorMessage("use_mv hint should only have one mv in one table: " + + tableName + "." + columnName); + break; + } + tempUseMvTableColumnMap.put(tableName, columnName); + } + return tempUseMvTableColumnMap; + } + + private Map> initNoUseMvTableColumnMap(List parameters) { + Map> tempNoUseMvTableColumnMap = new HashMap<>(); + if (isUseMv) { + return tempNoUseMvTableColumnMap; + } + if (parameters.size() % 2 == 1) { + this.setStatus(HintStatus.SYNTAX_ERROR); + this.setErrorMessage("parameter of no_use_mv hint must be in pairs"); + return tempNoUseMvTableColumnMap; + } + for (int i = 0; i < parameters.size(); i += 2) { + String tableName = parameters.get(i); + String columnName = parameters.get(i + 1); + if (tempNoUseMvTableColumnMap.containsKey(tableName)) { + tempNoUseMvTableColumnMap.get(tableName).add(columnName); + } else { + List list = new ArrayList<>(); + list.add(columnName); + tempNoUseMvTableColumnMap.put(tableName, list); + } + } + return tempNoUseMvTableColumnMap; + } + + public boolean isUseMv() { + return isUseMv; + } + + public boolean isAllMv() { + return isAllMv; + } + + public String getUseMvName(String tableName) { + return useMvTableColumnMap.get(tableName); + } + + public List getNoUseMVName(String tableName) { + return noUseMvTableColumnMap.get(tableName); + } + + @Override + public String getExplainString() { + StringBuilder out = new StringBuilder(); + if (isUseMv) { + out.append("use_mv"); + } else { + out.append("no_use_mv"); + } + if (!parameters.isEmpty()) { + out.append("("); + for (int i = 0; i < parameters.size(); i++) { + if (i % 2 == 0) { + out.append(parameters.get(i)); + } else { + out.append("."); + out.append(parameters.get(i)); + out.append(" "); + } + } + out.append(")"); + } + + return out.toString(); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index 7369b714f7439c1..324ab808226930b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -232,6 +232,7 @@ import org.apache.doris.nereids.properties.SelectHintOrdered; import org.apache.doris.nereids.properties.SelectHintSetVar; import org.apache.doris.nereids.properties.SelectHintUseCboRule; +import org.apache.doris.nereids.properties.SelectHintUseMv; import org.apache.doris.nereids.trees.TableSample; import org.apache.doris.nereids.trees.expressions.Add; import org.apache.doris.nereids.trees.expressions.And; @@ -3165,7 +3166,7 @@ private LogicalPlan withSelectHint(LogicalPlan logicalPlan, List hints = Maps.newLinkedHashMap(); + ImmutableList.Builder hints = ImmutableList.builder(); for (ParserRuleContext hintContext : hintContexts) { SelectHintContext selectHintContext = (SelectHintContext) hintContext; for (HintStatementContext hintStatement : selectHintContext.hintStatements) { @@ -3187,7 +3188,7 @@ private LogicalPlan withSelectHint(LogicalPlan logicalPlan, List leadingParameters = new ArrayList<>(); @@ -3197,10 +3198,10 @@ private LogicalPlan withSelectHint(LogicalPlan logicalPlan, List useRuleParameters = new ArrayList<>(); @@ -3210,7 +3211,7 @@ private LogicalPlan withSelectHint(LogicalPlan logicalPlan, List noUseRuleParameters = new ArrayList<>(); @@ -3220,14 +3221,34 @@ private LogicalPlan withSelectHint(LogicalPlan logicalPlan, List useIndexParameters = new ArrayList(); + for (HintAssignmentContext kv : hintStatement.parameters) { + String parameterName = visitIdentifierOrText(kv.key); + if (kv.key != null) { + useIndexParameters.add(parameterName); + } + } + hints.add(new SelectHintUseMv(hintName, useIndexParameters, true)); + break; + case "no_use_mv": + List noUseIndexParameters = new ArrayList(); + for (HintAssignmentContext kv : hintStatement.parameters) { + String parameterName = visitIdentifierOrText(kv.key); + if (kv.key != null) { + noUseIndexParameters.add(parameterName); + } + } + hints.add(new SelectHintUseMv(hintName, noUseIndexParameters, false)); break; default: break; } } } - return new LogicalSelectHint<>(hints, logicalPlan); + return new LogicalSelectHint<>(hints.build(), logicalPlan); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/pre/PullUpSubqueryAliasToCTE.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/pre/PullUpSubqueryAliasToCTE.java index 8e8889f5e62df25..31a205d5ed50542 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/pre/PullUpSubqueryAliasToCTE.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/pre/PullUpSubqueryAliasToCTE.java @@ -59,7 +59,7 @@ public Plan visitUnboundResultSink(UnboundResultSink unboundResu public Plan visitLogicalSubQueryAlias(LogicalSubQueryAlias alias, StatementContext context) { if (alias.child() instanceof LogicalSelectHint - && ((LogicalSelectHint) alias.child()).isIncludeLeading()) { + && ((LogicalSelectHint) alias.child()).isIncludeHint("Leading")) { aliasQueries.add((LogicalSubQueryAlias) alias); List tableName = new ArrayList<>(); tableName.add(alias.getAlias()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/SelectHintUseMv.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/SelectHintUseMv.java new file mode 100644 index 000000000000000..35ce25fb4f47c6d --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/SelectHintUseMv.java @@ -0,0 +1,53 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.properties; + +import java.util.List; + +/** + * select hint UseMv. + */ +public class SelectHintUseMv extends SelectHint { + private final List parameters; + + private final boolean isUseMv; + + public SelectHintUseMv(String hintName, List parameters, boolean isUseMv) { + super(hintName); + this.parameters = parameters; + this.isUseMv = isUseMv; + } + + public List getParameters() { + return parameters; + } + + public boolean isUseMv() { + return isUseMv; + } + + @Override + public String getHintName() { + return super.getHintName(); + } + + @Override + public String toString() { + return super.getHintName(); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java index be4d8b390c9f1f1..e63d4d77bca26e7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java @@ -88,6 +88,7 @@ import org.apache.doris.nereids.rules.rewrite.ConvertOuterJoinToAntiJoin; import org.apache.doris.nereids.rules.rewrite.CreatePartitionTopNFromWindow; import org.apache.doris.nereids.rules.rewrite.EliminateOuterJoin; +import org.apache.doris.nereids.rules.rewrite.MaxMinFilterPushDown; import org.apache.doris.nereids.rules.rewrite.MergeFilters; import org.apache.doris.nereids.rules.rewrite.MergeGenerates; import org.apache.doris.nereids.rules.rewrite.MergeLimits; @@ -132,6 +133,7 @@ public class RuleSet { .build(); public static final List PUSH_DOWN_FILTERS = ImmutableList.of( + new MaxMinFilterPushDown(), new CreatePartitionTopNFromWindow(), new PushDownFilterThroughProject(), new PushDownFilterThroughSort(), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java index f2c572f7779e916..ca26ab1d9f843c5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java @@ -176,6 +176,7 @@ public enum RuleType { PUSH_DOWN_FILTER_THROUGH_CTE(RuleTypeClass.REWRITE), PUSH_DOWN_FILTER_THROUGH_CTE_ANCHOR(RuleTypeClass.REWRITE), + MAX_MIN_FILTER_PUSH_DOWN(RuleTypeClass.REWRITE), PUSH_DOWN_DISTINCT_THROUGH_JOIN(RuleTypeClass.REWRITE), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/EliminateLogicalSelectHint.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/EliminateLogicalSelectHint.java index ea2c99946065025..ebff9f838a447e1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/EliminateLogicalSelectHint.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/EliminateLogicalSelectHint.java @@ -27,10 +27,12 @@ import org.apache.doris.nereids.hint.LeadingHint; import org.apache.doris.nereids.hint.OrderedHint; import org.apache.doris.nereids.hint.UseCboRuleHint; +import org.apache.doris.nereids.hint.UseMvHint; import org.apache.doris.nereids.properties.SelectHint; import org.apache.doris.nereids.properties.SelectHintLeading; import org.apache.doris.nereids.properties.SelectHintSetVar; import org.apache.doris.nereids.properties.SelectHintUseCboRule; +import org.apache.doris.nereids.properties.SelectHintUseMv; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory; @@ -43,7 +45,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.Map; import java.util.Map.Entry; import java.util.Optional; @@ -57,10 +58,10 @@ public class EliminateLogicalSelectHint extends OneRewriteRuleFactory { public Rule build() { return logicalSelectHint().thenApply(ctx -> { LogicalSelectHint selectHintPlan = ctx.root; - for (Entry hint : selectHintPlan.getHints().entrySet()) { - String hintName = hint.getKey(); + for (SelectHint hint : selectHintPlan.getHints()) { + String hintName = hint.getHintName(); if (hintName.equalsIgnoreCase("SET_VAR")) { - setVar((SelectHintSetVar) hint.getValue(), ctx.statementContext); + setVar((SelectHintSetVar) hint, ctx.statementContext); } else if (hintName.equalsIgnoreCase("ORDERED")) { try { ctx.cascadesContext.getConnectContext().getSessionVariable() @@ -73,12 +74,16 @@ public Rule build() { ctx.cascadesContext.getHintMap().put("Ordered", ordered); ctx.statementContext.addHint(ordered); } else if (hintName.equalsIgnoreCase("LEADING")) { - extractLeading((SelectHintLeading) hint.getValue(), ctx.cascadesContext, - ctx.statementContext, selectHintPlan.getHints()); + extractLeading((SelectHintLeading) hint, ctx.cascadesContext, + ctx.statementContext, selectHintPlan); } else if (hintName.equalsIgnoreCase("USE_CBO_RULE")) { - extractRule((SelectHintUseCboRule) hint.getValue(), ctx.statementContext); + extractRule((SelectHintUseCboRule) hint, ctx.statementContext); + } else if (hintName.equalsIgnoreCase("USE_MV")) { + extractMv((SelectHintUseMv) hint, ConnectContext.get().getStatementContext()); + } else if (hintName.equalsIgnoreCase("NO_USE_MV")) { + extractMv((SelectHintUseMv) hint, ConnectContext.get().getStatementContext()); } else { - logger.warn("Can not process select hint '{}' and skip it", hint.getKey()); + logger.warn("Can not process select hint '{}' and skip it", hint.getHintName()); } } return selectHintPlan.child(); @@ -116,7 +121,7 @@ private void setVar(SelectHintSetVar selectHint, StatementContext context) { } private void extractLeading(SelectHintLeading selectHint, CascadesContext context, - StatementContext statementContext, Map hints) { + StatementContext statementContext, LogicalSelectHint selectHintPlan) { LeadingHint hint = new LeadingHint("Leading", selectHint.getParameters(), selectHint.toString()); if (context.getHintMap().get("Leading") != null) { hint.setStatus(Hint.HintStatus.SYNTAX_ERROR); @@ -139,7 +144,8 @@ private void extractLeading(SelectHintLeading selectHint, CascadesContext contex if (!hint.isSyntaxError()) { hint.setStatus(Hint.HintStatus.SUCCESS); } - if (hints.get("ordered") != null || ConnectContext.get().getSessionVariable().isDisableJoinReorder() + if (selectHintPlan.isIncludeHint("Ordered") + || ConnectContext.get().getSessionVariable().isDisableJoinReorder() || context.isLeadingDisableJoinReorder()) { context.setLeadingJoin(false); hint.setStatus(Hint.HintStatus.UNUSED); @@ -158,4 +164,22 @@ private void extractRule(SelectHintUseCboRule selectHint, StatementContext state } } + private void extractMv(SelectHintUseMv selectHint, StatementContext statementContext) { + boolean isAllMv = selectHint.getParameters().isEmpty(); + UseMvHint useMvHint = new UseMvHint(selectHint.getHintName(), selectHint.getParameters(), + selectHint.isUseMv(), isAllMv); + for (Hint hint : statementContext.getHints()) { + if (hint.getHintName().equals(selectHint.getHintName())) { + hint.setStatus(Hint.HintStatus.SYNTAX_ERROR); + hint.setErrorMessage("only one " + selectHint.getHintName() + " hint is allowed"); + useMvHint.setStatus(Hint.HintStatus.SYNTAX_ERROR); + useMvHint.setErrorMessage("only one " + selectHint.getHintName() + " hint is allowed"); + } + } + if (!useMvHint.isSyntaxError()) { + ConnectContext.get().getSessionVariable().setEnableSyncMvCostBasedRewrite(false); + } + statementContext.addHint(useMvHint); + } + } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MaxMinFilterPushDown.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MaxMinFilterPushDown.java new file mode 100644 index 000000000000000..a54c3785b35a723 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MaxMinFilterPushDown.java @@ -0,0 +1,133 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.rewrite; + +import org.apache.doris.nereids.annotation.DependsRules; +import org.apache.doris.nereids.rules.Rule; +import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.rules.expression.ExpressionRewrite; +import org.apache.doris.nereids.trees.expressions.Alias; +import org.apache.doris.nereids.trees.expressions.ExprId; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.GreaterThan; +import org.apache.doris.nereids.trees.expressions.GreaterThanEqual; +import org.apache.doris.nereids.trees.expressions.LessThan; +import org.apache.doris.nereids.trees.expressions.LessThanEqual; +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction; +import org.apache.doris.nereids.trees.expressions.functions.agg.Max; +import org.apache.doris.nereids.trees.expressions.functions.agg.Min; +import org.apache.doris.nereids.trees.expressions.literal.Literal; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; +import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; +import org.apache.doris.nereids.util.ExpressionUtils; +import org.apache.doris.nereids.util.PlanUtils; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; + +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; + +/** + * select id, max(a) from t group by id having max(a)>10; + * -> + * select id, max(a) from t where a>10 group by id; + * select id, min(a) from t group by id having min(a)<10; + * -> + * select id, min(a) from t where a<10 group by id; + */ +@DependsRules({ + ExpressionRewrite.class +}) +public class MaxMinFilterPushDown extends OneRewriteRuleFactory { + @Override + public Rule build() { + return logicalFilter(logicalAggregate().whenNot(agg -> agg.getGroupByExpressions().isEmpty())) + .then(this::pushDownMaxMinFilter) + .toRule(RuleType.MAX_MIN_FILTER_PUSH_DOWN); + } + + private Plan pushDownMaxMinFilter(LogicalFilter> filter) { + Set conjuncts = filter.getConjuncts(); + LogicalAggregate agg = filter.child(); + Plan aggChild = agg.child(); + List aggOutputExpressions = agg.getOutputExpressions(); + Set aggFuncs = ExpressionUtils.collect(aggOutputExpressions, + expr -> expr instanceof AggregateFunction); + Set maxMinFunc = ExpressionUtils.collect(aggFuncs, + expr -> expr instanceof Max || expr instanceof Min); + // LogicalAggregate only outputs one aggregate function, which is max or min + if (aggFuncs.size() != 1 || maxMinFunc.size() != 1) { + return null; + } + ExprId exprId = null; + Expression func = maxMinFunc.iterator().next(); + for (NamedExpression expr : aggOutputExpressions) { + if (expr instanceof Alias && ((Alias) expr).child().equals(func)) { + Alias alias = (Alias) expr; + exprId = alias.getExprId(); + } + } + // try to find min(a)<10 or max(a)>10 + Expression originConjunct = findMatchingConjunct(conjuncts, func instanceof Max, exprId).orElse(null); + if (null == originConjunct) { + return null; + } + Set newUpperConjuncts = new HashSet<>(conjuncts); + newUpperConjuncts.remove(originConjunct); + Expression newPredicate = null; + if (func instanceof Max) { + if (originConjunct instanceof GreaterThan) { + newPredicate = new GreaterThan(func.child(0), originConjunct.child(1)); + } else if (originConjunct instanceof GreaterThanEqual) { + newPredicate = new GreaterThanEqual(func.child(0), originConjunct.child(1)); + } + } else { + if (originConjunct instanceof LessThan) { + newPredicate = new LessThan(func.child(0), originConjunct.child(1)); + } else if (originConjunct instanceof LessThanEqual) { + newPredicate = new LessThanEqual(func.child(0), originConjunct.child(1)); + } + } + Preconditions.checkState(newPredicate != null, "newPredicate is null"); + LogicalFilter newPushDownFilter = new LogicalFilter<>(ImmutableSet.of(newPredicate), aggChild); + LogicalAggregate newAgg = agg.withChildren(ImmutableList.of(newPushDownFilter)); + return PlanUtils.filterOrSelf(newUpperConjuncts, newAgg); + } + + private Optional findMatchingConjunct(Set conjuncts, boolean isMax, ExprId exprId) { + for (Expression conjunct : conjuncts) { + if ((isMax && (conjunct instanceof GreaterThan || conjunct instanceof GreaterThanEqual)) + || (!isMax && (conjunct instanceof LessThan || conjunct instanceof LessThanEqual))) { + if (conjunct.child(0) instanceof SlotReference && conjunct.child(1) instanceof Literal) { + SlotReference slot = (SlotReference) conjunct.child(0); + if (slot.getExprId().equals(exprId)) { + return Optional.of(conjunct); + } + } + } + } + return Optional.empty(); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/AbstractSelectMaterializedIndexRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/AbstractSelectMaterializedIndexRule.java index 1124c141416f3f4..f17ab1c96bd9cfd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/AbstractSelectMaterializedIndexRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/AbstractSelectMaterializedIndexRule.java @@ -276,7 +276,7 @@ protected static long selectBestIndex( .thenComparing(rid -> (Long) rid)) .collect(Collectors.toList()); - return sortedIndexIds.get(0); + return table.getBestMvIdWithHint(sortedIndexIds); } protected static List matchPrefixMost( diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java index 3bc2a880da7c6ab..a65a07fea30bdfc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java @@ -607,11 +607,13 @@ private Statistics estimateBinaryComparisonFilter(Expression leftExpr, DataType .setMaxExpr(intersectRange.getHighExpr()) .setNdv(intersectRange.getDistinctValues()) .setNumNulls(0); - double sel = leftRange.overlapPercentWith(rightRange); + double sel = leftRange.getDistinctValues() == 0 + ? 1.0 + : intersectRange.getDistinctValues() / leftRange.getDistinctValues(); if (!(dataType instanceof RangeScalable) && (sel != 0.0 && sel != 1.0)) { sel = DEFAULT_INEQUALITY_COEFFICIENT; - } else if (sel < RANGE_SELECTIVITY_THRESHOLD) { - sel = RANGE_SELECTIVITY_THRESHOLD; + } else { + sel = Math.max(sel, RANGE_SELECTIVITY_THRESHOLD); } sel = getNotNullSelectivity(leftStats, sel); updatedStatistics = context.statistics.withSel(sel); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Acos.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Acos.java index c99af81123fc633..2193221c3263639 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Acos.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Acos.java @@ -19,8 +19,8 @@ import org.apache.doris.catalog.FunctionSignature; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; -import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable; import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.DoubleType; @@ -34,7 +34,7 @@ * ScalarFunction 'acos'. This class is generated by GenerateFunction. */ public class Acos extends ScalarFunction - implements UnaryExpression, ExplicitlyCastableSignature, PropagateNullable { + implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNullable { public static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(DoubleType.INSTANCE).args(DoubleType.INSTANCE) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/AesDecrypt.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/AesDecrypt.java index 5e7d760b0810a01..7608cf4e40ea35d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/AesDecrypt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/AesDecrypt.java @@ -18,7 +18,6 @@ package org.apache.doris.nereids.trees.expressions.functions.scalar; import org.apache.doris.catalog.FunctionSignature; -import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.literal.StringLiteral; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; @@ -58,16 +57,7 @@ public class AesDecrypt extends AesCryptoFunction { * AesDecrypt */ public AesDecrypt(Expression arg0, Expression arg1) { - // if there are only 2 params, we need set encryption mode to AES_128_ECB - // this keeps the behavior consistent with old doris ver. - super("aes_decrypt", arg0, arg1, new StringLiteral("AES_128_ECB")); - - // check if encryptionMode from session variables is valid - StringLiteral encryptionMode = CryptoFunction.getDefaultBlockEncryptionMode("AES_128_ECB"); - if (!AES_MODES.contains(encryptionMode.getValue())) { - throw new AnalysisException( - "session variable block_encryption_mode is invalid with aes"); - } + super("aes_decrypt", arg0, arg1, new StringLiteral(""), getDefaultBlockEncryptionMode()); } public AesDecrypt(Expression arg0, Expression arg1, Expression arg2) { @@ -89,7 +79,7 @@ public AesDecrypt withChildren(List children) { } else if (children().size() == 3) { return new AesDecrypt(children.get(0), children.get(1), children.get(2)); } else { - return new AesDecrypt(children.get(0), children.get(1), children.get(2), (StringLiteral) children.get(3)); + return new AesDecrypt(children.get(0), children.get(1), children.get(2), children.get(3)); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/AesDecryptV2.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/AesDecryptV2.java deleted file mode 100644 index 3a5a8121cff1193..000000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/AesDecryptV2.java +++ /dev/null @@ -1,74 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.trees.expressions.functions.scalar; - -import org.apache.doris.nereids.exceptions.AnalysisException; -import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.expressions.literal.StringLiteral; -import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; - -import com.google.common.base.Preconditions; - -import java.util.List; - -/** - * ScalarFunction 'aes_decrypt'. This class is generated by GenerateFunction. - */ -public class AesDecryptV2 extends AesDecrypt { - - /** - * AesDecryptV2 - */ - public AesDecryptV2(Expression arg0, Expression arg1) { - super(arg0, arg1, getDefaultBlockEncryptionMode()); - String blockEncryptionMode = String.valueOf(getDefaultBlockEncryptionMode()); - if (!blockEncryptionMode.toUpperCase().equals("'AES_128_ECB'") - && !blockEncryptionMode.toUpperCase().equals("'AES_192_ECB'") - && !blockEncryptionMode.toUpperCase().equals("'AES_256_ECB'")) { - throw new AnalysisException("Incorrect parameter count in the call to native function 'aes_decrypt'"); - } - } - - public AesDecryptV2(Expression arg0, Expression arg1, Expression arg2) { - super(arg0, arg1, arg2); - } - - public AesDecryptV2(Expression arg0, Expression arg1, Expression arg2, Expression arg3) { - super(arg0, arg1, arg2, arg3); - } - - /** - * withChildren. - */ - @Override - public AesDecryptV2 withChildren(List children) { - Preconditions.checkArgument(children.size() >= 2 && children.size() <= 4); - if (children.size() == 2) { - return new AesDecryptV2(children.get(0), children.get(1)); - } else if (children().size() == 3) { - return new AesDecryptV2(children.get(0), children.get(1), children.get(2)); - } else { - return new AesDecryptV2(children.get(0), children.get(1), children.get(2), (StringLiteral) children.get(3)); - } - } - - @Override - public R accept(ExpressionVisitor visitor, C context) { - return visitor.visitAesDecryptV2(this, context); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/AesEncrypt.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/AesEncrypt.java index ef99bdbe21db601..455d6b0dbd56454 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/AesEncrypt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/AesEncrypt.java @@ -18,7 +18,6 @@ package org.apache.doris.nereids.trees.expressions.functions.scalar; import org.apache.doris.catalog.FunctionSignature; -import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.literal.StringLiteral; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; @@ -58,16 +57,7 @@ public class AesEncrypt extends AesCryptoFunction { * Some javadoc for checkstyle... */ public AesEncrypt(Expression arg0, Expression arg1) { - // if there are only 2 params, we need set encryption mode to AES_128_ECB - // this keeps the behavior consistent with old doris ver. - super("aes_encrypt", arg0, arg1, new StringLiteral("AES_128_ECB")); - - // check if encryptionMode from session variables is valid - StringLiteral encryptionMode = CryptoFunction.getDefaultBlockEncryptionMode("AES_128_ECB"); - if (!AES_MODES.contains(encryptionMode.getValue())) { - throw new AnalysisException( - "session variable block_encryption_mode is invalid with aes"); - } + super("aes_encrypt", arg0, arg1, new StringLiteral(""), getDefaultBlockEncryptionMode()); } public AesEncrypt(Expression arg0, Expression arg1, Expression arg2) { @@ -89,7 +79,7 @@ public AesEncrypt withChildren(List children) { } else if (children().size() == 3) { return new AesEncrypt(children.get(0), children.get(1), children.get(2)); } else { - return new AesEncrypt(children.get(0), children.get(1), children.get(2), (StringLiteral) children.get(3)); + return new AesEncrypt(children.get(0), children.get(1), children.get(2), children.get(3)); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/AesEncryptV2.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/AesEncryptV2.java deleted file mode 100644 index 08e1d1e915e52ec..000000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/AesEncryptV2.java +++ /dev/null @@ -1,74 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.trees.expressions.functions.scalar; - -import org.apache.doris.nereids.exceptions.AnalysisException; -import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.expressions.literal.StringLiteral; -import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; - -import com.google.common.base.Preconditions; - -import java.util.List; - -/** - * ScalarFunction 'aes_encrypt'. This class is generated by GenerateFunction. - */ -public class AesEncryptV2 extends AesEncrypt { - - /** - * AesEncryptV2 - */ - public AesEncryptV2(Expression arg0, Expression arg1) { - super(arg0, arg1, getDefaultBlockEncryptionMode()); - String blockEncryptionMode = String.valueOf(getDefaultBlockEncryptionMode()); - if (!blockEncryptionMode.toUpperCase().equals("'AES_128_ECB'") - && !blockEncryptionMode.toUpperCase().equals("'AES_192_ECB'") - && !blockEncryptionMode.toUpperCase().equals("'AES_256_ECB'")) { - throw new AnalysisException("Incorrect parameter count in the call to native function 'aes_encrypt'"); - } - } - - public AesEncryptV2(Expression arg0, Expression arg1, Expression arg2) { - super(arg0, arg1, arg2); - } - - public AesEncryptV2(Expression arg0, Expression arg1, Expression arg2, Expression arg3) { - super(arg0, arg1, arg2, arg3); - } - - /** - * withChildren. - */ - @Override - public AesEncryptV2 withChildren(List children) { - Preconditions.checkArgument(children.size() >= 2 && children.size() <= 4); - if (children.size() == 2) { - return new AesEncryptV2(children.get(0), children.get(1)); - } else if (children().size() == 3) { - return new AesEncryptV2(children.get(0), children.get(1), children.get(2)); - } else { - return new AesEncryptV2(children.get(0), children.get(1), children.get(2), (StringLiteral) children.get(3)); - } - } - - @Override - public R accept(ExpressionVisitor visitor, C context) { - return visitor.visitAesEncryptV2(this, context); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Asin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Asin.java index 0e06d8d77edb10a..22e1ff59b7df28a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Asin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Asin.java @@ -19,8 +19,8 @@ import org.apache.doris.catalog.FunctionSignature; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; -import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable; import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.DoubleType; @@ -34,7 +34,7 @@ * ScalarFunction 'asin'. This class is generated by GenerateFunction. */ public class Asin extends ScalarFunction - implements UnaryExpression, ExplicitlyCastableSignature, PropagateNullable { + implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNullable { public static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(DoubleType.INSTANCE).args(DoubleType.INSTANCE) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Dsqrt.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Dsqrt.java index 874befd09dba4d4..3caef79776b3bba 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Dsqrt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Dsqrt.java @@ -19,8 +19,8 @@ import org.apache.doris.catalog.FunctionSignature; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; -import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable; import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.DoubleType; @@ -34,7 +34,7 @@ * ScalarFunction 'dsqrt'. This class is generated by GenerateFunction. */ public class Dsqrt extends ScalarFunction - implements UnaryExpression, ExplicitlyCastableSignature, PropagateNullable { + implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNullable { public static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(DoubleType.INSTANCE).args(DoubleType.INSTANCE) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Sm4Decrypt.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Sm4Decrypt.java index 6e0f2bb48de1f00..c87c2a42c16e5aa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Sm4Decrypt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Sm4Decrypt.java @@ -18,7 +18,6 @@ package org.apache.doris.nereids.trees.expressions.functions.scalar; import org.apache.doris.catalog.FunctionSignature; -import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.literal.StringLiteral; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; @@ -63,17 +62,7 @@ public class Sm4Decrypt extends Sm4CryptoFunction { * constructor with 2 arguments. */ public Sm4Decrypt(Expression arg0, Expression arg1) { - // if there are only 2 params, we need add an empty string as the third param - // and set encryption mode to SM4_128_ECB - // this keeps the behavior consistent with old doris ver. - super("sm4_decrypt", arg0, arg1, new StringLiteral(""), new StringLiteral("SM4_128_ECB")); - - // check if encryptionMode from session variables is valid - StringLiteral encryptionMode = CryptoFunction.getDefaultBlockEncryptionMode("SM4_128_ECB"); - if (!SM4_MODES.contains(encryptionMode.getValue())) { - throw new AnalysisException( - "session variable block_encryption_mode is invalid with sm4"); - } + super("sm4_decrypt", arg0, arg1, new StringLiteral(""), getDefaultBlockEncryptionMode()); } /** @@ -98,7 +87,7 @@ public Sm4Decrypt withChildren(List children) { } else if (children().size() == 3) { return new Sm4Decrypt(children.get(0), children.get(1), children.get(2)); } else { - return new Sm4Decrypt(children.get(0), children.get(1), children.get(2), (StringLiteral) children.get(3)); + return new Sm4Decrypt(children.get(0), children.get(1), children.get(2), children.get(3)); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Sm4DecryptV2.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Sm4DecryptV2.java deleted file mode 100644 index 4c743a829450685..000000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Sm4DecryptV2.java +++ /dev/null @@ -1,69 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.trees.expressions.functions.scalar; - -import org.apache.doris.nereids.exceptions.AnalysisException; -import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.expressions.literal.StringLiteral; -import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; - -import com.google.common.base.Preconditions; - -import java.util.List; - -/** - * ScalarFunction 'sm4_decrypt'. This class is generated by GenerateFunction. - */ -public class Sm4DecryptV2 extends Sm4Decrypt { - - /** - * Sm4DecryptV2 - */ - public Sm4DecryptV2(Expression arg0, Expression arg1) { - super(arg0, arg1); - throw new AnalysisException("Incorrect parameter count in the call to native function 'sm4_decrypt'"); - } - - public Sm4DecryptV2(Expression arg0, Expression arg1, Expression arg2) { - super(arg0, arg1, arg2); - } - - public Sm4DecryptV2(Expression arg0, Expression arg1, Expression arg2, Expression arg3) { - super(arg0, arg1, arg2, arg3); - } - - /** - * withChildren. - */ - @Override - public Sm4DecryptV2 withChildren(List children) { - Preconditions.checkArgument(children.size() >= 2 && children.size() <= 4); - if (children.size() == 2) { - return new Sm4DecryptV2(children.get(0), children.get(1)); - } else if (children().size() == 3) { - return new Sm4DecryptV2(children.get(0), children.get(1), children.get(2)); - } else { - return new Sm4DecryptV2(children.get(0), children.get(1), children.get(2), (StringLiteral) children.get(3)); - } - } - - @Override - public R accept(ExpressionVisitor visitor, C context) { - return visitor.visitSm4DecryptV2(this, context); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Sm4Encrypt.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Sm4Encrypt.java index 00525558d419825..dbf5c61db07de5b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Sm4Encrypt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Sm4Encrypt.java @@ -18,7 +18,6 @@ package org.apache.doris.nereids.trees.expressions.functions.scalar; import org.apache.doris.catalog.FunctionSignature; -import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.literal.StringLiteral; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; @@ -58,17 +57,7 @@ public class Sm4Encrypt extends Sm4CryptoFunction { * constructor with 2 arguments. */ public Sm4Encrypt(Expression arg0, Expression arg1) { - // if there are only 2 params, we need add an empty string as the third param - // and set encryption mode to SM4_128_ECB - // this keeps the behavior consistent with old doris ver. - super("sm4_encrypt", arg0, arg1, new StringLiteral(""), new StringLiteral("SM4_128_ECB")); - - // check if encryptionMode from session variables is valid - StringLiteral encryptionMode = CryptoFunction.getDefaultBlockEncryptionMode("SM4_128_ECB"); - if (!SM4_MODES.contains(encryptionMode.getValue())) { - throw new AnalysisException( - "session variable block_encryption_mode is invalid with sm4"); - } + super("sm4_encrypt", arg0, arg1, new StringLiteral(""), getDefaultBlockEncryptionMode()); } /** @@ -93,7 +82,7 @@ public Sm4Encrypt withChildren(List children) { } else if (children().size() == 3) { return new Sm4Encrypt(children.get(0), children.get(1), children.get(2)); } else { - return new Sm4Encrypt(children.get(0), children.get(1), children.get(2), (StringLiteral) children.get(3)); + return new Sm4Encrypt(children.get(0), children.get(1), children.get(2), children.get(3)); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Sm4EncryptV2.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Sm4EncryptV2.java deleted file mode 100644 index 7c895fc30905966..000000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Sm4EncryptV2.java +++ /dev/null @@ -1,72 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.trees.expressions.functions.scalar; - -import org.apache.doris.nereids.exceptions.AnalysisException; -import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.expressions.literal.StringLiteral; -import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; - -import com.google.common.base.Preconditions; - -import java.util.List; - -/** - * ScalarFunction 'sm4_encrypt'. This class is generated by GenerateFunction. - */ -public class Sm4EncryptV2 extends Sm4Encrypt { - - /** - * constructor with 2 arguments. - */ - public Sm4EncryptV2(Expression arg0, Expression arg1) { - super(arg0, arg1); - throw new AnalysisException("Incorrect parameter count in the call to native function 'sm4_encrypt'"); - } - - /** - * constructor with 3 arguments. - */ - public Sm4EncryptV2(Expression arg0, Expression arg1, Expression arg2) { - super(arg0, arg1, arg2); - } - - public Sm4EncryptV2(Expression arg0, Expression arg1, Expression arg2, Expression arg3) { - super(arg0, arg1, arg2, arg3); - } - - /** - * withChildren. - */ - @Override - public Sm4EncryptV2 withChildren(List children) { - Preconditions.checkArgument(children.size() >= 2 && children.size() <= 4); - if (children.size() == 2) { - return new Sm4EncryptV2(children.get(0), children.get(1)); - } else if (children().size() == 3) { - return new Sm4EncryptV2(children.get(0), children.get(1), children.get(2)); - } else { - return new Sm4EncryptV2(children.get(0), children.get(1), children.get(2), (StringLiteral) children.get(3)); - } - } - - @Override - public R accept(ExpressionVisitor visitor, C context) { - return visitor.visitSm4EncryptV2(this, context); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Sqrt.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Sqrt.java index 495321c6dfa8c53..f954eb07a54083f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Sqrt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Sqrt.java @@ -19,8 +19,8 @@ import org.apache.doris.catalog.FunctionSignature; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; -import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable; import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.DoubleType; @@ -34,7 +34,7 @@ * ScalarFunction 'sqrt'. This class is generated by GenerateFunction. */ public class Sqrt extends ScalarFunction - implements UnaryExpression, ExplicitlyCastableSignature, PropagateNullable { + implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNullable { public static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(DoubleType.INSTANCE).args(DoubleType.INSTANCE) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java index 5d47094f247e8cb..20772ae716be2f8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java @@ -24,9 +24,7 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.Abs; import org.apache.doris.nereids.trees.expressions.functions.scalar.Acos; import org.apache.doris.nereids.trees.expressions.functions.scalar.AesDecrypt; -import org.apache.doris.nereids.trees.expressions.functions.scalar.AesDecryptV2; import org.apache.doris.nereids.trees.expressions.functions.scalar.AesEncrypt; -import org.apache.doris.nereids.trees.expressions.functions.scalar.AesEncryptV2; import org.apache.doris.nereids.trees.expressions.functions.scalar.AppendTrailingCharIfAbsent; import org.apache.doris.nereids.trees.expressions.functions.scalar.Array; import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayApply; @@ -368,9 +366,7 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.Sm3; import org.apache.doris.nereids.trees.expressions.functions.scalar.Sm3sum; import org.apache.doris.nereids.trees.expressions.functions.scalar.Sm4Decrypt; -import org.apache.doris.nereids.trees.expressions.functions.scalar.Sm4DecryptV2; import org.apache.doris.nereids.trees.expressions.functions.scalar.Sm4Encrypt; -import org.apache.doris.nereids.trees.expressions.functions.scalar.Sm4EncryptV2; import org.apache.doris.nereids.trees.expressions.functions.scalar.Space; import org.apache.doris.nereids.trees.expressions.functions.scalar.SplitByChar; import org.apache.doris.nereids.trees.expressions.functions.scalar.SplitByRegexp; @@ -480,18 +476,10 @@ default R visitAesDecrypt(AesDecrypt aesDecrypt, C context) { return visitScalarFunction(aesDecrypt, context); } - default R visitAesDecryptV2(AesDecryptV2 aesDecryptV2, C context) { - return visitScalarFunction(aesDecryptV2, context); - } - default R visitAesEncrypt(AesEncrypt aesEncrypt, C context) { return visitScalarFunction(aesEncrypt, context); } - default R visitAesEncryptV2(AesEncryptV2 aesEncryptV2, C context) { - return visitScalarFunction(aesEncryptV2, context); - } - default R visitAppendTrailingCharIfAbsent(AppendTrailingCharIfAbsent function, C context) { return visitScalarFunction(function, context); } @@ -1836,18 +1824,10 @@ default R visitSm4Decrypt(Sm4Decrypt sm4Decrypt, C context) { return visitScalarFunction(sm4Decrypt, context); } - default R visitSm4DecryptV2(Sm4DecryptV2 sm4DecryptV2, C context) { - return visitScalarFunction(sm4DecryptV2, context); - } - default R visitSm4Encrypt(Sm4Encrypt sm4Encrypt, C context) { return visitScalarFunction(sm4Encrypt, context); } - default R visitSm4EncryptV2(Sm4EncryptV2 sm4EncryptV2, C context) { - return visitScalarFunction(sm4EncryptV2, context); - } - default R visitSpace(Space space, C context) { return visitScalarFunction(space, context); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/OlapGroupCommitInsertExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/OlapGroupCommitInsertExecutor.java index a44528e93b379d2..239328ce93d8132 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/OlapGroupCommitInsertExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/OlapGroupCommitInsertExecutor.java @@ -120,7 +120,7 @@ protected void beforeExec() { } try { this.coordinator.setGroupCommitBe(Env.getCurrentEnv().getGroupCommitManager() - .selectBackendForGroupCommit(table.getId(), ctx, false)); + .selectBackendForGroupCommit(table.getId(), ctx)); } catch (LoadException | DdlException e) { throw new RuntimeException(e); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSelectHint.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSelectHint.java index 127889ea7ed4710..a33e2194131c8d5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSelectHint.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSelectHint.java @@ -29,10 +29,8 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; import java.util.List; -import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.stream.Collectors; @@ -44,9 +42,9 @@ public class LogicalSelectHint extends LogicalUnary implements BlockFuncDepsPropagation { - private final Map hints; + private final ImmutableList hints; - public LogicalSelectHint(Map hints, CHILD_TYPE child) { + public LogicalSelectHint(ImmutableList hints, CHILD_TYPE child) { this(hints, Optional.empty(), Optional.empty(), child); } @@ -57,19 +55,29 @@ public LogicalSelectHint(Map hints, CHILD_TYPE child) { * @param logicalProperties logicalProperties is use for compute output * @param child child plan */ - public LogicalSelectHint(Map hints, + public LogicalSelectHint(ImmutableList hints, Optional groupExpression, Optional logicalProperties, CHILD_TYPE child) { super(PlanType.LOGICAL_SELECT_HINT, groupExpression, logicalProperties, child); - this.hints = ImmutableMap.copyOf(Objects.requireNonNull(hints, "hints can not be null")); + this.hints = ImmutableList.copyOf(Objects.requireNonNull(hints, "hints can not be null")); } - public Map getHints() { + public List getHints() { return hints; } - public boolean isIncludeLeading() { - return hints.containsKey("leading"); + /** + * check if current select hint include some hint + * @param hintName hint name + * @return boolean which indicate have hint + */ + public boolean isIncludeHint(String hintName) { + for (SelectHint hint : hints) { + if (hint.getHintName().equalsIgnoreCase(hintName)) { + return true; + } + } + return false; } @Override @@ -107,9 +115,9 @@ public List computeOutput() { @Override public String toString() { - String hintStr = this.hints.entrySet() + String hintStr = this.hints .stream() - .map(entry -> entry.getValue().toString()) + .map(hint -> hint.toString()) .collect(Collectors.joining(", ")); return "LogicalSelectHint (" + hintStr + ")"; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/ModifyTablePropertyOperationLog.java b/fe/fe-core/src/main/java/org/apache/doris/persist/ModifyTablePropertyOperationLog.java index a782db9f9c069e9..50b49074ce16a3b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/ModifyTablePropertyOperationLog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/ModifyTablePropertyOperationLog.java @@ -35,6 +35,10 @@ public class ModifyTablePropertyOperationLog implements Writable { private long dbId; @SerializedName(value = "tableId") private long tableId; + @SerializedName(value = "ctlName") + private String ctlName; + @SerializedName(value = "dbName") + private String dbName; @SerializedName(value = "tableName") private String tableName; @SerializedName(value = "properties") @@ -58,6 +62,14 @@ public ModifyTablePropertyOperationLog(long dbId, long tableId, String tableName this.sql = sb.toString(); } + public ModifyTablePropertyOperationLog(String ctlName, String dbName, String tableName, + Map properties) { + this.ctlName = ctlName; + this.dbName = dbName; + this.tableName = tableName; + this.properties = properties; + } + public long getDbId() { return dbId; } @@ -66,6 +78,18 @@ public long getTableId() { return tableId; } + public String getCtlName() { + return ctlName; + } + + public String getDbName() { + return dbName; + } + + public String getTableName() { + return tableName; + } + public Map getProperties() { return properties; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/GroupCommitPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/planner/GroupCommitPlanner.java index bc3759d4e628673..234f8e99a882e87 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/GroupCommitPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/GroupCommitPlanner.java @@ -148,7 +148,7 @@ public PGroupCommitInsertResponse executeGroupCommitInsert(ConnectContext ctx, protected void selectBackends(ConnectContext ctx) throws DdlException { try { backend = Env.getCurrentEnv().getGroupCommitManager() - .selectBackendForGroupCommit(this.table.getId(), ctx, false); + .selectBackendForGroupCommit(this.table.getId(), ctx); } catch (LoadException e) { throw new DdlException("No suitable backend"); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java index 50b0f5a026909cf..a92cac7b5102600 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java @@ -739,8 +739,20 @@ public int getScanRangeNum() { } public boolean shouldUseOneInstance(ConnectContext ctx) { - long limitRowsForSingleInstance = ctx == null ? 10000 : ctx.getSessionVariable().limitRowsForSingleInstance; - return hasLimit() && getLimit() < limitRowsForSingleInstance && conjuncts.isEmpty(); + int adaptivePipelineTaskSerialReadOnLimit = 10000; + + if (ctx != null) { + if (ctx.getSessionVariable().enableAdaptivePipelineTaskSerialReadOnLimit) { + adaptivePipelineTaskSerialReadOnLimit = ctx.getSessionVariable().adaptivePipelineTaskSerialReadOnLimit; + } else { + return false; + } + } else { + // No connection context, typically for broker load. + } + + // For UniqueKey table, we will use multiple instance. + return hasLimit() && getLimit() <= adaptivePipelineTaskSerialReadOnLimit && conjuncts.isEmpty(); } // In cloud mode, meta read lock is not enough to keep a snapshot of the partition versions. diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java index 91a3dbaad947b58..8f6faf41598cc43 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java @@ -21,7 +21,6 @@ import org.apache.doris.analysis.InsertStmt; import org.apache.doris.analysis.KillStmt; import org.apache.doris.analysis.LiteralExpr; -import org.apache.doris.analysis.NotFallbackInParser; import org.apache.doris.analysis.QueryStmt; import org.apache.doris.analysis.SqlParser; import org.apache.doris.analysis.SqlScanner; @@ -300,6 +299,27 @@ public void executeQuery(MysqlCommand mysqlCommand, String originStmt) throws Ex nereidsSyntaxException = e; } } + + if (stmts == null && !ctx.getSessionVariable().enableFallbackToOriginalPlanner) { + String errMsg; + Throwable exception = null; + if (nereidsParseException != null) { + errMsg = nereidsParseException.getMessage(); + exception = nereidsParseException; + } else if (nereidsSyntaxException != null) { + errMsg = nereidsSyntaxException.getMessage(); + exception = nereidsSyntaxException; + } else { + errMsg = "Nereids parse statements failed. " + originStmt; + } + if (exception == null) { + exception = new AnalysisException(errMsg); + } else { + exception = new AnalysisException(errMsg, exception); + } + handleQueryException(exception, originStmt, null, null); + return; + } } // stmts == null when Nereids cannot planner this query or Nereids is disabled. @@ -325,31 +345,6 @@ public void executeQuery(MysqlCommand mysqlCommand, String originStmt) throws Ex } } - if (mysqlCommand == MysqlCommand.COM_QUERY - && ctx.getSessionVariable().isEnableNereidsPlanner() - && !ctx.getSessionVariable().enableFallbackToOriginalPlanner - && !stmts.isEmpty() - && stmts.stream().allMatch(s -> s instanceof NotFallbackInParser)) { - String errMsg; - Throwable exception = null; - if (nereidsParseException != null) { - errMsg = nereidsParseException.getMessage(); - exception = nereidsParseException; - } else if (nereidsSyntaxException != null) { - errMsg = nereidsSyntaxException.getMessage(); - exception = nereidsSyntaxException; - } else { - errMsg = "Nereids parse DQL failed. " + originStmt; - } - if (exception == null) { - exception = new AnalysisException(errMsg); - } else { - exception = new AnalysisException(errMsg, exception); - } - handleQueryException(exception, originStmt, null, null); - return; - } - List origSingleStmtList = null; // if stmts.size() > 1, split originStmt to multi singleStmts if (stmts.size() > 1) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/MasterOpExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/MasterOpExecutor.java index eaab01df55635af..edc174d9d2f7e48 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/MasterOpExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/MasterOpExecutor.java @@ -102,8 +102,8 @@ public void syncJournal() throws Exception { waitOnReplaying(); } - public long getGroupCommitLoadBeId(long tableId, String cluster, boolean isCloud) throws Exception { - result = forward(buildGetGroupCommitLoadBeIdParmas(tableId, cluster, isCloud)); + public long getGroupCommitLoadBeId(long tableId, String cluster) throws Exception { + result = forward(buildGetGroupCommitLoadBeIdParmas(tableId, cluster)); waitOnReplaying(); return result.groupCommitLoadBeId; } @@ -244,12 +244,11 @@ private TMasterOpRequest buildSyncJournalParmas() { return params; } - private TMasterOpRequest buildGetGroupCommitLoadBeIdParmas(long tableId, String cluster, boolean isCloud) { + private TMasterOpRequest buildGetGroupCommitLoadBeIdParmas(long tableId, String cluster) { final TGroupCommitInfo groupCommitParams = new TGroupCommitInfo(); groupCommitParams.setGetGroupCommitLoadBeId(true); groupCommitParams.setGroupCommitLoadTableId(tableId); groupCommitParams.setCluster(cluster); - groupCommitParams.setIsCloud(isCloud); final TMasterOpRequest params = new TMasterOpRequest(); // node ident diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 4140835b6570046..c1d3c7a3efb0ee4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -654,6 +654,11 @@ public class SessionVariable implements Serializable, Writable { public static final String IN_LIST_VALUE_COUNT_THRESHOLD = "in_list_value_count_threshold"; + public static final String ENABLE_ADAPTIVE_PIPELINE_TASK_SERIAL_READ_ON_LIMIT = + "enable_adaptive_pipeline_task_serial_read_on_limit"; + public static final String ADAPTIVE_PIPELINE_TASK_SERIAL_READ_ON_LIMIT = + "adaptive_pipeline_task_serial_read_on_limit"; + /** * If set false, user couldn't submit analyze SQL and FE won't allocate any related resources. */ @@ -2115,6 +2120,7 @@ public void setIgnoreShapePlanNodes(String ignoreShapePlanNodes) { }) public boolean enableFallbackOnMissingInvertedIndex = true; + @VariableMgr.VarAttr(name = IN_LIST_VALUE_COUNT_THRESHOLD, description = { "in条件value数量大于这个threshold后将不会走fast_execute", "When the number of values in the IN condition exceeds this threshold," @@ -2122,6 +2128,22 @@ public void setIgnoreShapePlanNodes(String ignoreShapePlanNodes) { }) public int inListValueCountThreshold = 10; + @VariableMgr.VarAttr(name = ENABLE_ADAPTIVE_PIPELINE_TASK_SERIAL_READ_ON_LIMIT, needForward = true, description = { + "开启后将会允许自动调整 pipeline task 的并发数。当 scan 节点没有过滤条件，且 limit 参数小于" + + "adaptive_pipeline_task_serial_read_on_limit 中指定的行数时，scan 的并行度将会被设置为 1", + "When enabled, the pipeline task concurrency will be adjusted automatically. When the scan node has no filter " + + "conditions and the limit parameter is less than the number of rows specified in " + + "adaptive_pipeline_task_serial_read_on_limit, the parallelism of the scan will be set to 1." + }) + public boolean enableAdaptivePipelineTaskSerialReadOnLimit = true; + + @VariableMgr.VarAttr(name = ADAPTIVE_PIPELINE_TASK_SERIAL_READ_ON_LIMIT, needForward = true, description = { + "当 enable_adaptive_pipeline_task_serial_read_on_limit 开启时，scan 的并行度将会被设置为 1 的行数阈值", + "When enable_adaptive_pipeline_task_serial_read_on_limit is enabled, " + + "the number of rows at which the parallelism of the scan will be set to 1." + }) + public int adaptivePipelineTaskSerialReadOnLimit = 10000; + public void setEnableEsParallelScroll(boolean enableESParallelScroll) { this.enableESParallelScroll = enableESParallelScroll; } @@ -3700,6 +3722,9 @@ public TQueryOptions toThrift() { tResult.setKeepCarriageReturn(keepCarriageReturn); tResult.setEnableSegmentCache(enableSegmentCache); + + tResult.setEnableAdaptivePipelineTaskSerialReadOnLimit(enableAdaptivePipelineTaskSerialReadOnLimit); + tResult.setAdaptivePipelineTaskSerialReadOnLimit(adaptivePipelineTaskSerialReadOnLimit); tResult.setInListValueCountThreshold(inListValueCountThreshold); return tResult; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java index 1cbfcf8ba37fa94..819c98759917cf6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java @@ -2720,7 +2720,7 @@ private void handleShowTableStats() { if (tableStats == null) { resultSet = showTableStatsStmt.constructEmptyResultSet(); } else { - resultSet = showTableStatsStmt.constructResultSet(tableStats); + resultSet = showTableStatsStmt.constructResultSet(tableStats, tableIf); } return; } @@ -2729,9 +2729,9 @@ private void handleShowTableStats() { tableStats == null means it's not analyzed, in this case show the estimated row count. */ if (tableStats == null) { - resultSet = showTableStatsStmt.constructResultSet(tableIf.getCachedRowCount()); + resultSet = showTableStatsStmt.constructResultSet(tableIf); } else { - resultSet = showTableStatsStmt.constructResultSet(tableStats); + resultSet = showTableStatsStmt.constructResultSet(tableStats, tableIf); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java index 99bcadee120e9f9..cc613cb5d8cf761 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java +++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java @@ -1041,7 +1041,7 @@ public TMasterOpResult forward(TMasterOpRequest params) throws TException { final TMasterOpResult result = new TMasterOpResult(); try { result.setGroupCommitLoadBeId(Env.getCurrentEnv().getGroupCommitManager() - .selectBackendForGroupCommitInternal(info.groupCommitLoadTableId, info.cluster, info.isCloud)); + .selectBackendForGroupCommitInternal(info.groupCommitLoadTableId, info.cluster)); } catch (LoadException | DdlException e) { throw new TException(e.getMessage()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticRange.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticRange.java index 7b7b08ab24669de..ca9735b56654b12 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticRange.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticRange.java @@ -124,6 +124,10 @@ public boolean isInfinite() { return Double.isInfinite(low) || Double.isInfinite(high); } + public boolean isOneSideInfinite() { + return isInfinite() && !isBothInfinite(); + } + public boolean isFinite() { return Double.isFinite(low) && Double.isFinite(high); } @@ -175,8 +179,7 @@ public Pair maxPair(double r1, LiteralExpr e1, double r2, L } public StatisticRange cover(StatisticRange other) { - // double newLow = Math.max(low, other.low); - // double newHigh = Math.min(high, other.high); + StatisticRange resultRange; Pair biggerLow = maxPair(low, lowExpr, other.low, other.lowExpr); double newLow = biggerLow.first; LiteralExpr newLowExpr = biggerLow.second; @@ -188,9 +191,18 @@ public StatisticRange cover(StatisticRange other) { double overlapPercentOfLeft = overlapPercentWith(other); double overlapDistinctValuesLeft = overlapPercentOfLeft * distinctValues; double coveredDistinctValues = minExcludeNaN(distinctValues, overlapDistinctValuesLeft); - return new StatisticRange(newLow, newLowExpr, newHigh, newHighExpr, coveredDistinctValues, dataType); + if (this.isBothInfinite() && other.isOneSideInfinite()) { + resultRange = new StatisticRange(newLow, newLowExpr, newHigh, newHighExpr, + distinctValues * INFINITE_TO_INFINITE_RANGE_INTERSECT_OVERLAP_HEURISTIC_FACTOR, + dataType); + } else { + resultRange = new StatisticRange(newLow, newLowExpr, newHigh, newHighExpr, coveredDistinctValues, + dataType); + } + } else { + resultRange = empty(dataType); } - return empty(dataType); + return resultRange; } public StatisticRange union(StatisticRange other) { @@ -241,6 +253,6 @@ public double getDistinctValues() { @Override public String toString() { - return "(" + lowExpr + "," + highExpr + ")"; + return "range=(" + lowExpr + "," + highExpr + "), ndv=" + distinctValues; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java index 0f5c81b1cf06929..fd8d6fc2bf59772 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java @@ -981,12 +981,7 @@ public static boolean needAnalyzeColumn(TableIf table, Pair colu if (column == null) { return false; } - try { - if (!table.getDatabase().getCatalog().enableAutoAnalyze()) { - return false; - } - } catch (Throwable t) { - LOG.warn("Failed to get catalog property. {}", t.getMessage()); + if (!table.autoAnalyzeEnabled()) { return false; } AnalysisManager manager = Env.getServingEnv().getAnalysisManager(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java index 6a122d6640b4b80..667943313775583 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java @@ -41,8 +41,12 @@ import org.apache.doris.common.util.TimeUtils; import org.apache.doris.datasource.CatalogIf; import org.apache.doris.datasource.ExternalCatalog; +import org.apache.doris.datasource.ExternalMetaCacheMgr; import org.apache.doris.datasource.InternalCatalog; import org.apache.doris.datasource.hive.HMSExternalCatalog; +import org.apache.doris.datasource.hive.HiveMetaStoreCache; +import org.apache.doris.datasource.hudi.source.HudiCachedPartitionProcessor; +import org.apache.doris.datasource.iceberg.IcebergExternalCatalog; import org.apache.doris.datasource.iceberg.IcebergMetadataCache; import org.apache.doris.datasource.maxcompute.MaxComputeExternalCatalog; import org.apache.doris.job.common.JobType; @@ -117,6 +121,8 @@ public class MetadataGenerator { private static final ImmutableMap TABLE_PROPERTIES_COLUMN_TO_INDEX; + private static final ImmutableMap META_CACHE_STATS_COLUMN_TO_INDEX; + static { ImmutableMap.Builder activeQueriesbuilder = new ImmutableMap.Builder(); List activeQueriesColList = SchemaTable.TABLE_MAP.get("active_queries").getFullSchema(); @@ -164,6 +170,13 @@ public class MetadataGenerator { propertiesBuilder.put(propertiesColList.get(i).getName().toLowerCase(), i); } TABLE_PROPERTIES_COLUMN_TO_INDEX = propertiesBuilder.build(); + + ImmutableMap.Builder metaCacheBuilder = new ImmutableMap.Builder(); + List metaCacheColList = SchemaTable.TABLE_MAP.get("catalog_meta_cache_statistics").getFullSchema(); + for (int i = 0; i < metaCacheColList.size(); i++) { + metaCacheBuilder.put(metaCacheColList.get(i).getName().toLowerCase(), i); + } + META_CACHE_STATS_COLUMN_TO_INDEX = metaCacheBuilder.build(); } public static TFetchSchemaTableDataResult getMetadataTable(TFetchSchemaTableDataRequest request) throws TException { @@ -255,6 +268,10 @@ public static TFetchSchemaTableDataResult getSchemaTableData(TFetchSchemaTableDa result = tablePropertiesMetadataResult(schemaTableParams); columnIndex = TABLE_PROPERTIES_COLUMN_TO_INDEX; break; + case CATALOG_META_CACHE_STATS: + result = metaCacheStatsMetadataResult(schemaTableParams); + columnIndex = META_CACHE_STATS_COLUMN_TO_INDEX; + break; default: return errorResult("invalid schema table name."); } @@ -1250,4 +1267,50 @@ private static TFetchSchemaTableDataResult tablePropertiesMetadataResult(TSchema result.setStatus(new TStatus(TStatusCode.OK)); return result; } + + private static TFetchSchemaTableDataResult metaCacheStatsMetadataResult(TSchemaTableRequestParams params) { + List dataBatch = Lists.newArrayList(); + TFetchSchemaTableDataResult result = new TFetchSchemaTableDataResult(); + ExternalMetaCacheMgr mgr = Env.getCurrentEnv().getExtMetaCacheMgr(); + for (CatalogIf catalogIf : Env.getCurrentEnv().getCatalogMgr().getCopyOfCatalog()) { + if (catalogIf instanceof HMSExternalCatalog) { + HMSExternalCatalog catalog = (HMSExternalCatalog) catalogIf; + // 1. hive metastore cache + HiveMetaStoreCache cache = mgr.getMetaStoreCache(catalog); + if (cache != null) { + fillBatch(dataBatch, cache.getStats(), catalog.getName()); + } + // 2. hudi cache + HudiCachedPartitionProcessor processor + = (HudiCachedPartitionProcessor) mgr.getHudiPartitionProcess(catalog); + fillBatch(dataBatch, processor.getCacheStats(), catalog.getName()); + } else if (catalogIf instanceof IcebergExternalCatalog) { + // 3. iceberg cache + IcebergMetadataCache icebergCache = mgr.getIcebergMetadataCache(); + fillBatch(dataBatch, icebergCache.getCacheStats(), catalogIf.getName()); + } + } + + result.setDataBatch(dataBatch); + result.setStatus(new TStatus(TStatusCode.OK)); + return result; + } + + private static void fillBatch(List dataBatch, Map> stats, + String catalogName) { + for (Map.Entry> entry : stats.entrySet()) { + String cacheName = entry.getKey(); + Map cacheStats = entry.getValue(); + for (Map.Entry cacheStatsEntry : cacheStats.entrySet()) { + String metricName = cacheStatsEntry.getKey(); + String metricValue = cacheStatsEntry.getValue(); + TRow trow = new TRow(); + trow.addToColumnValue(new TCell().setStringVal(catalogName)); // CATALOG_NAME + trow.addToColumnValue(new TCell().setStringVal(cacheName)); // CACHE_NAME + trow.addToColumnValue(new TCell().setStringVal(metricName)); // METRIC_NAME + trow.addToColumnValue(new TCell().setStringVal(metricValue)); // METRIC_VALUE + dataBatch.add(trow); + } + } + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/joinhint/DistributeHintTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/joinhint/DistributeHintTest.java index 0b2ed8069ade838..f96fe7e918c4102 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/joinhint/DistributeHintTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/joinhint/DistributeHintTest.java @@ -28,13 +28,12 @@ import org.apache.doris.nereids.util.MemoTestUtils; import org.apache.doris.nereids.util.PlanChecker; -import com.google.common.collect.Maps; +import com.google.common.collect.ImmutableList; import org.junit.jupiter.api.Test; import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.Map; import java.util.Set; public class DistributeHintTest extends TPCHTestBase { @@ -85,15 +84,15 @@ public void testHintJoin() { } private Plan generateLeadingHintPlan(int tableNum, Plan childPlan) { - Map hints = Maps.newLinkedHashMap(); + ImmutableList.Builder hints = ImmutableList.builder(); List leadingParameters = new ArrayList(); for (int i = 0; i < tableNum; i++) { leadingParameters.add(String.valueOf(i)); } Collections.shuffle(leadingParameters); System.out.println("LeadingHint: " + leadingParameters.toString()); - hints.put("leading", new SelectHintLeading("leading", leadingParameters)); - return new LogicalSelectHint<>(hints, childPlan); + hints.add(new SelectHintLeading("Leading", leadingParameters)); + return new LogicalSelectHint<>(hints.build(), childPlan); } private void randomTest(int tableNum, int edgeNum, boolean withJoinHint, boolean withLeading) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/MaxMinFilterPushDownTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/MaxMinFilterPushDownTest.java new file mode 100644 index 000000000000000..bc7d32fb3fb24a6 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/MaxMinFilterPushDownTest.java @@ -0,0 +1,115 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.rewrite; + +import org.apache.doris.nereids.util.MemoPatternMatchSupported; +import org.apache.doris.nereids.util.PlanChecker; +import org.apache.doris.utframe.TestWithFeService; + +import org.junit.jupiter.api.Test; + +public class MaxMinFilterPushDownTest extends TestWithFeService implements MemoPatternMatchSupported { + @Override + protected void runBeforeAll() throws Exception { + createDatabase("test"); + connectContext.setDatabase("test"); + createTable("CREATE TABLE IF NOT EXISTS max_t(\n" + + "`id` int(32),\n" + + "`score` int(64) NULL,\n" + + "`name` varchar(64) NULL\n" + + ") properties('replication_num'='1');"); + connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); + } + + @Test + public void testMaxRewrite() { + String sql = "select id, max(score) from max_t group by id having max(score)>10"; + PlanChecker.from(connectContext).analyze(sql).rewrite() + .matches(logicalFilter(logicalOlapScan()).when(filter -> filter.getConjuncts().size() == 1)); + } + + @Test + public void testMinRewrite() { + String sql = "select id, min(score) from max_t group by id having min(score)<10"; + PlanChecker.from(connectContext).analyze(sql).rewrite() + .matches(logicalFilter(logicalOlapScan()).when(filter -> filter.getConjuncts().size() == 1)); + } + + @Test + public void testNotRewriteBecauseFuncIsMoreThanOne1() { + String sql = "select id, min(score), max(name) from max_t group by id having min(score)<10 and max(name)>'abc'"; + PlanChecker.from(connectContext).analyze(sql).rewrite() + .nonMatch(logicalFilter(logicalOlapScan())); + } + + @Test + public void testNotRewriteBecauseFuncIsMoreThanOne2() { + String sql = "select id, min(score), min(name) from max_t group by id having min(score)<10 and min(name)<'abc'"; + PlanChecker.from(connectContext).analyze(sql).rewrite() + .nonMatch(logicalFilter(logicalOlapScan())); + } + + @Test + public void testMaxNotRewriteBecauseLessThan() { + String sql = "select id, max(score) from max_t group by id having max(score)<10"; + PlanChecker.from(connectContext).analyze(sql).rewrite() + .nonMatch(logicalFilter(logicalOlapScan())); + } + + @Test + public void testMinNotRewriteBecauseGreaterThan() { + String sql = "select id, min(score) from max_t group by id having min(score)>10"; + PlanChecker.from(connectContext).analyze(sql).rewrite() + .nonMatch(logicalFilter(logicalOlapScan())); + } + + @Test + public void testMinNotRewriteBecauseHasMaxFunc() { + String sql = "select id, min(score), max(score) from max_t group by id having min(score)<10"; + PlanChecker.from(connectContext).analyze(sql).rewrite() + .nonMatch(logicalFilter(logicalOlapScan())); + } + + @Test + public void testMinNotRewriteBecauseHasCountFunc() { + String sql = "select id, min(score), count(score) from max_t group by id having min(score)<10"; + PlanChecker.from(connectContext).analyze(sql).rewrite() + .nonMatch(logicalFilter(logicalOlapScan())); + } + + @Test + public void testNotRewriteBecauseConjunctLeftNotSlot() { + String sql = "select id, max(score) from max_t group by id having abs(max(score))>10"; + PlanChecker.from(connectContext).analyze(sql).rewrite() + .nonMatch(logicalFilter(logicalOlapScan())); + } + + @Test + public void testRewriteAggFuncHasExpr() { + String sql = "select id, max(score+1) from max_t group by id having max(score+1)>10"; + PlanChecker.from(connectContext).analyze(sql).rewrite() + .matches(logicalFilter(logicalOlapScan()).when(filter -> filter.getConjuncts().size() == 1)); + } + + @Test + public void testNotRewriteScalarAgg() { + String sql = "select max(score+1) from max_t having max(score+1)>10"; + PlanChecker.from(connectContext).analyze(sql).rewrite() + .nonMatch(logicalFilter(logicalOlapScan())); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/FilterEstimationTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/FilterEstimationTest.java index 0158dd9587c70b9..6e76c3f6a33d1c5 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/FilterEstimationTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/FilterEstimationTest.java @@ -1394,4 +1394,39 @@ public void testLargeRange() { out = estimation.estimate(greater, stats); Assertions.assertEquals(out.getRowCount(), row * FilterEstimation.RANGE_SELECTIVITY_THRESHOLD); } + + @Test + void testAndWithInfinity() { + Double row = 1000.0; + SlotReference a = new SlotReference("a", new VarcharType(25)); + ColumnStatisticBuilder columnStatisticBuilderA = new ColumnStatisticBuilder() + .setNdv(10) + .setAvgSizeByte(4) + .setNumNulls(0) + .setCount(row); + + SlotReference b = new SlotReference("b", IntegerType.INSTANCE); + ColumnStatisticBuilder columnStatisticBuilderB = new ColumnStatisticBuilder() + .setNdv(488) + .setAvgSizeByte(25) + .setNumNulls(0) + .setCount(row); + StatisticsBuilder statsBuilder = new StatisticsBuilder(); + statsBuilder.setRowCount(row); + statsBuilder.putColumnStatistics(a, columnStatisticBuilderA.build()); + statsBuilder.putColumnStatistics(b, columnStatisticBuilderB.build()); + Expression strGE = new GreaterThanEqual(a, + new org.apache.doris.nereids.trees.expressions.literal.StringLiteral("2024-05-14")); + Statistics strStats = new FilterEstimation().estimate(strGE, statsBuilder.build()); + Assertions.assertEquals(500, strStats.getRowCount()); + + Expression intGE = new GreaterThan(b, new IntegerLiteral(0)); + Statistics intStats = new FilterEstimation().estimate(intGE, statsBuilder.build()); + Assertions.assertEquals(500, intStats.getRowCount()); + + Expression predicate = new And(strGE, intGE); + + Statistics stats = new FilterEstimation().estimate(predicate, statsBuilder.build()); + Assertions.assertEquals(250, stats.getRowCount()); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/statistics/util/StatisticsUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/statistics/util/StatisticsUtilTest.java index 6dccd3af46e7285..32521882939d348 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/statistics/util/StatisticsUtilTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/statistics/util/StatisticsUtilTest.java @@ -18,21 +18,21 @@ package org.apache.doris.statistics.util; import org.apache.doris.catalog.Column; -import org.apache.doris.catalog.Database; -import org.apache.doris.catalog.DatabaseIf; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.catalog.TableProperty; import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; import org.apache.doris.common.Pair; -import org.apache.doris.datasource.CatalogIf; +import org.apache.doris.common.util.PropertyAnalyzer; import org.apache.doris.datasource.ExternalCatalog; import org.apache.doris.datasource.ExternalTable; import org.apache.doris.datasource.hive.HMSExternalCatalog; import org.apache.doris.datasource.hive.HMSExternalTable; import org.apache.doris.datasource.hive.HMSExternalTable.DLAType; +import org.apache.doris.datasource.jdbc.JdbcExternalCatalog; import org.apache.doris.datasource.jdbc.JdbcExternalTable; import org.apache.doris.qe.SessionVariable; import org.apache.doris.statistics.AnalysisManager; @@ -51,7 +51,9 @@ import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Base64; +import java.util.HashMap; import java.util.List; +import java.util.Map; class StatisticsUtilTest { @Test @@ -176,42 +178,46 @@ void testNeedAnalyzeColumn() throws DdlException { List schema = new ArrayList<>(); schema.add(column); OlapTable table = new OlapTable(200, "testTable", schema, null, null, null); - Database db = new Database(111, "TestDb"); - ExternalCatalog externalCatalog = new HMSExternalCatalog(); + HMSExternalCatalog externalCatalog = new HMSExternalCatalog(); - // Test get database/catalog exception - new MockUp() { - @Mock - public DatabaseIf getDatabase() { - throw new RuntimeException(); - } - }; + // Test olap table auto analyze disabled. + Map properties = new HashMap<>(); + properties.put(PropertyAnalyzer.PROPERTIES_AUTO_ANALYZE_POLICY, "disable"); + table.setTableProperty(new TableProperty(properties)); Assertions.assertFalse(StatisticsUtil.needAnalyzeColumn(table, Pair.of("index", column.getName()))); + table.setTableProperty(null); - // Test auto analyze disabled. - new MockUp() { + new MockUp() { @Mock - public DatabaseIf getDatabase() { - return db; + protected synchronized void makeSureInitialized() { } }; - new MockUp() { + + // Test auto analyze catalog disabled. + HMSExternalTable hmsTable = new HMSExternalTable(1, "name", "dbName", externalCatalog); + Assertions.assertFalse(StatisticsUtil.needAnalyzeColumn(hmsTable, Pair.of("index", column.getName()))); + + // Test catalog auto analyze enabled. + new MockUp() { @Mock - public CatalogIf getCatalog() { - return externalCatalog; + public TableStatsMeta findTableStatsStatus(long tblId) { + return null; } }; - Assertions.assertFalse(StatisticsUtil.needAnalyzeColumn(table, Pair.of("index", column.getName()))); + externalCatalog.getCatalogProperty().addProperty(ExternalCatalog.ENABLE_AUTO_ANALYZE, "true"); + Assertions.assertTrue(StatisticsUtil.needAnalyzeColumn(table, Pair.of("index", column.getName()))); - // Test auto analyze enabled. + // Test external table auto analyze enabled. new MockUp() { @Mock public TableStatsMeta findTableStatsStatus(long tblId) { return null; } }; - externalCatalog.getCatalogProperty().addProperty(ExternalCatalog.ENABLE_AUTO_ANALYZE, "true"); - Assertions.assertTrue(StatisticsUtil.needAnalyzeColumn(table, Pair.of("index", column.getName()))); + externalCatalog.getCatalogProperty().addProperty(ExternalCatalog.ENABLE_AUTO_ANALYZE, "false"); + HMSExternalTable hmsTable1 = new HMSExternalTable(1, "name", "dbName", externalCatalog); + externalCatalog.setAutoAnalyzePolicy("dbName", "name", "enable"); + Assertions.assertTrue(StatisticsUtil.needAnalyzeColumn(hmsTable1, Pair.of("index", column.getName()))); // Test table stats meta is null. @@ -256,8 +262,14 @@ public ColStatsMeta findColumnStatsMeta(String indexName, String colName) { } }; + new MockUp() { + @Mock + protected synchronized void makeSureInitialized() { + } + }; // Test not supported external table type. - ExternalTable externalTable = new JdbcExternalTable(1, "jdbctable", "jdbcdb", null); + ExternalTable externalTable = new JdbcExternalTable(1, "jdbctable", "jdbcdb", + new JdbcExternalCatalog(1, "name", "resource", new HashMap<>(), "")); Assertions.assertFalse(StatisticsUtil.needAnalyzeColumn(externalTable, Pair.of("index", column.getName()))); // Test hms external table not hive type. @@ -267,7 +279,7 @@ public DLAType getDlaType() { return DLAType.ICEBERG; } }; - ExternalTable hmsExternalTable = new HMSExternalTable(1, "hmsTable", "hmsDb", null); + ExternalTable hmsExternalTable = new HMSExternalTable(1, "hmsTable", "hmsDb", externalCatalog); Assertions.assertFalse(StatisticsUtil.needAnalyzeColumn(hmsExternalTable, Pair.of("index", column.getName()))); // Test partition first load. diff --git a/gensrc/thrift/DataSinks.thrift b/gensrc/thrift/DataSinks.thrift index 14f528666a3121e..e46f7e6067cfef3 100644 --- a/gensrc/thrift/DataSinks.thrift +++ b/gensrc/thrift/DataSinks.thrift @@ -133,6 +133,13 @@ struct TResultFileSinkOptions { 18: optional bool with_bom; 19: optional PlanNodes.TFileCompressType orc_compression_type; + + // Since we have changed the type mapping from Doris to Orc type, + // using the Outfile to export Date/Datetime types will cause BE core dump + // when only upgrading BE without upgrading FE. + // orc_writer_version = 1 means doris FE is higher than version 2.1.5 + // orc_writer_version = 0 means doris FE is less than or equal to version 2.1.5 + 20: optional i64 orc_writer_version; } struct TMemoryScratchSink { diff --git a/gensrc/thrift/Descriptors.thrift b/gensrc/thrift/Descriptors.thrift index e11160caa7f5afa..56222c2338a2680 100644 --- a/gensrc/thrift/Descriptors.thrift +++ b/gensrc/thrift/Descriptors.thrift @@ -137,7 +137,8 @@ enum TSchemaTableType { SCH_WORKLOAD_GROUP_PRIVILEGES = 48, SCH_WORKLOAD_GROUP_RESOURCE_USAGE = 49, SCH_TABLE_PROPERTIES = 50, - SCH_FILE_CACHE_STATISTICS = 51 + SCH_FILE_CACHE_STATISTICS = 51, + SCH_CATALOG_META_CACHE_STATISTICS = 52; } enum THdfsCompression { diff --git a/gensrc/thrift/FrontendService.thrift b/gensrc/thrift/FrontendService.thrift index 2dcdf9b718036f1..765e7790e907812 100644 --- a/gensrc/thrift/FrontendService.thrift +++ b/gensrc/thrift/FrontendService.thrift @@ -545,7 +545,6 @@ struct TGroupCommitInfo{ 1: optional bool getGroupCommitLoadBeId 2: optional i64 groupCommitLoadTableId 3: optional string cluster - 4: optional bool isCloud 5: optional bool updateLoadData 6: optional i64 tableId 7: optional i64 receiveData @@ -1005,6 +1004,7 @@ enum TSchemaTableName { TABLE_OPTIONS = 6, WORKLOAD_GROUP_PRIVILEGES = 7, TABLE_PROPERTIES = 8, + CATALOG_META_CACHE_STATS = 9, } struct TMetadataTableRequestParams { @@ -1019,6 +1019,7 @@ struct TMetadataTableRequestParams { 9: optional PlanNodes.TJobsMetadataParams jobs_metadata_params 10: optional PlanNodes.TTasksMetadataParams tasks_metadata_params 11: optional PlanNodes.TPartitionsMetadataParams partitions_metadata_params + 12: optional PlanNodes.TMetaCacheStatsParams meta_cache_stats_params } struct TSchemaTableRequestParams { diff --git a/gensrc/thrift/PaloInternalService.thrift b/gensrc/thrift/PaloInternalService.thrift index 85e4ade4ca4adcf..9da87117154dcc3 100644 --- a/gensrc/thrift/PaloInternalService.thrift +++ b/gensrc/thrift/PaloInternalService.thrift @@ -335,6 +335,9 @@ struct TQueryOptions { 127: optional i32 in_list_value_count_threshold = 10; + 128: optional bool enable_adaptive_pipeline_task_serial_read_on_limit = true; + 129: optional i32 adaptive_pipeline_task_serial_read_on_limit = 10000; + // For cloud, to control if the content would be written into file cache // In write path, to control if the content would be written into file cache. // In read path, read from file cache or remote storage when execute query. diff --git a/gensrc/thrift/PlanNodes.thrift b/gensrc/thrift/PlanNodes.thrift index 758ead765325309..e53289c1b865fde 100644 --- a/gensrc/thrift/PlanNodes.thrift +++ b/gensrc/thrift/PlanNodes.thrift @@ -554,6 +554,9 @@ struct TQueriesMetadataParams { 6: optional TPartitionsMetadataParams partitions_params } +struct TMetaCacheStatsParams { +} + struct TMetaScanRange { 1: optional Types.TMetadataType metadata_type 2: optional TIcebergMetadataParams iceberg_params @@ -564,6 +567,7 @@ struct TMetaScanRange { 7: optional TJobsMetadataParams jobs_params 8: optional TTasksMetadataParams tasks_params 9: optional TPartitionsMetadataParams partitions_params + 10: optional TMetaCacheStatsParams meta_cache_stats_params } // Specification of an individual data range which is held in its entirety diff --git a/gensrc/thrift/Types.thrift b/gensrc/thrift/Types.thrift index e947dfc27c9a21f..ee684a72df2e9aa 100644 --- a/gensrc/thrift/Types.thrift +++ b/gensrc/thrift/Types.thrift @@ -732,7 +732,7 @@ enum TMetadataType { JOBS, TASKS, WORKLOAD_SCHED_POLICY, - PARTITIONS + PARTITIONS; } enum TIcebergQueryType { diff --git a/regression-test/data/datatype_p0/decimalv3/test_decimal256_cast.out b/regression-test/data/datatype_p0/decimalv3/test_decimal256_cast.out index 0d79c0606906f20..ac1f3da16dd4fea 100644 --- a/regression-test/data/datatype_p0/decimalv3/test_decimal256_cast.out +++ b/regression-test/data/datatype_p0/decimalv3/test_decimal256_cast.out @@ -26,3 +26,13 @@ -- !decimal256_cast8 -- 0 +-- !decimal256_cast9 -- +-9 -999999999999999999999999999999999999999999999999999999999999999999.9999999999 +9 999999999999999999999999999999999999999999999999999999999999999999.9999999999 + +-- !decimal256_cast10 -- +10 0 + +-- !decimal256_cast_to_double_1 -- +1.2345678E7 + diff --git a/regression-test/data/external_table_p0/jdbc/test_mariadb_jdbc_catalog.out b/regression-test/data/external_table_p0/jdbc/test_mariadb_jdbc_catalog.out index 902a0c668383ab0..b6817f090f5bd70 100644 --- a/regression-test/data/external_table_p0/jdbc/test_mariadb_jdbc_catalog.out +++ b/regression-test/data/external_table_p0/jdbc/test_mariadb_jdbc_catalog.out @@ -28,42 +28,7 @@ mariadb_jdbc_catalog 115 abg -- !information_schema -- -active_queries -backend_active_tasks -character_sets -collations -column_privileges -column_statistics -columns -engines -events -files -global_variables -key_column_usage -metadata_name_ids -parameters -partitions processlist -profiling -referential_constraints -routines -rowsets -schema_privileges -schemata -session_variables -statistics -table_constraints -table_options -table_privileges -table_properties -tables -triggers -user_privileges -views -workload_group_privileges -workload_group_resource_usage -workload_groups -workload_policy -- !auto_default_t -- 0 diff --git a/regression-test/data/external_table_p0/jdbc/test_mysql_jdbc_catalog.out b/regression-test/data/external_table_p0/jdbc/test_mysql_jdbc_catalog.out index 2be6d4a141f7fd8..d697c8e5e600ecc 100644 --- a/regression-test/data/external_table_p0/jdbc/test_mysql_jdbc_catalog.out +++ b/regression-test/data/external_table_p0/jdbc/test_mysql_jdbc_catalog.out @@ -192,42 +192,7 @@ bca 2022-11-02 2022-11-02 8012 vivo 2 2 -- !information_schema -- -active_queries -backend_active_tasks -character_sets -collations -column_privileges -column_statistics -columns -engines -events -files -global_variables -key_column_usage -metadata_name_ids -parameters -partitions processlist -profiling -referential_constraints -routines -rowsets -schema_privileges -schemata -session_variables -statistics -table_constraints -table_options -table_privileges -table_properties -tables -triggers -user_privileges -views -workload_group_privileges -workload_group_resource_usage -workload_groups -workload_policy -- !dt -- 2023-06-17T10:00 2023-06-17T10:00:01.100 2023-06-17T10:00:02.220 2023-06-17T10:00:03.333 2023-06-17T10:00:04.444400 2023-06-17T10:00:05.555550 2023-06-17T10:00:06.666666 diff --git a/regression-test/data/external_table_p0/jdbc/test_mysql_jdbc_catalog_nereids.out b/regression-test/data/external_table_p0/jdbc/test_mysql_jdbc_catalog_nereids.out index 08f8a3689d02898..171cee4d958178b 100644 --- a/regression-test/data/external_table_p0/jdbc/test_mysql_jdbc_catalog_nereids.out +++ b/regression-test/data/external_table_p0/jdbc/test_mysql_jdbc_catalog_nereids.out @@ -160,42 +160,7 @@ bca 2022-11-02 2022-11-02 8012 vivo 123456789012345678901234567890123.12345 12345678901234567890123456789012.12345 1234567890123456789012345678901234.12345 123456789012345678901234567890123.12345 123456789012345678901234567890123456789012345678901234567890.12345 123456789012345678901234567890123456789012345678901234567890.12345 -- !information_schema -- -active_queries -backend_active_tasks -character_sets -collations -column_privileges -column_statistics -columns -engines -events -files -global_variables -key_column_usage -metadata_name_ids -parameters -partitions processlist -profiling -referential_constraints -routines -rowsets -schema_privileges -schemata -session_variables -statistics -table_constraints -table_options -table_privileges -table_properties -tables -triggers -user_privileges -views -workload_group_privileges -workload_group_resource_usage -workload_groups -workload_policy -- !test_insert1 -- doris1 18 diff --git a/regression-test/data/external_table_p0/jdbc/test_mysql_jdbc_driver5_catalog.out b/regression-test/data/external_table_p0/jdbc/test_mysql_jdbc_driver5_catalog.out index 47ee7c60f387700..736f2b57b4a768b 100644 --- a/regression-test/data/external_table_p0/jdbc/test_mysql_jdbc_driver5_catalog.out +++ b/regression-test/data/external_table_p0/jdbc/test_mysql_jdbc_driver5_catalog.out @@ -202,42 +202,7 @@ bca 2022-11-02 2022-11-02 8012 vivo 2 2 -- !information_schema -- -active_queries -backend_active_tasks -character_sets -collations -column_privileges -column_statistics -columns -engines -events -files -global_variables -key_column_usage -metadata_name_ids -parameters -partitions processlist -profiling -referential_constraints -routines -rowsets -schema_privileges -schemata -session_variables -statistics -table_constraints -table_options -table_privileges -table_properties -tables -triggers -user_privileges -views -workload_group_privileges -workload_group_resource_usage -workload_groups -workload_policy -- !dt -- 2023-06-17T10:00 2023-06-17T10:00:01 2023-06-17T10:00:02 2023-06-17T10:00:03 2023-06-17T10:00:04 2023-06-17T10:00:05 2023-06-17T10:00:06 diff --git a/regression-test/data/inverted_index_p0/test_ignore_above_in_index.out b/regression-test/data/inverted_index_p0/test_ignore_above_in_index.out index 718bd29e5d98758..66d0935327dd4bc 100644 --- a/regression-test/data/inverted_index_p0/test_ignore_above_in_index.out +++ b/regression-test/data/inverted_index_p0/test_ignore_above_in_index.out @@ -5,3 +5,6 @@ -- !sql -- 772 +-- !sql -- +971 + diff --git a/regression-test/data/nereids_p0/sql_functions/encryption_digest/test_encryption_function.out b/regression-test/data/nereids_p0/sql_functions/encryption_digest/test_encryption_function.out index c1f7d7b1c9d4076..7a91c1dbf720ed7 100644 --- a/regression-test/data/nereids_p0/sql_functions/encryption_digest/test_encryption_function.out +++ b/regression-test/data/nereids_p0/sql_functions/encryption_digest/test_encryption_function.out @@ -11,33 +11,6 @@ text -- !sql -- \N --- !sql -- -wr2JEDVXzL9+2XtRhgIloA== - --- !sql -- -wr2JEDVXzL9+2XtRhgIloA== - --- !sql -- -text - --- !sql -- -\N - --- !sql -- -wr2JEDVXzL9+2XtRhgIloA== - --- !sql -- -BO2vxHeUcw5BQQalSBbo1w== - --- !sql -- -text - --- !sql -- -\N - --- !sql -- -text - -- !sql -- BO2vxHeUcw5BQQalSBbo1w== @@ -54,25 +27,7 @@ text text -- !sql -- -wr2JEDVXzL9+2XtRhgIloA== - --- !sql -- -tsmK1HzbpnEdR2//WhO+MA== - --- !sql -- -ciacXDLHMNG7CD9Kws8png== - --- !sql -- -text - --- !sql -- -\N - --- !sql -- -text - --- !sql -- -text +3dym0E7/+1zbrLIaBVNHSw== -- !sql -- tsmK1HzbpnEdR2//WhO+MA== @@ -83,45 +38,12 @@ ciacXDLHMNG7CD9Kws8png== -- !sql -- \N --- !sql -- -text - --- !sql -- -text - --- !sql -- -aDjwRflBrDjhBZIOFNw3Tg== - --- !sql -- -1Y4NGIukSbv9OrkZnRD1bQ== - --- !sql -- -G5POcFAJwiZHeTtN6DjInQ== - --- !sql -- -text - --- !sql -- -text - -- !sql -- \N -- !sql -- text --- !sql -- -1Y4NGIukSbv9OrkZnRD1bQ== - --- !sql -- -G5POcFAJwiZHeTtN6DjInQ== - --- !sql -- -text - --- !sql -- -\N - -- !sql -- text diff --git a/regression-test/data/nereids_rules_p0/eager_aggregate/push_down_max_through_join.out b/regression-test/data/nereids_rules_p0/eager_aggregate/push_down_max_through_join.out index 281de8ea61b88e6..79b4ed890ded953 100644 --- a/regression-test/data/nereids_rules_p0/eager_aggregate/push_down_max_through_join.out +++ b/regression-test/data/nereids_rules_p0/eager_aggregate/push_down_max_through_join.out @@ -91,11 +91,11 @@ PhysicalResultSink -- !groupby_pushdown_having -- PhysicalResultSink ---filter((max(score) > 100)) -----hashAgg[GLOBAL] -------hashAgg[LOCAL] ---------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=() -----------PhysicalOlapScan[max_t] +--hashAgg[GLOBAL] +----hashAgg[LOCAL] +------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=() +--------PhysicalOlapScan[max_t] +--------filter((t1.score > 100)) ----------PhysicalOlapScan[max_t] -- !groupby_pushdown_mixed_aggregates -- @@ -366,11 +366,11 @@ SyntaxError: -- !with_hint_groupby_pushdown_having -- PhysicalResultSink ---filter((max(score) > 100)) -----hashAgg[GLOBAL] -------hashAgg[LOCAL] ---------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=() -----------PhysicalOlapScan[max_t] +--hashAgg[GLOBAL] +----hashAgg[LOCAL] +------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=() +--------PhysicalOlapScan[max_t] +--------filter((t1.score > 100)) ----------PhysicalOlapScan[max_t] Hint log: diff --git a/regression-test/data/nereids_rules_p0/max_min_filter_push_down/max_min_filter_push_down.out b/regression-test/data/nereids_rules_p0/max_min_filter_push_down/max_min_filter_push_down.out new file mode 100644 index 000000000000000..2e0ac41d5ebd31e --- /dev/null +++ b/regression-test/data/nereids_rules_p0/max_min_filter_push_down/max_min_filter_push_down.out @@ -0,0 +1,290 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !scalar_agg_empty_table -- +PhysicalResultSink +--filter((min(value1) < 20)) +----hashAgg[GLOBAL] +------hashAgg[LOCAL] +--------PhysicalEmptyRelation + +-- !min -- +PhysicalResultSink +--hashAgg[GLOBAL] +----hashAgg[LOCAL] +------filter((max_min_filter_push_down1.value1 < 20)) +--------PhysicalOlapScan[max_min_filter_push_down1] + +-- !max -- +PhysicalResultSink +--hashAgg[GLOBAL] +----hashAgg[LOCAL] +------filter((max_min_filter_push_down1.value1 > 40)) +--------PhysicalOlapScan[max_min_filter_push_down1] + +-- !min_expr -- +PhysicalResultSink +--hashAgg[GLOBAL] +----hashAgg[LOCAL] +------filter((cast(value1 as BIGINT) < 19)) +--------PhysicalOlapScan[max_min_filter_push_down1] + +-- !max_expr -- +PhysicalResultSink +--hashAgg[GLOBAL] +----hashAgg[LOCAL] +------filter((abs(value1) > 39)) +--------PhysicalOlapScan[max_min_filter_push_down1] + +-- !min_commute -- +PhysicalResultSink +--hashAgg[GLOBAL] +----hashAgg[LOCAL] +------filter((max_min_filter_push_down1.value1 < 40)) +--------PhysicalOlapScan[max_min_filter_push_down1] + +-- !max -- +PhysicalResultSink +--hashAgg[GLOBAL] +----hashAgg[LOCAL] +------filter((max_min_filter_push_down1.value1 > 40)) +--------PhysicalOlapScan[max_min_filter_push_down1] + +-- !min_equal -- +PhysicalResultSink +--hashAgg[GLOBAL] +----hashAgg[LOCAL] +------filter((max_min_filter_push_down1.value1 <= 20)) +--------PhysicalOlapScan[max_min_filter_push_down1] + +-- !max_equal -- +PhysicalResultSink +--hashAgg[GLOBAL] +----hashAgg[LOCAL] +------filter((max_min_filter_push_down1.value1 >= 40)) +--------PhysicalOlapScan[max_min_filter_push_down1] + +-- !min_commute_equal -- +PhysicalResultSink +--hashAgg[GLOBAL] +----hashAgg[LOCAL] +------filter((max_min_filter_push_down1.value1 <= 40)) +--------PhysicalOlapScan[max_min_filter_push_down1] + +-- !max_commute_equal -- +PhysicalResultSink +--hashAgg[GLOBAL] +----hashAgg[LOCAL] +------filter((max_min_filter_push_down1.value1 >= 40)) +--------PhysicalOlapScan[max_min_filter_push_down1] + +-- !has_other_agg_func -- +PhysicalResultSink +--filter((max(value1) >= 40)) +----hashAgg[GLOBAL] +------hashAgg[LOCAL] +--------PhysicalOlapScan[max_min_filter_push_down1] + +-- !min_scalar_agg -- +PhysicalResultSink +--filter((min(value1) < 40)) +----hashAgg[GLOBAL] +------hashAgg[LOCAL] +--------PhysicalStorageLayerAggregate[max_min_filter_push_down1] + +-- !max_scalar_agg -- +PhysicalResultSink +--filter((max(value1) > 40)) +----hashAgg[GLOBAL] +------hashAgg[LOCAL] +--------PhysicalStorageLayerAggregate[max_min_filter_push_down1] + +-- !max_scalar_agg -- +PhysicalResultSink +--filter((max(value1) > 40)) +----hashAgg[GLOBAL] +------hashAgg[LOCAL] +--------PhysicalStorageLayerAggregate[max_min_filter_push_down1] + +-- !min_equal_scalar_agg -- +PhysicalResultSink +--filter((min(value1) <= 20)) +----hashAgg[GLOBAL] +------hashAgg[LOCAL] +--------PhysicalStorageLayerAggregate[max_min_filter_push_down1] + +-- !max_equal_scalar_agg -- +PhysicalResultSink +--filter((max(value1) >= 40)) +----hashAgg[GLOBAL] +------hashAgg[LOCAL] +--------PhysicalStorageLayerAggregate[max_min_filter_push_down1] + +-- !depend_prune_column -- +PhysicalResultSink +--hashAgg[GLOBAL] +----hashAgg[LOCAL] +------filter((max_min_filter_push_down1.value1 < 10)) +--------PhysicalOlapScan[max_min_filter_push_down1] + +-- !scalar_agg_empty_table_res -- + +-- !min_res -- +1 10 +2 19 + +-- !max_res -- +2 73 +3 61 +4 45 + +-- !min_expr_res -- +1 11 + +-- !max_expr_res -- +2 74 +3 62 +4 46 + +-- !min_commute_res -- +1 10 +2 19 +3 30 + +-- !max_res -- +2 73 +3 61 +4 45 + +-- !min_equal_res -- +1 10 +2 19 + +-- !max_equal_res -- +2 73 +3 61 +4 45 + +-- !min_commute_equal_res -- +1 10 +2 19 +3 30 +4 40 + +-- !max_commute_equal_res -- +2 73 +3 61 +4 45 + +-- !has_other_agg_func_res -- +2 73 19 +3 61 30 +4 45 40 + +-- !min_scalar_agg_res -- +10 + +-- !max_scalar_agg_res -- +73 + +-- !max_scalar_agg_res -- +73 + +-- !min_equal_scalar_agg_res -- +10 + +-- !max_equal_scalar_agg_res -- +73 + +-- !depend_prune_column_res -- +10 +19 + +-- !smallint -- +PhysicalResultSink +--hashAgg[GLOBAL] +----hashAgg[LOCAL] +------filter((max_min_filter_push_down2.d_smallint > 10)) +--------PhysicalOlapScan[max_min_filter_push_down2] + +-- !tinyint -- +PhysicalResultSink +--hashAgg[GLOBAL] +----hashAgg[LOCAL] +------filter((max_min_filter_push_down2.d_tinyint < 10)) +--------PhysicalOlapScan[max_min_filter_push_down2] + +-- !char100 -- +PhysicalResultSink +--hashAgg[GLOBAL] +----hashAgg[LOCAL] +------filter((max_min_filter_push_down2.d_char100 > 'ab')) +--------PhysicalOlapScan[max_min_filter_push_down2] + +-- !char100_cmp_num_cannot_rewrite -- +PhysicalResultSink +--filter((cast(min(d_char100) as DOUBLE) < 10.0)) +----hashAgg[GLOBAL] +------hashAgg[LOCAL] +--------PhysicalOlapScan[max_min_filter_push_down2] + +-- !datetimev2 -- +PhysicalResultSink +--hashAgg[GLOBAL] +----hashAgg[LOCAL] +------filter((max_min_filter_push_down2.d_datetimev2 < '2020-01-09 00:00:00')) +--------PhysicalOlapScan[max_min_filter_push_down2] + +-- !datev2 -- +PhysicalResultSink +--hashAgg[GLOBAL] +----hashAgg[LOCAL] +------filter((max_min_filter_push_down2.d_datev2 > '2020-01-09')) +--------PhysicalOlapScan[max_min_filter_push_down2] + +-- !smallint_group_by_key -- +PhysicalResultSink +--hashAgg[GLOBAL] +----hashAgg[LOCAL] +------filter((max_min_filter_push_down2.d_smallint > 10)) +--------PhysicalOlapScan[max_min_filter_push_down2] + +-- !tinyint_group_by_key -- +PhysicalResultSink +--hashAgg[GLOBAL] +----hashAgg[LOCAL] +------filter((max_min_filter_push_down2.d_tinyint < 10)) +--------PhysicalOlapScan[max_min_filter_push_down2] + +-- !char100_group_by_key -- +PhysicalResultSink +--hashAgg[GLOBAL] +----hashAgg[LOCAL] +------filter((max_min_filter_push_down2.d_char100 > 'ab')) +--------PhysicalOlapScan[max_min_filter_push_down2] + +-- !smallint_res -- +14 32 + +-- !tinyint_res -- +1 3 + +-- !char100_res -- + +-- !char100_cmp_num_cannot_rewrite_res -- + +-- !datetimev2_res -- +1 2020-01-07T10:00:01 +14 2020-01-07T10:00:01 + +-- !datev2_res -- +1 2020-01-11 +14 2020-01-11 + +-- !smallint_group_by_key_res -- +29 +32 + +-- !tinyint_group_by_key_res -- +3 + +-- !char100_group_by_key_res -- + diff --git a/regression-test/data/nereids_rules_p0/mv/ssb/mv_ssb_test.out b/regression-test/data/nereids_rules_p0/mv/ssb/mv_ssb_test.out index ddb04ee99511fac..b3d45a8b45bdf16 100644 --- a/regression-test/data/nereids_rules_p0/mv/ssb/mv_ssb_test.out +++ b/regression-test/data/nereids_rules_p0/mv/ssb/mv_ssb_test.out @@ -1,3955 +1,617 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !query1_1_before -- -446031203850 +\N -- !query1_1_after -- -446031203850 +\N -- !query1_2_before -- -98714004603 +\N -- !query1_2_after -- -98714004603 +\N -- !query1_3before -- -26232701005 +\N -- !query1_3_after -- -26232701005 +\N -- !query2_1before -- -244739231 1998 MFGR#1238 -251549955 1998 MFGR#1228 -290208878 1998 MFGR#1225 -296330561 1998 MFGR#123 -304873002 1998 MFGR#1214 -312440027 1998 MFGR#122 -317068168 1998 MFGR#1211 -318769573 1998 MFGR#129 -320623334 1998 MFGR#1218 -330082371 1998 MFGR#124 -332740903 1998 MFGR#1213 -339845398 1998 MFGR#1221 -341618569 1998 MFGR#1237 -341657580 1998 MFGR#126 -344575925 1998 MFGR#121 -344889822 1998 MFGR#1223 -346182862 1998 MFGR#1219 -348123961 1998 MFGR#1220 -352277781 1998 MFGR#1235 -355416161 1998 MFGR#1222 -358466340 1998 MFGR#1227 -359508497 1998 MFGR#1217 -360289624 1998 MFGR#125 -361416497 1998 MFGR#128 -361827086 1998 MFGR#1236 -366101132 1998 MFGR#1215 -374341516 1998 MFGR#1212 -377507061 1998 MFGR#127 -379133898 1998 MFGR#1216 -383138860 1998 MFGR#1229 -396906691 1998 MFGR#1224 -398944492 1998 MFGR#1231 -401953419 1997 MFGR#1238 -406967188 1998 MFGR#1233 -414151803 1998 MFGR#1239 -415312453 1998 MFGR#1240 -417152416 1998 MFGR#1210 -419415707 1998 MFGR#1226 -424062455 1998 MFGR#1232 -424306670 1997 MFGR#1213 -428867240 1998 MFGR#1234 -437181243 1998 MFGR#1230 -444614010 1993 MFGR#1238 -450402292 1997 MFGR#1225 -466676148 1996 MFGR#1238 -468535087 1992 MFGR#1230 -473007381 1993 MFGR#1225 -476864333 1996 MFGR#125 -477423770 1992 MFGR#1238 -478648074 1996 MFGR#1235 -478777095 1992 MFGR#125 -479099365 1997 MFGR#125 -483838085 1996 MFGR#1227 -493270412 1994 MFGR#1230 -496297087 1996 MFGR#1213 -499319414 1996 MFGR#1221 -499852146 1995 MFGR#1231 -501892561 1992 MFGR#1236 -502601790 1997 MFGR#1231 -506011570 1997 MFGR#1227 -508749401 1995 MFGR#129 -511971910 1997 MFGR#1214 -515571416 1993 MFGR#1233 -517956131 1995 MFGR#1220 -518444215 1996 MFGR#1220 -519659003 1995 MFGR#1219 -523879145 1996 MFGR#121 -528843086 1996 MFGR#1218 -528879998 1996 MFGR#1230 -528988960 1997 MFGR#1222 -530361300 1995 MFGR#1227 -530830127 1993 MFGR#1220 -533544350 1993 MFGR#1223 -534002330 1994 MFGR#1214 -534455976 1997 MFGR#1233 -534541525 1994 MFGR#129 -535448651 1992 MFGR#1223 -537098211 1997 MFGR#1223 -538043594 1992 MFGR#1219 -538246872 1995 MFGR#125 -540262882 1992 MFGR#1220 -542306646 1992 MFGR#128 -542569815 1994 MFGR#1237 -543248087 1997 MFGR#124 -543346337 1993 MFGR#1221 -546674347 1994 MFGR#1218 -547960244 1995 MFGR#1229 -548048395 1993 MFGR#129 -548588761 1997 MFGR#1211 -549318912 1995 MFGR#1238 -550769662 1992 MFGR#1211 -551010618 1996 MFGR#128 -555134404 1994 MFGR#1223 -556120633 1992 MFGR#1221 -557149571 1995 MFGR#1233 -558030884 1996 MFGR#126 -558646341 1994 MFGR#1216 -560488304 1994 MFGR#125 -560570630 1996 MFGR#129 -560667719 1996 MFGR#124 -561169527 1997 MFGR#1219 -562570804 1995 MFGR#1214 -562582172 1994 MFGR#1220 -564368410 1995 MFGR#1222 -564405648 1994 MFGR#121 -566217852 1996 MFGR#123 -567272942 1994 MFGR#1219 -567838207 1992 MFGR#121 -568249365 1996 MFGR#1236 -568332348 1994 MFGR#1212 -570696568 1997 MFGR#126 -570745955 1994 MFGR#1225 -570832868 1992 MFGR#1225 -571149450 1996 MFGR#1234 -571800941 1997 MFGR#1230 -572354196 1992 MFGR#124 -572568748 1995 MFGR#1226 -572847270 1995 MFGR#1237 -573510781 1994 MFGR#1238 -573693547 1994 MFGR#1228 -575779480 1993 MFGR#1235 -579855853 1994 MFGR#1239 -580327635 1993 MFGR#1231 -580449592 1994 MFGR#123 -581759388 1992 MFGR#1227 -581987352 1992 MFGR#129 -583074592 1997 MFGR#127 -583249505 1996 MFGR#1214 -583507058 1993 MFGR#1228 -585421815 1995 MFGR#124 -585578737 1997 MFGR#1228 -586246330 1996 MFGR#1219 -586845664 1994 MFGR#128 -587013207 1997 MFGR#121 -588848171 1997 MFGR#1221 -589481194 1996 MFGR#1231 -589593892 1997 MFGR#1212 -590762777 1992 MFGR#1222 -591481503 1992 MFGR#1237 -591878667 1993 MFGR#1236 -592174616 1992 MFGR#126 -592178887 1993 MFGR#1227 -592436656 1992 MFGR#1231 -592616167 1996 MFGR#1237 -593851712 1995 MFGR#1239 -594466157 1997 MFGR#1220 -595065339 1996 MFGR#1211 -595256327 1993 MFGR#1239 -598618997 1994 MFGR#1221 -598680959 1994 MFGR#1235 -599586479 1995 MFGR#1213 -600202070 1992 MFGR#1218 -600665149 1995 MFGR#1223 -601016441 1994 MFGR#1222 -601809334 1996 MFGR#1216 -602735858 1995 MFGR#1230 -602892803 1995 MFGR#1210 -603546148 1994 MFGR#1231 -605799021 1995 MFGR#126 -609618576 1993 MFGR#1237 -609855391 1996 MFGR#1228 -610663790 1992 MFGR#1210 -610756714 1997 MFGR#1239 -613289283 1995 MFGR#1212 -613885100 1992 MFGR#1233 -614061593 1992 MFGR#1226 -615726097 1994 MFGR#1236 -616224539 1993 MFGR#125 -616287892 1997 MFGR#1210 -616837237 1993 MFGR#1214 -617126754 1993 MFGR#126 -617453491 1993 MFGR#1229 -618877179 1997 MFGR#1236 -622425239 1995 MFGR#1235 -622532984 1993 MFGR#122 -622571183 1994 MFGR#127 -622744016 1997 MFGR#1229 -624031241 1992 MFGR#1213 -625534310 1993 MFGR#1230 -628762754 1996 MFGR#1223 -631620635 1994 MFGR#1211 -631772246 1997 MFGR#1215 -633152470 1992 MFGR#1215 -633357085 1995 MFGR#1228 -634565501 1995 MFGR#1236 -634687975 1993 MFGR#1215 -634743898 1993 MFGR#1211 -635741351 1995 MFGR#1221 -635873891 1996 MFGR#127 -637863868 1993 MFGR#123 -638255029 1993 MFGR#1213 -638259374 1992 MFGR#1239 -638353900 1993 MFGR#1216 -638982238 1994 MFGR#1233 -639638057 1997 MFGR#1237 -640290070 1992 MFGR#1235 -640858430 1992 MFGR#1229 -643645053 1996 MFGR#1210 -644642592 1992 MFGR#1228 -645166092 1995 MFGR#1211 -645404849 1994 MFGR#1210 -646503168 1997 MFGR#123 -646950033 1993 MFGR#1219 -646960956 1995 MFGR#128 -647918373 1994 MFGR#1229 -648160706 1993 MFGR#1210 -649205856 1992 MFGR#1212 -651707481 1994 MFGR#1227 -651935758 1993 MFGR#1234 -654400242 1994 MFGR#1215 -654438324 1993 MFGR#127 -655326672 1992 MFGR#122 -655638776 1997 MFGR#129 -657026635 1995 MFGR#1218 -657036514 1994 MFGR#126 -658959557 1996 MFGR#1229 -659884062 1994 MFGR#122 -660586237 1993 MFGR#124 -660620587 1996 MFGR#1225 -660711077 1995 MFGR#123 -663372951 1993 MFGR#1217 -663679947 1996 MFGR#1233 -664275152 1992 MFGR#1232 -664916245 1997 MFGR#122 -665978112 1995 MFGR#127 -667399281 1992 MFGR#1234 -667674729 1996 MFGR#1226 -669000972 1995 MFGR#1216 -670488468 1992 MFGR#1214 -670693719 1996 MFGR#1239 -671669586 1995 MFGR#1225 -672528755 1995 MFGR#1215 -674626440 1996 MFGR#1212 -674763166 1997 MFGR#1224 -674846781 1992 MFGR#1216 -675093435 1992 MFGR#1217 -675132692 1997 MFGR#1240 -677924656 1997 MFGR#1232 -678785857 1994 MFGR#1213 -679469356 1996 MFGR#1222 -683985855 1993 MFGR#1218 -684573322 1994 MFGR#124 -687845641 1994 MFGR#1217 -692135140 1997 MFGR#1216 -695133104 1997 MFGR#128 -697373098 1993 MFGR#1232 -701360722 1997 MFGR#1226 -702184857 1996 MFGR#1215 -702805896 1996 MFGR#1232 -703752611 1992 MFGR#1224 -704898387 1996 MFGR#1217 -706151632 1992 MFGR#127 -706469511 1995 MFGR#121 -707053720 1997 MFGR#1218 -707207888 1995 MFGR#1240 -710023059 1995 MFGR#1234 -712110492 1996 MFGR#122 -714934715 1997 MFGR#1234 -715300753 1995 MFGR#1232 -719865331 1994 MFGR#1232 -724727741 1995 MFGR#122 -724844856 1996 MFGR#1224 -725362449 1995 MFGR#1217 -731657001 1993 MFGR#128 -737422302 1994 MFGR#1224 -739036124 1993 MFGR#1226 -740479248 1992 MFGR#1240 -743247677 1994 MFGR#1234 -746302245 1994 MFGR#1226 -756921203 1993 MFGR#1222 -762700351 1995 MFGR#1224 -767151420 1997 MFGR#1235 -777994957 1997 MFGR#1217 -785639283 1993 MFGR#1212 -788730059 1993 MFGR#1240 -789755835 1992 MFGR#123 -821167950 1996 MFGR#1240 -823087702 1993 MFGR#121 -873735737 1994 MFGR#1240 -915916085 1993 MFGR#1224 +15288453 1992 MFGR#1213 +16558051 1992 MFGR#1222 +17199862 1992 MFGR#123 +19716439 1992 MFGR#1216 +21732451 1992 MFGR#129 +22246540 1992 MFGR#1215 +22759602 1992 MFGR#1218 +23120066 1992 MFGR#1210 +23318799 1992 MFGR#1219 +24245603 1992 MFGR#124 +24678908 1992 MFGR#1230 +24946678 1992 MFGR#1233 +26231337 1992 MFGR#1231 +26690787 1992 MFGR#1223 +27691433 1992 MFGR#1227 +28194770 1992 MFGR#125 +29165996 1992 MFGR#121 +30954680 1992 MFGR#1212 +31379822 1992 MFGR#1239 +31443810 1992 MFGR#1237 +32513490 1992 MFGR#1228 +32608903 1992 MFGR#1225 +35514258 1992 MFGR#1229 +35881895 1992 MFGR#128 +36027836 1992 MFGR#127 +36330900 1992 MFGR#1232 +36431683 1992 MFGR#1234 +38878674 1992 MFGR#1221 +39368479 1992 MFGR#1235 +40503844 1992 MFGR#126 +43666251 1992 MFGR#1217 +44456974 1992 MFGR#1236 +47636685 1992 MFGR#1226 +49003021 1992 MFGR#1238 +49870826 1992 MFGR#1240 +51050565 1992 MFGR#1220 +52982362 1992 MFGR#1211 +74056106 1992 MFGR#122 +76498594 1992 MFGR#1224 +7655070 1992 MFGR#1214 -- !query2_1_after -- -244739231 1998 MFGR#1238 -251549955 1998 MFGR#1228 -290208878 1998 MFGR#1225 -296330561 1998 MFGR#123 -304873002 1998 MFGR#1214 -312440027 1998 MFGR#122 -317068168 1998 MFGR#1211 -318769573 1998 MFGR#129 -320623334 1998 MFGR#1218 -330082371 1998 MFGR#124 -332740903 1998 MFGR#1213 -339845398 1998 MFGR#1221 -341618569 1998 MFGR#1237 -341657580 1998 MFGR#126 -344575925 1998 MFGR#121 -344889822 1998 MFGR#1223 -346182862 1998 MFGR#1219 -348123961 1998 MFGR#1220 -352277781 1998 MFGR#1235 -355416161 1998 MFGR#1222 -358466340 1998 MFGR#1227 -359508497 1998 MFGR#1217 -360289624 1998 MFGR#125 -361416497 1998 MFGR#128 -361827086 1998 MFGR#1236 -366101132 1998 MFGR#1215 -374341516 1998 MFGR#1212 -377507061 1998 MFGR#127 -379133898 1998 MFGR#1216 -383138860 1998 MFGR#1229 -396906691 1998 MFGR#1224 -398944492 1998 MFGR#1231 -401953419 1997 MFGR#1238 -406967188 1998 MFGR#1233 -414151803 1998 MFGR#1239 -415312453 1998 MFGR#1240 -417152416 1998 MFGR#1210 -419415707 1998 MFGR#1226 -424062455 1998 MFGR#1232 -424306670 1997 MFGR#1213 -428867240 1998 MFGR#1234 -437181243 1998 MFGR#1230 -444614010 1993 MFGR#1238 -450402292 1997 MFGR#1225 -466676148 1996 MFGR#1238 -468535087 1992 MFGR#1230 -473007381 1993 MFGR#1225 -476864333 1996 MFGR#125 -477423770 1992 MFGR#1238 -478648074 1996 MFGR#1235 -478777095 1992 MFGR#125 -479099365 1997 MFGR#125 -483838085 1996 MFGR#1227 -493270412 1994 MFGR#1230 -496297087 1996 MFGR#1213 -499319414 1996 MFGR#1221 -499852146 1995 MFGR#1231 -501892561 1992 MFGR#1236 -502601790 1997 MFGR#1231 -506011570 1997 MFGR#1227 -508749401 1995 MFGR#129 -511971910 1997 MFGR#1214 -515571416 1993 MFGR#1233 -517956131 1995 MFGR#1220 -518444215 1996 MFGR#1220 -519659003 1995 MFGR#1219 -523879145 1996 MFGR#121 -528843086 1996 MFGR#1218 -528879998 1996 MFGR#1230 -528988960 1997 MFGR#1222 -530361300 1995 MFGR#1227 -530830127 1993 MFGR#1220 -533544350 1993 MFGR#1223 -534002330 1994 MFGR#1214 -534455976 1997 MFGR#1233 -534541525 1994 MFGR#129 -535448651 1992 MFGR#1223 -537098211 1997 MFGR#1223 -538043594 1992 MFGR#1219 -538246872 1995 MFGR#125 -540262882 1992 MFGR#1220 -542306646 1992 MFGR#128 -542569815 1994 MFGR#1237 -543248087 1997 MFGR#124 -543346337 1993 MFGR#1221 -546674347 1994 MFGR#1218 -547960244 1995 MFGR#1229 -548048395 1993 MFGR#129 -548588761 1997 MFGR#1211 -549318912 1995 MFGR#1238 -550769662 1992 MFGR#1211 -551010618 1996 MFGR#128 -555134404 1994 MFGR#1223 -556120633 1992 MFGR#1221 -557149571 1995 MFGR#1233 -558030884 1996 MFGR#126 -558646341 1994 MFGR#1216 -560488304 1994 MFGR#125 -560570630 1996 MFGR#129 -560667719 1996 MFGR#124 -561169527 1997 MFGR#1219 -562570804 1995 MFGR#1214 -562582172 1994 MFGR#1220 -564368410 1995 MFGR#1222 -564405648 1994 MFGR#121 -566217852 1996 MFGR#123 -567272942 1994 MFGR#1219 -567838207 1992 MFGR#121 -568249365 1996 MFGR#1236 -568332348 1994 MFGR#1212 -570696568 1997 MFGR#126 -570745955 1994 MFGR#1225 -570832868 1992 MFGR#1225 -571149450 1996 MFGR#1234 -571800941 1997 MFGR#1230 -572354196 1992 MFGR#124 -572568748 1995 MFGR#1226 -572847270 1995 MFGR#1237 -573510781 1994 MFGR#1238 -573693547 1994 MFGR#1228 -575779480 1993 MFGR#1235 -579855853 1994 MFGR#1239 -580327635 1993 MFGR#1231 -580449592 1994 MFGR#123 -581759388 1992 MFGR#1227 -581987352 1992 MFGR#129 -583074592 1997 MFGR#127 -583249505 1996 MFGR#1214 -583507058 1993 MFGR#1228 -585421815 1995 MFGR#124 -585578737 1997 MFGR#1228 -586246330 1996 MFGR#1219 -586845664 1994 MFGR#128 -587013207 1997 MFGR#121 -588848171 1997 MFGR#1221 -589481194 1996 MFGR#1231 -589593892 1997 MFGR#1212 -590762777 1992 MFGR#1222 -591481503 1992 MFGR#1237 -591878667 1993 MFGR#1236 -592174616 1992 MFGR#126 -592178887 1993 MFGR#1227 -592436656 1992 MFGR#1231 -592616167 1996 MFGR#1237 -593851712 1995 MFGR#1239 -594466157 1997 MFGR#1220 -595065339 1996 MFGR#1211 -595256327 1993 MFGR#1239 -598618997 1994 MFGR#1221 -598680959 1994 MFGR#1235 -599586479 1995 MFGR#1213 -600202070 1992 MFGR#1218 -600665149 1995 MFGR#1223 -601016441 1994 MFGR#1222 -601809334 1996 MFGR#1216 -602735858 1995 MFGR#1230 -602892803 1995 MFGR#1210 -603546148 1994 MFGR#1231 -605799021 1995 MFGR#126 -609618576 1993 MFGR#1237 -609855391 1996 MFGR#1228 -610663790 1992 MFGR#1210 -610756714 1997 MFGR#1239 -613289283 1995 MFGR#1212 -613885100 1992 MFGR#1233 -614061593 1992 MFGR#1226 -615726097 1994 MFGR#1236 -616224539 1993 MFGR#125 -616287892 1997 MFGR#1210 -616837237 1993 MFGR#1214 -617126754 1993 MFGR#126 -617453491 1993 MFGR#1229 -618877179 1997 MFGR#1236 -622425239 1995 MFGR#1235 -622532984 1993 MFGR#122 -622571183 1994 MFGR#127 -622744016 1997 MFGR#1229 -624031241 1992 MFGR#1213 -625534310 1993 MFGR#1230 -628762754 1996 MFGR#1223 -631620635 1994 MFGR#1211 -631772246 1997 MFGR#1215 -633152470 1992 MFGR#1215 -633357085 1995 MFGR#1228 -634565501 1995 MFGR#1236 -634687975 1993 MFGR#1215 -634743898 1993 MFGR#1211 -635741351 1995 MFGR#1221 -635873891 1996 MFGR#127 -637863868 1993 MFGR#123 -638255029 1993 MFGR#1213 -638259374 1992 MFGR#1239 -638353900 1993 MFGR#1216 -638982238 1994 MFGR#1233 -639638057 1997 MFGR#1237 -640290070 1992 MFGR#1235 -640858430 1992 MFGR#1229 -643645053 1996 MFGR#1210 -644642592 1992 MFGR#1228 -645166092 1995 MFGR#1211 -645404849 1994 MFGR#1210 -646503168 1997 MFGR#123 -646950033 1993 MFGR#1219 -646960956 1995 MFGR#128 -647918373 1994 MFGR#1229 -648160706 1993 MFGR#1210 -649205856 1992 MFGR#1212 -651707481 1994 MFGR#1227 -651935758 1993 MFGR#1234 -654400242 1994 MFGR#1215 -654438324 1993 MFGR#127 -655326672 1992 MFGR#122 -655638776 1997 MFGR#129 -657026635 1995 MFGR#1218 -657036514 1994 MFGR#126 -658959557 1996 MFGR#1229 -659884062 1994 MFGR#122 -660586237 1993 MFGR#124 -660620587 1996 MFGR#1225 -660711077 1995 MFGR#123 -663372951 1993 MFGR#1217 -663679947 1996 MFGR#1233 -664275152 1992 MFGR#1232 -664916245 1997 MFGR#122 -665978112 1995 MFGR#127 -667399281 1992 MFGR#1234 -667674729 1996 MFGR#1226 -669000972 1995 MFGR#1216 -670488468 1992 MFGR#1214 -670693719 1996 MFGR#1239 -671669586 1995 MFGR#1225 -672528755 1995 MFGR#1215 -674626440 1996 MFGR#1212 -674763166 1997 MFGR#1224 -674846781 1992 MFGR#1216 -675093435 1992 MFGR#1217 -675132692 1997 MFGR#1240 -677924656 1997 MFGR#1232 -678785857 1994 MFGR#1213 -679469356 1996 MFGR#1222 -683985855 1993 MFGR#1218 -684573322 1994 MFGR#124 -687845641 1994 MFGR#1217 -692135140 1997 MFGR#1216 -695133104 1997 MFGR#128 -697373098 1993 MFGR#1232 -701360722 1997 MFGR#1226 -702184857 1996 MFGR#1215 -702805896 1996 MFGR#1232 -703752611 1992 MFGR#1224 -704898387 1996 MFGR#1217 -706151632 1992 MFGR#127 -706469511 1995 MFGR#121 -707053720 1997 MFGR#1218 -707207888 1995 MFGR#1240 -710023059 1995 MFGR#1234 -712110492 1996 MFGR#122 -714934715 1997 MFGR#1234 -715300753 1995 MFGR#1232 -719865331 1994 MFGR#1232 -724727741 1995 MFGR#122 -724844856 1996 MFGR#1224 -725362449 1995 MFGR#1217 -731657001 1993 MFGR#128 -737422302 1994 MFGR#1224 -739036124 1993 MFGR#1226 -740479248 1992 MFGR#1240 -743247677 1994 MFGR#1234 -746302245 1994 MFGR#1226 -756921203 1993 MFGR#1222 -762700351 1995 MFGR#1224 -767151420 1997 MFGR#1235 -777994957 1997 MFGR#1217 -785639283 1993 MFGR#1212 -788730059 1993 MFGR#1240 -789755835 1992 MFGR#123 -821167950 1996 MFGR#1240 -823087702 1993 MFGR#121 -873735737 1994 MFGR#1240 -915916085 1993 MFGR#1224 +15288453 1992 MFGR#1213 +16558051 1992 MFGR#1222 +17199862 1992 MFGR#123 +19716439 1992 MFGR#1216 +21732451 1992 MFGR#129 +22246540 1992 MFGR#1215 +22759602 1992 MFGR#1218 +23120066 1992 MFGR#1210 +23318799 1992 MFGR#1219 +24245603 1992 MFGR#124 +24678908 1992 MFGR#1230 +24946678 1992 MFGR#1233 +26231337 1992 MFGR#1231 +26690787 1992 MFGR#1223 +27691433 1992 MFGR#1227 +28194770 1992 MFGR#125 +29165996 1992 MFGR#121 +30954680 1992 MFGR#1212 +31379822 1992 MFGR#1239 +31443810 1992 MFGR#1237 +32513490 1992 MFGR#1228 +32608903 1992 MFGR#1225 +35514258 1992 MFGR#1229 +35881895 1992 MFGR#128 +36027836 1992 MFGR#127 +36330900 1992 MFGR#1232 +36431683 1992 MFGR#1234 +38878674 1992 MFGR#1221 +39368479 1992 MFGR#1235 +40503844 1992 MFGR#126 +43666251 1992 MFGR#1217 +44456974 1992 MFGR#1236 +47636685 1992 MFGR#1226 +49003021 1992 MFGR#1238 +49870826 1992 MFGR#1240 +51050565 1992 MFGR#1220 +52982362 1992 MFGR#1211 +74056106 1992 MFGR#122 +76498594 1992 MFGR#1224 +7655070 1992 MFGR#1214 -- !query2_2before -- -335304504 1998 MFGR#2221 -390506405 1998 MFGR#2227 -397939103 1998 MFGR#2228 -409347137 1998 MFGR#2222 -410402095 1998 MFGR#2225 -413318072 1998 MFGR#2224 -453515044 1998 MFGR#2226 -459109577 1998 MFGR#2223 -589765707 1995 MFGR#2224 -630422081 1997 MFGR#2224 -637832575 1995 MFGR#2223 -646528589 1992 MFGR#2225 -651488962 1992 MFGR#2224 -656095314 1996 MFGR#2223 -656859917 1996 MFGR#2224 -664533779 1997 MFGR#2222 -666524807 1994 MFGR#2222 -669662707 1993 MFGR#2224 -680880216 1993 MFGR#2228 -685777518 1994 MFGR#2221 -691475597 1993 MFGR#2222 -700010008 1996 MFGR#2221 -707869040 1994 MFGR#2224 -708290039 1995 MFGR#2225 -709524929 1992 MFGR#2221 -709650548 1992 MFGR#2226 -720837128 1994 MFGR#2227 -721251967 1994 MFGR#2225 -727342382 1997 MFGR#2221 -728857899 1997 MFGR#2227 -729563303 1996 MFGR#2227 -733993590 1994 MFGR#2223 -737087518 1993 MFGR#2226 -745556316 1992 MFGR#2227 -747356383 1995 MFGR#2226 -747889257 1997 MFGR#2226 -748288392 1997 MFGR#2223 -756901875 1992 MFGR#2228 -757391203 1997 MFGR#2225 -758220752 1993 MFGR#2223 -761354792 1995 MFGR#2222 -765052002 1992 MFGR#2223 -765820896 1996 MFGR#2225 -766521103 1993 MFGR#2221 -773854228 1993 MFGR#2225 -775312985 1997 MFGR#2228 -775437074 1995 MFGR#2221 -777310085 1996 MFGR#2222 -781967766 1993 MFGR#2227 -783846394 1992 MFGR#2222 -802502540 1995 MFGR#2227 -808177734 1996 MFGR#2226 -819665874 1996 MFGR#2228 -822495919 1994 MFGR#2226 -826225350 1994 MFGR#2228 -895936786 1995 MFGR#2228 +20416501 1992 MFGR#2224 +28235270 1992 MFGR#2221 +39273349 1992 MFGR#2227 +48591160 1992 MFGR#2223 +60628045 1992 MFGR#2226 +64071827 1992 MFGR#2222 +66658087 1992 MFGR#2228 +74950776 1992 MFGR#2225 -- !query2_2_after -- -335304504 1998 MFGR#2221 -390506405 1998 MFGR#2227 -397939103 1998 MFGR#2228 -409347137 1998 MFGR#2222 -410402095 1998 MFGR#2225 -413318072 1998 MFGR#2224 -453515044 1998 MFGR#2226 -459109577 1998 MFGR#2223 -589765707 1995 MFGR#2224 -630422081 1997 MFGR#2224 -637832575 1995 MFGR#2223 -646528589 1992 MFGR#2225 -651488962 1992 MFGR#2224 -656095314 1996 MFGR#2223 -656859917 1996 MFGR#2224 -664533779 1997 MFGR#2222 -666524807 1994 MFGR#2222 -669662707 1993 MFGR#2224 -680880216 1993 MFGR#2228 -685777518 1994 MFGR#2221 -691475597 1993 MFGR#2222 -700010008 1996 MFGR#2221 -707869040 1994 MFGR#2224 -708290039 1995 MFGR#2225 -709524929 1992 MFGR#2221 -709650548 1992 MFGR#2226 -720837128 1994 MFGR#2227 -721251967 1994 MFGR#2225 -727342382 1997 MFGR#2221 -728857899 1997 MFGR#2227 -729563303 1996 MFGR#2227 -733993590 1994 MFGR#2223 -737087518 1993 MFGR#2226 -745556316 1992 MFGR#2227 -747356383 1995 MFGR#2226 -747889257 1997 MFGR#2226 -748288392 1997 MFGR#2223 -756901875 1992 MFGR#2228 -757391203 1997 MFGR#2225 -758220752 1993 MFGR#2223 -761354792 1995 MFGR#2222 -765052002 1992 MFGR#2223 -765820896 1996 MFGR#2225 -766521103 1993 MFGR#2221 -773854228 1993 MFGR#2225 -775312985 1997 MFGR#2228 -775437074 1995 MFGR#2221 -777310085 1996 MFGR#2222 -781967766 1993 MFGR#2227 -783846394 1992 MFGR#2222 -802502540 1995 MFGR#2227 -808177734 1996 MFGR#2226 -819665874 1996 MFGR#2228 -822495919 1994 MFGR#2226 -826225350 1994 MFGR#2228 -895936786 1995 MFGR#2228 +20416501 1992 MFGR#2224 +28235270 1992 MFGR#2221 +39273349 1992 MFGR#2227 +48591160 1992 MFGR#2223 +60628045 1992 MFGR#2226 +64071827 1992 MFGR#2222 +66658087 1992 MFGR#2228 +74950776 1992 MFGR#2225 -- !query2_3before -- -380255731 1998 MFGR#2239 -664145134 1994 MFGR#2239 -705238959 1993 MFGR#2239 -713839336 1995 MFGR#2239 -726427486 1992 MFGR#2239 -728492865 1996 MFGR#2239 -733896532 1997 MFGR#2239 +89380397 1992 MFGR#2239 -- !query2_3_after -- -380255731 1998 MFGR#2239 -664145134 1994 MFGR#2239 -705238959 1993 MFGR#2239 -713839336 1995 MFGR#2239 -726427486 1992 MFGR#2239 -728492865 1996 MFGR#2239 -733896532 1997 MFGR#2239 +89380397 1992 MFGR#2239 -- !query3_1before -- -CHINA CHINA 1992 6456431477 -CHINA CHINA 1993 6444513787 -CHINA CHINA 1994 6648942965 -CHINA CHINA 1995 6624666612 -CHINA CHINA 1996 6591697915 -CHINA CHINA 1997 6463794795 -CHINA INDIA 1992 6189831351 -CHINA INDIA 1993 6421694121 -CHINA INDIA 1994 6297582811 -CHINA INDIA 1995 6246656374 -CHINA INDIA 1996 6153787365 -CHINA INDIA 1997 6084283983 -CHINA INDONESIA 1992 6616078869 -CHINA INDONESIA 1993 6581513366 -CHINA INDONESIA 1994 6234787896 -CHINA INDONESIA 1995 7074539444 -CHINA INDONESIA 1996 6657125075 -CHINA INDONESIA 1997 6653234696 -CHINA JAPAN 1992 6157678893 -CHINA JAPAN 1993 6195466884 -CHINA JAPAN 1994 6349037021 -CHINA JAPAN 1995 6021854130 -CHINA JAPAN 1996 6105566324 -CHINA JAPAN 1997 6284733518 -CHINA VIETNAM 1992 5412322026 -CHINA VIETNAM 1993 5885023228 -CHINA VIETNAM 1994 5885318733 -CHINA VIETNAM 1995 5489936063 -CHINA VIETNAM 1996 5434772818 -CHINA VIETNAM 1997 5908254176 -INDIA CHINA 1992 6066461811 -INDIA CHINA 1993 6209353792 -INDIA CHINA 1994 5815022714 -INDIA CHINA 1995 6010532793 -INDIA CHINA 1996 6198567819 -INDIA CHINA 1997 6153282214 -INDIA INDIA 1992 5361243755 -INDIA INDIA 1993 5571575799 -INDIA INDIA 1994 5584936515 -INDIA INDIA 1995 5565995550 -INDIA INDIA 1996 5532073103 -INDIA INDIA 1997 5693028907 -INDIA INDONESIA 1992 6238807354 -INDIA INDONESIA 1993 6331058594 -INDIA INDONESIA 1994 6168189859 -INDIA INDONESIA 1995 5975518597 -INDIA INDONESIA 1996 6189242058 -INDIA INDONESIA 1997 5983439300 -INDIA JAPAN 1992 5421959281 -INDIA JAPAN 1993 5995208985 -INDIA JAPAN 1994 5394134974 -INDIA JAPAN 1995 5469203427 -INDIA JAPAN 1996 5841664805 -INDIA JAPAN 1997 5587827913 -INDIA VIETNAM 1992 5245330621 -INDIA VIETNAM 1993 5233871487 -INDIA VIETNAM 1994 5251039815 -INDIA VIETNAM 1995 5207063284 -INDIA VIETNAM 1996 5293891531 -INDIA VIETNAM 1997 5003624837 -INDONESIA CHINA 1992 6310709533 -INDONESIA CHINA 1993 6612891249 -INDONESIA CHINA 1994 6483600910 -INDONESIA CHINA 1995 6928620620 -INDONESIA CHINA 1996 6613439112 -INDONESIA CHINA 1997 6577396314 -INDONESIA INDIA 1992 5770203218 -INDONESIA INDIA 1993 6151664306 -INDONESIA INDIA 1994 6198412245 -INDONESIA INDIA 1995 6173805183 -INDONESIA INDIA 1996 6138810863 -INDONESIA INDIA 1997 5789367105 -INDONESIA INDONESIA 1992 6905698988 -INDONESIA INDONESIA 1993 6931363262 -INDONESIA INDONESIA 1994 6315331086 -INDONESIA INDONESIA 1995 6268531321 -INDONESIA INDONESIA 1996 6750056283 -INDONESIA INDONESIA 1997 6398427821 -INDONESIA JAPAN 1992 5663765610 -INDONESIA JAPAN 1993 6147605249 -INDONESIA JAPAN 1994 6141195969 -INDONESIA JAPAN 1995 6019093871 -INDONESIA JAPAN 1996 6055638222 -INDONESIA JAPAN 1997 6001427956 -INDONESIA VIETNAM 1992 5302638372 -INDONESIA VIETNAM 1993 5840287722 -INDONESIA VIETNAM 1994 5624346368 -INDONESIA VIETNAM 1995 5574974066 -INDONESIA VIETNAM 1996 5438514387 -INDONESIA VIETNAM 1997 5665678132 -JAPAN CHINA 1992 6067100655 -JAPAN CHINA 1993 6292293995 -JAPAN CHINA 1994 6320009218 -JAPAN CHINA 1995 6519889385 -JAPAN CHINA 1996 6088985564 -JAPAN CHINA 1997 5973849687 -JAPAN INDIA 1992 5802749985 -JAPAN INDIA 1993 5823343157 -JAPAN INDIA 1994 5645818872 -JAPAN INDIA 1995 5730743126 -JAPAN INDIA 1996 5644579616 -JAPAN INDIA 1997 5796921992 -JAPAN INDONESIA 1992 6222565638 -JAPAN INDONESIA 1993 6053946313 -JAPAN INDONESIA 1994 6013034618 -JAPAN INDONESIA 1995 5947000735 -JAPAN INDONESIA 1996 6227447856 -JAPAN INDONESIA 1997 6429270126 -JAPAN JAPAN 1992 5648034711 -JAPAN JAPAN 1993 5430318583 -JAPAN JAPAN 1994 5753821482 -JAPAN JAPAN 1995 5952279129 -JAPAN JAPAN 1996 5566204795 -JAPAN JAPAN 1997 5790450039 -JAPAN VIETNAM 1992 5401692555 -JAPAN VIETNAM 1993 5406569653 -JAPAN VIETNAM 1994 5275821637 -JAPAN VIETNAM 1995 5501313775 -JAPAN VIETNAM 1996 5334277549 -JAPAN VIETNAM 1997 5249874614 -VIETNAM CHINA 1992 6332844390 -VIETNAM CHINA 1993 6413579999 -VIETNAM CHINA 1994 6709883442 -VIETNAM CHINA 1995 6332731268 -VIETNAM CHINA 1996 6428653890 -VIETNAM CHINA 1997 6164218723 -VIETNAM INDIA 1992 5698511401 -VIETNAM INDIA 1993 6161174477 -VIETNAM INDIA 1994 5789398691 -VIETNAM INDIA 1995 6198441957 -VIETNAM INDIA 1996 5807486983 -VIETNAM INDIA 1997 5730121608 -VIETNAM INDONESIA 1992 6340181707 -VIETNAM INDONESIA 1993 6511639352 -VIETNAM INDONESIA 1994 6347441931 -VIETNAM INDONESIA 1995 6057820148 -VIETNAM INDONESIA 1996 6295626732 -VIETNAM INDONESIA 1997 6363583120 -VIETNAM JAPAN 1992 5863786349 -VIETNAM JAPAN 1993 5874266867 -VIETNAM JAPAN 1994 6231973047 -VIETNAM JAPAN 1995 6263957020 -VIETNAM JAPAN 1996 5919997150 -VIETNAM JAPAN 1997 5689345230 -VIETNAM VIETNAM 1992 5111731778 -VIETNAM VIETNAM 1993 5554161183 -VIETNAM VIETNAM 1994 5418498159 -VIETNAM VIETNAM 1995 5754727607 -VIETNAM VIETNAM 1996 5860842302 -VIETNAM VIETNAM 1997 5620251324 +CHINA CHINA 1992 614550901 +CHINA INDIA 1992 348359904 +CHINA INDONESIA 1992 352903905 +CHINA JAPAN 1992 327558220 +CHINA VIETNAM 1992 324763767 +INDIA CHINA 1992 561966207 +INDIA INDIA 1992 329354089 +INDIA INDONESIA 1992 487449629 +INDIA JAPAN 1992 412186106 +INDIA VIETNAM 1992 334582962 +INDONESIA CHINA 1992 621316255 +INDONESIA INDIA 1992 310417666 +INDONESIA INDONESIA 1992 477417717 +INDONESIA JAPAN 1992 465870469 +INDONESIA VIETNAM 1992 278083418 +JAPAN CHINA 1992 637991852 +JAPAN INDIA 1992 393835589 +JAPAN INDONESIA 1992 476513261 +JAPAN JAPAN 1992 399179790 +JAPAN VIETNAM 1992 468999429 +VIETNAM CHINA 1992 621845377 +VIETNAM INDIA 1992 296225919 +VIETNAM INDONESIA 1992 462424521 +VIETNAM JAPAN 1992 395247587 +VIETNAM VIETNAM 1992 342176333 -- !query3_1_after -- -CHINA CHINA 1992 6456431477 -CHINA CHINA 1993 6444513787 -CHINA CHINA 1994 6648942965 -CHINA CHINA 1995 6624666612 -CHINA CHINA 1996 6591697915 -CHINA CHINA 1997 6463794795 -CHINA INDIA 1992 6189831351 -CHINA INDIA 1993 6421694121 -CHINA INDIA 1994 6297582811 -CHINA INDIA 1995 6246656374 -CHINA INDIA 1996 6153787365 -CHINA INDIA 1997 6084283983 -CHINA INDONESIA 1992 6616078869 -CHINA INDONESIA 1993 6581513366 -CHINA INDONESIA 1994 6234787896 -CHINA INDONESIA 1995 7074539444 -CHINA INDONESIA 1996 6657125075 -CHINA INDONESIA 1997 6653234696 -CHINA JAPAN 1992 6157678893 -CHINA JAPAN 1993 6195466884 -CHINA JAPAN 1994 6349037021 -CHINA JAPAN 1995 6021854130 -CHINA JAPAN 1996 6105566324 -CHINA JAPAN 1997 6284733518 -CHINA VIETNAM 1992 5412322026 -CHINA VIETNAM 1993 5885023228 -CHINA VIETNAM 1994 5885318733 -CHINA VIETNAM 1995 5489936063 -CHINA VIETNAM 1996 5434772818 -CHINA VIETNAM 1997 5908254176 -INDIA CHINA 1992 6066461811 -INDIA CHINA 1993 6209353792 -INDIA CHINA 1994 5815022714 -INDIA CHINA 1995 6010532793 -INDIA CHINA 1996 6198567819 -INDIA CHINA 1997 6153282214 -INDIA INDIA 1992 5361243755 -INDIA INDIA 1993 5571575799 -INDIA INDIA 1994 5584936515 -INDIA INDIA 1995 5565995550 -INDIA INDIA 1996 5532073103 -INDIA INDIA 1997 5693028907 -INDIA INDONESIA 1992 6238807354 -INDIA INDONESIA 1993 6331058594 -INDIA INDONESIA 1994 6168189859 -INDIA INDONESIA 1995 5975518597 -INDIA INDONESIA 1996 6189242058 -INDIA INDONESIA 1997 5983439300 -INDIA JAPAN 1992 5421959281 -INDIA JAPAN 1993 5995208985 -INDIA JAPAN 1994 5394134974 -INDIA JAPAN 1995 5469203427 -INDIA JAPAN 1996 5841664805 -INDIA JAPAN 1997 5587827913 -INDIA VIETNAM 1992 5245330621 -INDIA VIETNAM 1993 5233871487 -INDIA VIETNAM 1994 5251039815 -INDIA VIETNAM 1995 5207063284 -INDIA VIETNAM 1996 5293891531 -INDIA VIETNAM 1997 5003624837 -INDONESIA CHINA 1992 6310709533 -INDONESIA CHINA 1993 6612891249 -INDONESIA CHINA 1994 6483600910 -INDONESIA CHINA 1995 6928620620 -INDONESIA CHINA 1996 6613439112 -INDONESIA CHINA 1997 6577396314 -INDONESIA INDIA 1992 5770203218 -INDONESIA INDIA 1993 6151664306 -INDONESIA INDIA 1994 6198412245 -INDONESIA INDIA 1995 6173805183 -INDONESIA INDIA 1996 6138810863 -INDONESIA INDIA 1997 5789367105 -INDONESIA INDONESIA 1992 6905698988 -INDONESIA INDONESIA 1993 6931363262 -INDONESIA INDONESIA 1994 6315331086 -INDONESIA INDONESIA 1995 6268531321 -INDONESIA INDONESIA 1996 6750056283 -INDONESIA INDONESIA 1997 6398427821 -INDONESIA JAPAN 1992 5663765610 -INDONESIA JAPAN 1993 6147605249 -INDONESIA JAPAN 1994 6141195969 -INDONESIA JAPAN 1995 6019093871 -INDONESIA JAPAN 1996 6055638222 -INDONESIA JAPAN 1997 6001427956 -INDONESIA VIETNAM 1992 5302638372 -INDONESIA VIETNAM 1993 5840287722 -INDONESIA VIETNAM 1994 5624346368 -INDONESIA VIETNAM 1995 5574974066 -INDONESIA VIETNAM 1996 5438514387 -INDONESIA VIETNAM 1997 5665678132 -JAPAN CHINA 1992 6067100655 -JAPAN CHINA 1993 6292293995 -JAPAN CHINA 1994 6320009218 -JAPAN CHINA 1995 6519889385 -JAPAN CHINA 1996 6088985564 -JAPAN CHINA 1997 5973849687 -JAPAN INDIA 1992 5802749985 -JAPAN INDIA 1993 5823343157 -JAPAN INDIA 1994 5645818872 -JAPAN INDIA 1995 5730743126 -JAPAN INDIA 1996 5644579616 -JAPAN INDIA 1997 5796921992 -JAPAN INDONESIA 1992 6222565638 -JAPAN INDONESIA 1993 6053946313 -JAPAN INDONESIA 1994 6013034618 -JAPAN INDONESIA 1995 5947000735 -JAPAN INDONESIA 1996 6227447856 -JAPAN INDONESIA 1997 6429270126 -JAPAN JAPAN 1992 5648034711 -JAPAN JAPAN 1993 5430318583 -JAPAN JAPAN 1994 5753821482 -JAPAN JAPAN 1995 5952279129 -JAPAN JAPAN 1996 5566204795 -JAPAN JAPAN 1997 5790450039 -JAPAN VIETNAM 1992 5401692555 -JAPAN VIETNAM 1993 5406569653 -JAPAN VIETNAM 1994 5275821637 -JAPAN VIETNAM 1995 5501313775 -JAPAN VIETNAM 1996 5334277549 -JAPAN VIETNAM 1997 5249874614 -VIETNAM CHINA 1992 6332844390 -VIETNAM CHINA 1993 6413579999 -VIETNAM CHINA 1994 6709883442 -VIETNAM CHINA 1995 6332731268 -VIETNAM CHINA 1996 6428653890 -VIETNAM CHINA 1997 6164218723 -VIETNAM INDIA 1992 5698511401 -VIETNAM INDIA 1993 6161174477 -VIETNAM INDIA 1994 5789398691 -VIETNAM INDIA 1995 6198441957 -VIETNAM INDIA 1996 5807486983 -VIETNAM INDIA 1997 5730121608 -VIETNAM INDONESIA 1992 6340181707 -VIETNAM INDONESIA 1993 6511639352 -VIETNAM INDONESIA 1994 6347441931 -VIETNAM INDONESIA 1995 6057820148 -VIETNAM INDONESIA 1996 6295626732 -VIETNAM INDONESIA 1997 6363583120 -VIETNAM JAPAN 1992 5863786349 -VIETNAM JAPAN 1993 5874266867 -VIETNAM JAPAN 1994 6231973047 -VIETNAM JAPAN 1995 6263957020 -VIETNAM JAPAN 1996 5919997150 -VIETNAM JAPAN 1997 5689345230 -VIETNAM VIETNAM 1992 5111731778 -VIETNAM VIETNAM 1993 5554161183 -VIETNAM VIETNAM 1994 5418498159 -VIETNAM VIETNAM 1995 5754727607 -VIETNAM VIETNAM 1996 5860842302 -VIETNAM VIETNAM 1997 5620251324 +CHINA CHINA 1992 614550901 +CHINA INDIA 1992 348359904 +CHINA INDONESIA 1992 352903905 +CHINA JAPAN 1992 327558220 +CHINA VIETNAM 1992 324763767 +INDIA CHINA 1992 561966207 +INDIA INDIA 1992 329354089 +INDIA INDONESIA 1992 487449629 +INDIA JAPAN 1992 412186106 +INDIA VIETNAM 1992 334582962 +INDONESIA CHINA 1992 621316255 +INDONESIA INDIA 1992 310417666 +INDONESIA INDONESIA 1992 477417717 +INDONESIA JAPAN 1992 465870469 +INDONESIA VIETNAM 1992 278083418 +JAPAN CHINA 1992 637991852 +JAPAN INDIA 1992 393835589 +JAPAN INDONESIA 1992 476513261 +JAPAN JAPAN 1992 399179790 +JAPAN VIETNAM 1992 468999429 +VIETNAM CHINA 1992 621845377 +VIETNAM INDIA 1992 296225919 +VIETNAM INDONESIA 1992 462424521 +VIETNAM JAPAN 1992 395247587 +VIETNAM VIETNAM 1992 342176333 -- !query3_2before -- -UNITED ST0 UNITED ST0 1992 75622122 -UNITED ST0 UNITED ST0 1993 106742548 -UNITED ST0 UNITED ST0 1994 83526877 -UNITED ST0 UNITED ST0 1995 38984502 -UNITED ST0 UNITED ST0 1996 61528855 -UNITED ST0 UNITED ST0 1997 66308516 -UNITED ST0 UNITED ST1 1992 77267396 -UNITED ST0 UNITED ST1 1993 91196593 -UNITED ST0 UNITED ST1 1994 76613379 -UNITED ST0 UNITED ST1 1995 90763599 -UNITED ST0 UNITED ST1 1996 91948777 -UNITED ST0 UNITED ST1 1997 80167139 -UNITED ST0 UNITED ST2 1992 27718733 -UNITED ST0 UNITED ST2 1993 15042145 -UNITED ST0 UNITED ST2 1994 22017954 -UNITED ST0 UNITED ST2 1995 4776460 -UNITED ST0 UNITED ST2 1996 6907199 -UNITED ST0 UNITED ST2 1997 23584229 -UNITED ST0 UNITED ST3 1992 56359317 -UNITED ST0 UNITED ST3 1993 76171283 -UNITED ST0 UNITED ST3 1994 27327829 -UNITED ST0 UNITED ST3 1995 56360708 -UNITED ST0 UNITED ST3 1996 29795665 -UNITED ST0 UNITED ST3 1997 40078508 -UNITED ST0 UNITED ST4 1992 49676807 -UNITED ST0 UNITED ST4 1993 49260067 -UNITED ST0 UNITED ST4 1994 61519744 -UNITED ST0 UNITED ST4 1995 29170038 -UNITED ST0 UNITED ST4 1996 103228749 -UNITED ST0 UNITED ST4 1997 41193803 -UNITED ST0 UNITED ST5 1992 40120886 -UNITED ST0 UNITED ST5 1993 52701052 -UNITED ST0 UNITED ST5 1994 41808301 -UNITED ST0 UNITED ST5 1995 11453709 -UNITED ST0 UNITED ST5 1996 57762895 -UNITED ST0 UNITED ST5 1997 47988418 -UNITED ST0 UNITED ST6 1992 42471806 -UNITED ST0 UNITED ST6 1993 50432282 -UNITED ST0 UNITED ST6 1994 61288778 -UNITED ST0 UNITED ST6 1995 40718942 -UNITED ST0 UNITED ST6 1996 37232062 -UNITED ST0 UNITED ST6 1997 22924046 -UNITED ST0 UNITED ST7 1992 50583409 -UNITED ST0 UNITED ST7 1993 48796360 -UNITED ST0 UNITED ST7 1994 61480820 -UNITED ST0 UNITED ST7 1995 33395313 -UNITED ST0 UNITED ST7 1996 31391181 -UNITED ST0 UNITED ST7 1997 72577490 -UNITED ST0 UNITED ST8 1992 49732640 -UNITED ST0 UNITED ST8 1993 42002682 -UNITED ST0 UNITED ST8 1994 31845433 -UNITED ST0 UNITED ST8 1995 16393249 -UNITED ST0 UNITED ST8 1996 15168262 -UNITED ST0 UNITED ST8 1997 30241370 -UNITED ST0 UNITED ST9 1992 63979996 -UNITED ST0 UNITED ST9 1993 109834040 -UNITED ST0 UNITED ST9 1994 89865202 -UNITED ST0 UNITED ST9 1995 63253499 -UNITED ST0 UNITED ST9 1996 75935209 -UNITED ST0 UNITED ST9 1997 119856268 -UNITED ST1 UNITED ST0 1992 96989491 -UNITED ST1 UNITED ST0 1993 99305952 -UNITED ST1 UNITED ST0 1994 43225201 -UNITED ST1 UNITED ST0 1995 81676623 -UNITED ST1 UNITED ST0 1996 94861554 -UNITED ST1 UNITED ST0 1997 77667866 -UNITED ST1 UNITED ST1 1992 64669070 -UNITED ST1 UNITED ST1 1993 47538610 -UNITED ST1 UNITED ST1 1994 54383174 -UNITED ST1 UNITED ST1 1995 64199332 -UNITED ST1 UNITED ST1 1996 87265848 -UNITED ST1 UNITED ST1 1997 76340444 -UNITED ST1 UNITED ST2 1992 7706728 -UNITED ST1 UNITED ST2 1993 16844538 -UNITED ST1 UNITED ST2 1994 23556586 -UNITED ST1 UNITED ST2 1995 7991676 -UNITED ST1 UNITED ST2 1996 10082536 -UNITED ST1 UNITED ST2 1997 20599130 -UNITED ST1 UNITED ST3 1992 55727474 -UNITED ST1 UNITED ST3 1993 52097946 -UNITED ST1 UNITED ST3 1994 75992459 -UNITED ST1 UNITED ST3 1995 45823056 -UNITED ST1 UNITED ST3 1996 77610905 -UNITED ST1 UNITED ST3 1997 85175638 -UNITED ST1 UNITED ST4 1992 39326346 -UNITED ST1 UNITED ST4 1993 62884347 -UNITED ST1 UNITED ST4 1994 41381558 -UNITED ST1 UNITED ST4 1995 57535599 -UNITED ST1 UNITED ST4 1996 50135547 -UNITED ST1 UNITED ST4 1997 83092569 -UNITED ST1 UNITED ST5 1992 33971251 -UNITED ST1 UNITED ST5 1993 40693513 -UNITED ST1 UNITED ST5 1994 24939138 -UNITED ST1 UNITED ST5 1995 31857452 -UNITED ST1 UNITED ST5 1996 51189649 -UNITED ST1 UNITED ST5 1997 51052110 -UNITED ST1 UNITED ST6 1992 35404990 -UNITED ST1 UNITED ST6 1993 44728540 -UNITED ST1 UNITED ST6 1994 31287326 -UNITED ST1 UNITED ST6 1995 27139403 -UNITED ST1 UNITED ST6 1996 54056366 -UNITED ST1 UNITED ST6 1997 18205326 -UNITED ST1 UNITED ST7 1992 71472973 -UNITED ST1 UNITED ST7 1993 54851183 -UNITED ST1 UNITED ST7 1994 53853106 -UNITED ST1 UNITED ST7 1995 61515741 -UNITED ST1 UNITED ST7 1996 41721506 -UNITED ST1 UNITED ST7 1997 77045722 -UNITED ST1 UNITED ST8 1992 28565579 -UNITED ST1 UNITED ST8 1993 66146198 -UNITED ST1 UNITED ST8 1994 30245153 -UNITED ST1 UNITED ST8 1995 34918793 -UNITED ST1 UNITED ST8 1996 39430882 -UNITED ST1 UNITED ST8 1997 15375258 -UNITED ST1 UNITED ST9 1992 75367059 -UNITED ST1 UNITED ST9 1993 52514902 -UNITED ST1 UNITED ST9 1994 43000037 -UNITED ST1 UNITED ST9 1995 72470870 -UNITED ST1 UNITED ST9 1996 69823769 -UNITED ST1 UNITED ST9 1997 48944111 -UNITED ST2 UNITED ST0 1992 72210218 -UNITED ST2 UNITED ST0 1993 42845485 -UNITED ST2 UNITED ST0 1994 60182153 -UNITED ST2 UNITED ST0 1995 52560838 -UNITED ST2 UNITED ST0 1996 67095122 -UNITED ST2 UNITED ST0 1997 77591470 -UNITED ST2 UNITED ST1 1992 81839165 -UNITED ST2 UNITED ST1 1993 69631701 -UNITED ST2 UNITED ST1 1994 81300089 -UNITED ST2 UNITED ST1 1995 45437340 -UNITED ST2 UNITED ST1 1996 88256155 -UNITED ST2 UNITED ST1 1997 65396135 -UNITED ST2 UNITED ST2 1992 25743654 -UNITED ST2 UNITED ST2 1993 14521995 -UNITED ST2 UNITED ST2 1994 18068657 -UNITED ST2 UNITED ST2 1995 13455031 -UNITED ST2 UNITED ST2 1997 37034333 -UNITED ST2 UNITED ST3 1992 35981963 -UNITED ST2 UNITED ST3 1993 66972438 -UNITED ST2 UNITED ST3 1994 65620760 -UNITED ST2 UNITED ST3 1995 65218129 -UNITED ST2 UNITED ST3 1996 59648598 -UNITED ST2 UNITED ST3 1997 65429453 -UNITED ST2 UNITED ST4 1992 91180500 -UNITED ST2 UNITED ST4 1993 56650391 -UNITED ST2 UNITED ST4 1994 55679377 -UNITED ST2 UNITED ST4 1995 82432305 -UNITED ST2 UNITED ST4 1996 57487057 -UNITED ST2 UNITED ST4 1997 57012611 -UNITED ST2 UNITED ST5 1992 57021365 -UNITED ST2 UNITED ST5 1993 37406845 -UNITED ST2 UNITED ST5 1994 37640317 -UNITED ST2 UNITED ST5 1995 43382417 -UNITED ST2 UNITED ST5 1996 47028976 -UNITED ST2 UNITED ST5 1997 40662886 -UNITED ST2 UNITED ST6 1992 27813739 -UNITED ST2 UNITED ST6 1993 26482226 -UNITED ST2 UNITED ST6 1994 41526105 -UNITED ST2 UNITED ST6 1995 37146288 -UNITED ST2 UNITED ST6 1996 45416649 -UNITED ST2 UNITED ST6 1997 34274467 -UNITED ST2 UNITED ST7 1992 33702779 -UNITED ST2 UNITED ST7 1993 64185014 -UNITED ST2 UNITED ST7 1994 22182758 -UNITED ST2 UNITED ST7 1995 60000208 -UNITED ST2 UNITED ST7 1996 80468477 -UNITED ST2 UNITED ST7 1997 76483802 -UNITED ST2 UNITED ST8 1992 36046449 -UNITED ST2 UNITED ST8 1993 33445446 -UNITED ST2 UNITED ST8 1994 33767774 -UNITED ST2 UNITED ST8 1995 30504189 -UNITED ST2 UNITED ST8 1996 46865113 -UNITED ST2 UNITED ST8 1997 46391909 -UNITED ST2 UNITED ST9 1992 83705618 -UNITED ST2 UNITED ST9 1993 48188507 -UNITED ST2 UNITED ST9 1994 69671054 -UNITED ST2 UNITED ST9 1995 65055178 -UNITED ST2 UNITED ST9 1996 53491415 -UNITED ST2 UNITED ST9 1997 69543984 -UNITED ST3 UNITED ST0 1992 62441656 -UNITED ST3 UNITED ST0 1993 66626926 -UNITED ST3 UNITED ST0 1994 65522727 -UNITED ST3 UNITED ST0 1995 84462733 -UNITED ST3 UNITED ST0 1996 60226840 -UNITED ST3 UNITED ST0 1997 54050172 -UNITED ST3 UNITED ST1 1992 73637490 -UNITED ST3 UNITED ST1 1993 78230278 -UNITED ST3 UNITED ST1 1994 85523653 -UNITED ST3 UNITED ST1 1995 64762815 -UNITED ST3 UNITED ST1 1996 74746699 -UNITED ST3 UNITED ST1 1997 54150683 -UNITED ST3 UNITED ST2 1992 1097208 -UNITED ST3 UNITED ST2 1993 21559073 -UNITED ST3 UNITED ST2 1994 9995234 -UNITED ST3 UNITED ST2 1995 12886829 -UNITED ST3 UNITED ST2 1996 4311800 -UNITED ST3 UNITED ST2 1997 7891878 -UNITED ST3 UNITED ST3 1992 45087216 -UNITED ST3 UNITED ST3 1993 21777144 -UNITED ST3 UNITED ST3 1994 45627153 -UNITED ST3 UNITED ST3 1995 48158089 -UNITED ST3 UNITED ST3 1996 49263796 -UNITED ST3 UNITED ST3 1997 48385233 -UNITED ST3 UNITED ST4 1992 63144772 -UNITED ST3 UNITED ST4 1993 92493920 -UNITED ST3 UNITED ST4 1994 63412365 -UNITED ST3 UNITED ST4 1995 30422346 -UNITED ST3 UNITED ST4 1996 54838259 -UNITED ST3 UNITED ST4 1997 58356141 -UNITED ST3 UNITED ST5 1992 64239048 -UNITED ST3 UNITED ST5 1993 55907305 -UNITED ST3 UNITED ST5 1994 26197875 -UNITED ST3 UNITED ST5 1995 40057815 -UNITED ST3 UNITED ST5 1996 33717439 -UNITED ST3 UNITED ST5 1997 32811963 -UNITED ST3 UNITED ST6 1992 38798715 -UNITED ST3 UNITED ST6 1993 58584808 -UNITED ST3 UNITED ST6 1994 36290257 -UNITED ST3 UNITED ST6 1995 16738530 -UNITED ST3 UNITED ST6 1996 41881628 -UNITED ST3 UNITED ST6 1997 36075844 -UNITED ST3 UNITED ST7 1992 39028579 -UNITED ST3 UNITED ST7 1993 50135448 -UNITED ST3 UNITED ST7 1994 40935131 -UNITED ST3 UNITED ST7 1995 51579548 -UNITED ST3 UNITED ST7 1996 53987218 -UNITED ST3 UNITED ST7 1997 61095300 -UNITED ST3 UNITED ST8 1992 41275764 -UNITED ST3 UNITED ST8 1993 42251848 -UNITED ST3 UNITED ST8 1994 15936051 -UNITED ST3 UNITED ST8 1995 22505239 -UNITED ST3 UNITED ST8 1996 33924732 -UNITED ST3 UNITED ST8 1997 29102220 -UNITED ST3 UNITED ST9 1992 86142048 -UNITED ST3 UNITED ST9 1993 54762596 -UNITED ST3 UNITED ST9 1994 97476185 -UNITED ST3 UNITED ST9 1995 37364623 -UNITED ST3 UNITED ST9 1996 49612304 -UNITED ST3 UNITED ST9 1997 51370308 -UNITED ST4 UNITED ST0 1992 94644242 -UNITED ST4 UNITED ST0 1993 86918458 -UNITED ST4 UNITED ST0 1994 73719340 -UNITED ST4 UNITED ST0 1995 46142899 -UNITED ST4 UNITED ST0 1996 106897563 -UNITED ST4 UNITED ST0 1997 69875804 -UNITED ST4 UNITED ST1 1992 102027455 -UNITED ST4 UNITED ST1 1993 88521423 -UNITED ST4 UNITED ST1 1994 72219557 -UNITED ST4 UNITED ST1 1995 90878653 -UNITED ST4 UNITED ST1 1996 72819867 -UNITED ST4 UNITED ST1 1997 84611188 -UNITED ST4 UNITED ST2 1992 12061845 -UNITED ST4 UNITED ST2 1993 27012678 -UNITED ST4 UNITED ST2 1994 23035892 -UNITED ST4 UNITED ST2 1995 22787003 -UNITED ST4 UNITED ST2 1996 12451867 -UNITED ST4 UNITED ST2 1997 2848157 -UNITED ST4 UNITED ST3 1992 69001787 -UNITED ST4 UNITED ST3 1993 33129412 -UNITED ST4 UNITED ST3 1994 31599433 -UNITED ST4 UNITED ST3 1995 37356396 -UNITED ST4 UNITED ST3 1996 46460940 -UNITED ST4 UNITED ST3 1997 59634153 -UNITED ST4 UNITED ST4 1992 70111335 -UNITED ST4 UNITED ST4 1993 98150465 -UNITED ST4 UNITED ST4 1994 76837381 -UNITED ST4 UNITED ST4 1995 74587071 -UNITED ST4 UNITED ST4 1996 32787995 -UNITED ST4 UNITED ST4 1997 62532346 -UNITED ST4 UNITED ST5 1992 65057872 -UNITED ST4 UNITED ST5 1993 29119594 -UNITED ST4 UNITED ST5 1994 59397028 -UNITED ST4 UNITED ST5 1995 62081505 -UNITED ST4 UNITED ST5 1996 42808412 -UNITED ST4 UNITED ST5 1997 31995868 -UNITED ST4 UNITED ST6 1992 81886374 -UNITED ST4 UNITED ST6 1993 30699255 -UNITED ST4 UNITED ST6 1994 58633482 -UNITED ST4 UNITED ST6 1995 38109524 -UNITED ST4 UNITED ST6 1996 62189328 -UNITED ST4 UNITED ST6 1997 46236849 -UNITED ST4 UNITED ST7 1992 62529376 -UNITED ST4 UNITED ST7 1993 88172198 -UNITED ST4 UNITED ST7 1994 76954140 -UNITED ST4 UNITED ST7 1995 69264407 -UNITED ST4 UNITED ST7 1996 83677814 -UNITED ST4 UNITED ST7 1997 84606996 -UNITED ST4 UNITED ST8 1992 21928442 -UNITED ST4 UNITED ST8 1993 41657582 -UNITED ST4 UNITED ST8 1994 20685352 -UNITED ST4 UNITED ST8 1995 23053236 -UNITED ST4 UNITED ST8 1996 53035662 -UNITED ST4 UNITED ST8 1997 32599736 -UNITED ST4 UNITED ST9 1992 67837970 -UNITED ST4 UNITED ST9 1993 73296959 -UNITED ST4 UNITED ST9 1994 83822393 -UNITED ST4 UNITED ST9 1995 99490068 -UNITED ST4 UNITED ST9 1996 54134840 -UNITED ST4 UNITED ST9 1997 79697927 -UNITED ST5 UNITED ST0 1992 90432300 -UNITED ST5 UNITED ST0 1993 106822597 -UNITED ST5 UNITED ST0 1994 78648057 -UNITED ST5 UNITED ST0 1995 77615746 -UNITED ST5 UNITED ST0 1996 104501979 -UNITED ST5 UNITED ST0 1997 121457766 -UNITED ST5 UNITED ST1 1992 59945287 -UNITED ST5 UNITED ST1 1993 71454623 -UNITED ST5 UNITED ST1 1994 73508418 -UNITED ST5 UNITED ST1 1995 119793669 -UNITED ST5 UNITED ST1 1996 65346389 -UNITED ST5 UNITED ST1 1997 106900161 -UNITED ST5 UNITED ST2 1992 20495817 -UNITED ST5 UNITED ST2 1993 20610242 -UNITED ST5 UNITED ST2 1994 7162796 -UNITED ST5 UNITED ST2 1995 14951023 -UNITED ST5 UNITED ST2 1996 8544233 -UNITED ST5 UNITED ST2 1997 13863438 -UNITED ST5 UNITED ST3 1992 53963553 -UNITED ST5 UNITED ST3 1993 34031811 -UNITED ST5 UNITED ST3 1994 65334911 -UNITED ST5 UNITED ST3 1995 78099130 -UNITED ST5 UNITED ST3 1996 32919926 -UNITED ST5 UNITED ST3 1997 56789932 -UNITED ST5 UNITED ST4 1992 57625627 -UNITED ST5 UNITED ST4 1993 63415639 -UNITED ST5 UNITED ST4 1994 51799593 -UNITED ST5 UNITED ST4 1995 81016761 -UNITED ST5 UNITED ST4 1996 33424859 -UNITED ST5 UNITED ST4 1997 103432927 -UNITED ST5 UNITED ST5 1992 29279568 -UNITED ST5 UNITED ST5 1993 50357189 -UNITED ST5 UNITED ST5 1994 56235213 -UNITED ST5 UNITED ST5 1995 50394007 -UNITED ST5 UNITED ST5 1996 43020722 -UNITED ST5 UNITED ST5 1997 32568888 -UNITED ST5 UNITED ST6 1992 62628093 -UNITED ST5 UNITED ST6 1993 25357679 -UNITED ST5 UNITED ST6 1994 38658451 -UNITED ST5 UNITED ST6 1995 30204737 -UNITED ST5 UNITED ST6 1996 38332044 -UNITED ST5 UNITED ST6 1997 49030016 -UNITED ST5 UNITED ST7 1992 68333651 -UNITED ST5 UNITED ST7 1993 52745538 -UNITED ST5 UNITED ST7 1994 89089961 -UNITED ST5 UNITED ST7 1995 53740759 -UNITED ST5 UNITED ST7 1996 40946330 -UNITED ST5 UNITED ST7 1997 54534315 -UNITED ST5 UNITED ST8 1992 33878053 -UNITED ST5 UNITED ST8 1993 74359279 -UNITED ST5 UNITED ST8 1994 18931891 -UNITED ST5 UNITED ST8 1995 13346932 -UNITED ST5 UNITED ST8 1996 32415682 -UNITED ST5 UNITED ST8 1997 32910946 -UNITED ST5 UNITED ST9 1992 50191091 -UNITED ST5 UNITED ST9 1993 76286539 -UNITED ST5 UNITED ST9 1994 53367331 -UNITED ST5 UNITED ST9 1995 62364897 -UNITED ST5 UNITED ST9 1996 72614114 -UNITED ST5 UNITED ST9 1997 97392640 -UNITED ST6 UNITED ST0 1992 78696956 -UNITED ST6 UNITED ST0 1993 68247539 -UNITED ST6 UNITED ST0 1994 53728853 -UNITED ST6 UNITED ST0 1995 91718471 -UNITED ST6 UNITED ST0 1996 132286763 -UNITED ST6 UNITED ST0 1997 55824919 -UNITED ST6 UNITED ST1 1992 69408688 -UNITED ST6 UNITED ST1 1993 83345939 -UNITED ST6 UNITED ST1 1994 96937147 -UNITED ST6 UNITED ST1 1995 38328352 -UNITED ST6 UNITED ST1 1996 89705870 -UNITED ST6 UNITED ST1 1997 55117709 -UNITED ST6 UNITED ST2 1992 11820698 -UNITED ST6 UNITED ST2 1993 18814367 -UNITED ST6 UNITED ST2 1994 25809000 -UNITED ST6 UNITED ST2 1995 11363551 -UNITED ST6 UNITED ST2 1996 20447847 -UNITED ST6 UNITED ST2 1997 14746680 -UNITED ST6 UNITED ST3 1992 78778218 -UNITED ST6 UNITED ST3 1993 83417161 -UNITED ST6 UNITED ST3 1994 56123238 -UNITED ST6 UNITED ST3 1995 67513062 -UNITED ST6 UNITED ST3 1996 82168229 -UNITED ST6 UNITED ST3 1997 69725506 -UNITED ST6 UNITED ST4 1992 71086944 -UNITED ST6 UNITED ST4 1993 69646913 -UNITED ST6 UNITED ST4 1994 59829989 -UNITED ST6 UNITED ST4 1995 47697241 -UNITED ST6 UNITED ST4 1996 69296005 -UNITED ST6 UNITED ST4 1997 57215022 -UNITED ST6 UNITED ST5 1992 85882966 -UNITED ST6 UNITED ST5 1993 34229082 -UNITED ST6 UNITED ST5 1994 66461534 -UNITED ST6 UNITED ST5 1995 45428152 -UNITED ST6 UNITED ST5 1996 46030104 -UNITED ST6 UNITED ST5 1997 37571788 -UNITED ST6 UNITED ST6 1992 18137105 -UNITED ST6 UNITED ST6 1993 33050663 -UNITED ST6 UNITED ST6 1994 46199722 -UNITED ST6 UNITED ST6 1995 50954677 -UNITED ST6 UNITED ST6 1996 42923032 -UNITED ST6 UNITED ST6 1997 24983291 -UNITED ST6 UNITED ST7 1992 87658267 -UNITED ST6 UNITED ST7 1993 78197294 -UNITED ST6 UNITED ST7 1994 37057808 -UNITED ST6 UNITED ST7 1995 23833977 -UNITED ST6 UNITED ST7 1996 57090619 -UNITED ST6 UNITED ST7 1997 49795849 -UNITED ST6 UNITED ST8 1992 42463690 -UNITED ST6 UNITED ST8 1993 36456741 -UNITED ST6 UNITED ST8 1994 26592996 -UNITED ST6 UNITED ST8 1995 41973920 -UNITED ST6 UNITED ST8 1996 52136784 -UNITED ST6 UNITED ST8 1997 37988445 -UNITED ST6 UNITED ST9 1992 40591721 -UNITED ST6 UNITED ST9 1993 54900942 -UNITED ST6 UNITED ST9 1994 94683309 -UNITED ST6 UNITED ST9 1995 63457105 -UNITED ST6 UNITED ST9 1996 63256182 -UNITED ST6 UNITED ST9 1997 58231263 -UNITED ST7 UNITED ST0 1992 96078010 -UNITED ST7 UNITED ST0 1993 109788024 -UNITED ST7 UNITED ST0 1994 82265536 -UNITED ST7 UNITED ST0 1995 98031295 -UNITED ST7 UNITED ST0 1996 75602445 -UNITED ST7 UNITED ST0 1997 69354312 -UNITED ST7 UNITED ST1 1992 50167208 -UNITED ST7 UNITED ST1 1993 34391159 -UNITED ST7 UNITED ST1 1994 75320799 -UNITED ST7 UNITED ST1 1995 79964362 -UNITED ST7 UNITED ST1 1996 48784175 -UNITED ST7 UNITED ST1 1997 101326940 -UNITED ST7 UNITED ST2 1992 4943204 -UNITED ST7 UNITED ST2 1993 20219304 -UNITED ST7 UNITED ST2 1994 19655081 -UNITED ST7 UNITED ST2 1995 3973565 -UNITED ST7 UNITED ST2 1996 8390470 -UNITED ST7 UNITED ST2 1997 31409590 -UNITED ST7 UNITED ST3 1992 83765666 -UNITED ST7 UNITED ST3 1993 39523861 -UNITED ST7 UNITED ST3 1994 29424840 -UNITED ST7 UNITED ST3 1995 43699734 -UNITED ST7 UNITED ST3 1996 52356886 -UNITED ST7 UNITED ST3 1997 35168922 -UNITED ST7 UNITED ST4 1992 64900888 -UNITED ST7 UNITED ST4 1993 61656613 -UNITED ST7 UNITED ST4 1994 64778715 -UNITED ST7 UNITED ST4 1995 49899545 -UNITED ST7 UNITED ST4 1996 77941011 -UNITED ST7 UNITED ST4 1997 60123494 -UNITED ST7 UNITED ST5 1992 57566802 -UNITED ST7 UNITED ST5 1993 44763692 -UNITED ST7 UNITED ST5 1994 45895364 -UNITED ST7 UNITED ST5 1995 39015293 -UNITED ST7 UNITED ST5 1996 32784224 -UNITED ST7 UNITED ST5 1997 36889736 -UNITED ST7 UNITED ST6 1992 65513205 -UNITED ST7 UNITED ST6 1993 27100502 -UNITED ST7 UNITED ST6 1994 24631937 -UNITED ST7 UNITED ST6 1995 65020671 -UNITED ST7 UNITED ST6 1996 36535628 -UNITED ST7 UNITED ST6 1997 27539931 -UNITED ST7 UNITED ST7 1992 60121386 -UNITED ST7 UNITED ST7 1993 68109820 -UNITED ST7 UNITED ST7 1994 46446442 -UNITED ST7 UNITED ST7 1995 86571624 -UNITED ST7 UNITED ST7 1996 74040644 -UNITED ST7 UNITED ST7 1997 42566603 -UNITED ST7 UNITED ST8 1992 11895702 -UNITED ST7 UNITED ST8 1993 43902628 -UNITED ST7 UNITED ST8 1994 42776467 -UNITED ST7 UNITED ST8 1995 26444490 -UNITED ST7 UNITED ST8 1996 24348851 -UNITED ST7 UNITED ST8 1997 45476266 -UNITED ST7 UNITED ST9 1992 60326042 -UNITED ST7 UNITED ST9 1993 80310701 -UNITED ST7 UNITED ST9 1994 50144085 -UNITED ST7 UNITED ST9 1995 75317709 -UNITED ST7 UNITED ST9 1996 25028091 -UNITED ST7 UNITED ST9 1997 55039970 -UNITED ST8 UNITED ST0 1992 44368839 -UNITED ST8 UNITED ST0 1993 88356371 -UNITED ST8 UNITED ST0 1994 99220789 -UNITED ST8 UNITED ST0 1995 55231664 -UNITED ST8 UNITED ST0 1996 39845427 -UNITED ST8 UNITED ST0 1997 89434845 -UNITED ST8 UNITED ST1 1992 35680752 -UNITED ST8 UNITED ST1 1993 44415434 -UNITED ST8 UNITED ST1 1994 40152096 -UNITED ST8 UNITED ST1 1995 34217780 -UNITED ST8 UNITED ST1 1996 78971756 -UNITED ST8 UNITED ST1 1997 43284139 -UNITED ST8 UNITED ST2 1992 10971561 -UNITED ST8 UNITED ST2 1993 9984366 -UNITED ST8 UNITED ST2 1994 3789626 -UNITED ST8 UNITED ST2 1995 18282990 -UNITED ST8 UNITED ST2 1996 7028334 -UNITED ST8 UNITED ST2 1997 12053706 -UNITED ST8 UNITED ST3 1992 30949324 -UNITED ST8 UNITED ST3 1993 54049874 -UNITED ST8 UNITED ST3 1994 55203841 -UNITED ST8 UNITED ST3 1995 29349938 -UNITED ST8 UNITED ST3 1996 39535940 -UNITED ST8 UNITED ST3 1997 66918394 -UNITED ST8 UNITED ST4 1992 46757019 -UNITED ST8 UNITED ST4 1993 51419356 -UNITED ST8 UNITED ST4 1994 62141681 -UNITED ST8 UNITED ST4 1995 19287311 -UNITED ST8 UNITED ST4 1996 51033054 -UNITED ST8 UNITED ST4 1997 36600584 -UNITED ST8 UNITED ST5 1992 35187610 -UNITED ST8 UNITED ST5 1993 52237837 -UNITED ST8 UNITED ST5 1994 44553056 -UNITED ST8 UNITED ST5 1995 44994863 -UNITED ST8 UNITED ST5 1996 43480473 -UNITED ST8 UNITED ST5 1997 26673743 -UNITED ST8 UNITED ST6 1992 60540440 -UNITED ST8 UNITED ST6 1993 54633062 -UNITED ST8 UNITED ST6 1994 36776569 -UNITED ST8 UNITED ST6 1995 58990404 -UNITED ST8 UNITED ST6 1996 59746655 -UNITED ST8 UNITED ST6 1997 20073000 -UNITED ST8 UNITED ST7 1992 41897901 -UNITED ST8 UNITED ST7 1993 36902218 -UNITED ST8 UNITED ST7 1994 37600687 -UNITED ST8 UNITED ST7 1995 30918202 -UNITED ST8 UNITED ST7 1996 57523925 -UNITED ST8 UNITED ST7 1997 40878429 -UNITED ST8 UNITED ST8 1992 29309108 -UNITED ST8 UNITED ST8 1993 38790235 -UNITED ST8 UNITED ST8 1994 10023412 -UNITED ST8 UNITED ST8 1995 11285867 -UNITED ST8 UNITED ST8 1996 16340603 -UNITED ST8 UNITED ST8 1997 40749400 -UNITED ST8 UNITED ST9 1992 22684471 -UNITED ST8 UNITED ST9 1993 38516361 -UNITED ST8 UNITED ST9 1994 58666906 -UNITED ST8 UNITED ST9 1995 88855089 -UNITED ST8 UNITED ST9 1996 60466691 -UNITED ST8 UNITED ST9 1997 45852796 -UNITED ST9 UNITED ST0 1992 94813294 -UNITED ST9 UNITED ST0 1993 32653305 -UNITED ST9 UNITED ST0 1994 87670573 -UNITED ST9 UNITED ST0 1995 51548621 -UNITED ST9 UNITED ST0 1996 49590701 -UNITED ST9 UNITED ST0 1997 66066791 -UNITED ST9 UNITED ST1 1992 80795302 -UNITED ST9 UNITED ST1 1993 74059419 -UNITED ST9 UNITED ST1 1994 90440948 -UNITED ST9 UNITED ST1 1995 82938967 -UNITED ST9 UNITED ST1 1996 101983827 -UNITED ST9 UNITED ST1 1997 62418406 -UNITED ST9 UNITED ST2 1992 11623177 -UNITED ST9 UNITED ST2 1993 17718390 -UNITED ST9 UNITED ST2 1994 9700989 -UNITED ST9 UNITED ST2 1995 12653362 -UNITED ST9 UNITED ST2 1996 8108035 -UNITED ST9 UNITED ST2 1997 2661652 -UNITED ST9 UNITED ST3 1992 47681165 -UNITED ST9 UNITED ST3 1993 46943647 -UNITED ST9 UNITED ST3 1994 35846500 -UNITED ST9 UNITED ST3 1995 17539102 -UNITED ST9 UNITED ST3 1996 47021018 -UNITED ST9 UNITED ST3 1997 46121724 -UNITED ST9 UNITED ST4 1992 61489513 -UNITED ST9 UNITED ST4 1993 30644687 -UNITED ST9 UNITED ST4 1994 43910761 -UNITED ST9 UNITED ST4 1995 72525686 -UNITED ST9 UNITED ST4 1996 69616614 -UNITED ST9 UNITED ST4 1997 31765478 -UNITED ST9 UNITED ST5 1992 28075004 -UNITED ST9 UNITED ST5 1993 39685269 -UNITED ST9 UNITED ST5 1994 37146726 -UNITED ST9 UNITED ST5 1995 36009924 -UNITED ST9 UNITED ST5 1996 52825922 -UNITED ST9 UNITED ST5 1997 56091920 -UNITED ST9 UNITED ST6 1992 15879568 -UNITED ST9 UNITED ST6 1993 43867160 -UNITED ST9 UNITED ST6 1994 29212927 -UNITED ST9 UNITED ST6 1995 43482494 -UNITED ST9 UNITED ST6 1996 48685063 -UNITED ST9 UNITED ST6 1997 37596882 -UNITED ST9 UNITED ST7 1992 56883359 -UNITED ST9 UNITED ST7 1993 66625483 -UNITED ST9 UNITED ST7 1994 31586173 -UNITED ST9 UNITED ST7 1995 46464589 -UNITED ST9 UNITED ST7 1996 48798759 -UNITED ST9 UNITED ST7 1997 69052630 -UNITED ST9 UNITED ST8 1992 21386100 -UNITED ST9 UNITED ST8 1993 37184696 -UNITED ST9 UNITED ST8 1994 39278808 -UNITED ST9 UNITED ST8 1995 21292527 -UNITED ST9 UNITED ST8 1996 36112301 -UNITED ST9 UNITED ST8 1997 31272385 -UNITED ST9 UNITED ST9 1992 36048343 -UNITED ST9 UNITED ST9 1993 63260763 -UNITED ST9 UNITED ST9 1994 51980849 -UNITED ST9 UNITED ST9 1995 76515958 -UNITED ST9 UNITED ST9 1996 77301740 -UNITED ST9 UNITED ST9 1997 106222795 +UNITED ST0 UNITED ST0 1992 18293852 +UNITED ST0 UNITED ST3 1992 3201624 +UNITED ST0 UNITED ST7 1992 7137641 +UNITED ST1 UNITED ST0 1992 10878084 +UNITED ST1 UNITED ST2 1992 8004700 +UNITED ST1 UNITED ST3 1992 13938002 +UNITED ST1 UNITED ST6 1992 11000283 +UNITED ST1 UNITED ST7 1992 17410800 +UNITED ST1 UNITED ST9 1992 25644597 +UNITED ST2 UNITED ST0 1992 23943154 +UNITED ST2 UNITED ST2 1992 8313714 +UNITED ST2 UNITED ST3 1992 15331073 +UNITED ST2 UNITED ST7 1992 7759164 +UNITED ST2 UNITED ST9 1992 12343455 +UNITED ST3 UNITED ST3 1992 4080199 +UNITED ST3 UNITED ST9 1992 2614811 +UNITED ST4 UNITED ST0 1992 34626982 +UNITED ST4 UNITED ST2 1992 11696334 +UNITED ST4 UNITED ST3 1992 29767238 +UNITED ST4 UNITED ST6 1992 11369008 +UNITED ST4 UNITED ST7 1992 10151573 +UNITED ST4 UNITED ST9 1992 21189183 +UNITED ST5 UNITED ST0 1992 12301234 +UNITED ST5 UNITED ST2 1992 9917834 +UNITED ST5 UNITED ST3 1992 5862031 +UNITED ST5 UNITED ST6 1992 12398029 +UNITED ST5 UNITED ST7 1992 3936271 +UNITED ST5 UNITED ST9 1992 14448179 +UNITED ST6 UNITED ST0 1992 11900889 +UNITED ST6 UNITED ST2 1992 291566 +UNITED ST6 UNITED ST3 1992 1873819 +UNITED ST6 UNITED ST6 1992 8685228 +UNITED ST6 UNITED ST9 1992 6597261 +UNITED ST7 UNITED ST0 1992 1882015 +UNITED ST7 UNITED ST2 1992 6125476 +UNITED ST7 UNITED ST3 1992 17996772 +UNITED ST7 UNITED ST6 1992 6058017 +UNITED ST7 UNITED ST7 1992 9715656 +UNITED ST7 UNITED ST9 1992 5403152 +UNITED ST8 UNITED ST0 1992 3574169 +UNITED ST8 UNITED ST7 1992 2373825 +UNITED ST8 UNITED ST9 1992 5690491 +UNITED ST9 UNITED ST0 1992 4816370 +UNITED ST9 UNITED ST2 1992 12370917 +UNITED ST9 UNITED ST3 1992 17863433 +UNITED ST9 UNITED ST6 1992 2066609 +UNITED ST9 UNITED ST7 1992 6703890 +UNITED ST9 UNITED ST9 1992 4234523 -- !query3_2_after -- -UNITED ST0 UNITED ST0 1992 75622122 -UNITED ST0 UNITED ST0 1993 106742548 -UNITED ST0 UNITED ST0 1994 83526877 -UNITED ST0 UNITED ST0 1995 38984502 -UNITED ST0 UNITED ST0 1996 61528855 -UNITED ST0 UNITED ST0 1997 66308516 -UNITED ST0 UNITED ST1 1992 77267396 -UNITED ST0 UNITED ST1 1993 91196593 -UNITED ST0 UNITED ST1 1994 76613379 -UNITED ST0 UNITED ST1 1995 90763599 -UNITED ST0 UNITED ST1 1996 91948777 -UNITED ST0 UNITED ST1 1997 80167139 -UNITED ST0 UNITED ST2 1992 27718733 -UNITED ST0 UNITED ST2 1993 15042145 -UNITED ST0 UNITED ST2 1994 22017954 -UNITED ST0 UNITED ST2 1995 4776460 -UNITED ST0 UNITED ST2 1996 6907199 -UNITED ST0 UNITED ST2 1997 23584229 -UNITED ST0 UNITED ST3 1992 56359317 -UNITED ST0 UNITED ST3 1993 76171283 -UNITED ST0 UNITED ST3 1994 27327829 -UNITED ST0 UNITED ST3 1995 56360708 -UNITED ST0 UNITED ST3 1996 29795665 -UNITED ST0 UNITED ST3 1997 40078508 -UNITED ST0 UNITED ST4 1992 49676807 -UNITED ST0 UNITED ST4 1993 49260067 -UNITED ST0 UNITED ST4 1994 61519744 -UNITED ST0 UNITED ST4 1995 29170038 -UNITED ST0 UNITED ST4 1996 103228749 -UNITED ST0 UNITED ST4 1997 41193803 -UNITED ST0 UNITED ST5 1992 40120886 -UNITED ST0 UNITED ST5 1993 52701052 -UNITED ST0 UNITED ST5 1994 41808301 -UNITED ST0 UNITED ST5 1995 11453709 -UNITED ST0 UNITED ST5 1996 57762895 -UNITED ST0 UNITED ST5 1997 47988418 -UNITED ST0 UNITED ST6 1992 42471806 -UNITED ST0 UNITED ST6 1993 50432282 -UNITED ST0 UNITED ST6 1994 61288778 -UNITED ST0 UNITED ST6 1995 40718942 -UNITED ST0 UNITED ST6 1996 37232062 -UNITED ST0 UNITED ST6 1997 22924046 -UNITED ST0 UNITED ST7 1992 50583409 -UNITED ST0 UNITED ST7 1993 48796360 -UNITED ST0 UNITED ST7 1994 61480820 -UNITED ST0 UNITED ST7 1995 33395313 -UNITED ST0 UNITED ST7 1996 31391181 -UNITED ST0 UNITED ST7 1997 72577490 -UNITED ST0 UNITED ST8 1992 49732640 -UNITED ST0 UNITED ST8 1993 42002682 -UNITED ST0 UNITED ST8 1994 31845433 -UNITED ST0 UNITED ST8 1995 16393249 -UNITED ST0 UNITED ST8 1996 15168262 -UNITED ST0 UNITED ST8 1997 30241370 -UNITED ST0 UNITED ST9 1992 63979996 -UNITED ST0 UNITED ST9 1993 109834040 -UNITED ST0 UNITED ST9 1994 89865202 -UNITED ST0 UNITED ST9 1995 63253499 -UNITED ST0 UNITED ST9 1996 75935209 -UNITED ST0 UNITED ST9 1997 119856268 -UNITED ST1 UNITED ST0 1992 96989491 -UNITED ST1 UNITED ST0 1993 99305952 -UNITED ST1 UNITED ST0 1994 43225201 -UNITED ST1 UNITED ST0 1995 81676623 -UNITED ST1 UNITED ST0 1996 94861554 -UNITED ST1 UNITED ST0 1997 77667866 -UNITED ST1 UNITED ST1 1992 64669070 -UNITED ST1 UNITED ST1 1993 47538610 -UNITED ST1 UNITED ST1 1994 54383174 -UNITED ST1 UNITED ST1 1995 64199332 -UNITED ST1 UNITED ST1 1996 87265848 -UNITED ST1 UNITED ST1 1997 76340444 -UNITED ST1 UNITED ST2 1992 7706728 -UNITED ST1 UNITED ST2 1993 16844538 -UNITED ST1 UNITED ST2 1994 23556586 -UNITED ST1 UNITED ST2 1995 7991676 -UNITED ST1 UNITED ST2 1996 10082536 -UNITED ST1 UNITED ST2 1997 20599130 -UNITED ST1 UNITED ST3 1992 55727474 -UNITED ST1 UNITED ST3 1993 52097946 -UNITED ST1 UNITED ST3 1994 75992459 -UNITED ST1 UNITED ST3 1995 45823056 -UNITED ST1 UNITED ST3 1996 77610905 -UNITED ST1 UNITED ST3 1997 85175638 -UNITED ST1 UNITED ST4 1992 39326346 -UNITED ST1 UNITED ST4 1993 62884347 -UNITED ST1 UNITED ST4 1994 41381558 -UNITED ST1 UNITED ST4 1995 57535599 -UNITED ST1 UNITED ST4 1996 50135547 -UNITED ST1 UNITED ST4 1997 83092569 -UNITED ST1 UNITED ST5 1992 33971251 -UNITED ST1 UNITED ST5 1993 40693513 -UNITED ST1 UNITED ST5 1994 24939138 -UNITED ST1 UNITED ST5 1995 31857452 -UNITED ST1 UNITED ST5 1996 51189649 -UNITED ST1 UNITED ST5 1997 51052110 -UNITED ST1 UNITED ST6 1992 35404990 -UNITED ST1 UNITED ST6 1993 44728540 -UNITED ST1 UNITED ST6 1994 31287326 -UNITED ST1 UNITED ST6 1995 27139403 -UNITED ST1 UNITED ST6 1996 54056366 -UNITED ST1 UNITED ST6 1997 18205326 -UNITED ST1 UNITED ST7 1992 71472973 -UNITED ST1 UNITED ST7 1993 54851183 -UNITED ST1 UNITED ST7 1994 53853106 -UNITED ST1 UNITED ST7 1995 61515741 -UNITED ST1 UNITED ST7 1996 41721506 -UNITED ST1 UNITED ST7 1997 77045722 -UNITED ST1 UNITED ST8 1992 28565579 -UNITED ST1 UNITED ST8 1993 66146198 -UNITED ST1 UNITED ST8 1994 30245153 -UNITED ST1 UNITED ST8 1995 34918793 -UNITED ST1 UNITED ST8 1996 39430882 -UNITED ST1 UNITED ST8 1997 15375258 -UNITED ST1 UNITED ST9 1992 75367059 -UNITED ST1 UNITED ST9 1993 52514902 -UNITED ST1 UNITED ST9 1994 43000037 -UNITED ST1 UNITED ST9 1995 72470870 -UNITED ST1 UNITED ST9 1996 69823769 -UNITED ST1 UNITED ST9 1997 48944111 -UNITED ST2 UNITED ST0 1992 72210218 -UNITED ST2 UNITED ST0 1993 42845485 -UNITED ST2 UNITED ST0 1994 60182153 -UNITED ST2 UNITED ST0 1995 52560838 -UNITED ST2 UNITED ST0 1996 67095122 -UNITED ST2 UNITED ST0 1997 77591470 -UNITED ST2 UNITED ST1 1992 81839165 -UNITED ST2 UNITED ST1 1993 69631701 -UNITED ST2 UNITED ST1 1994 81300089 -UNITED ST2 UNITED ST1 1995 45437340 -UNITED ST2 UNITED ST1 1996 88256155 -UNITED ST2 UNITED ST1 1997 65396135 -UNITED ST2 UNITED ST2 1992 25743654 -UNITED ST2 UNITED ST2 1993 14521995 -UNITED ST2 UNITED ST2 1994 18068657 -UNITED ST2 UNITED ST2 1995 13455031 -UNITED ST2 UNITED ST2 1997 37034333 -UNITED ST2 UNITED ST3 1992 35981963 -UNITED ST2 UNITED ST3 1993 66972438 -UNITED ST2 UNITED ST3 1994 65620760 -UNITED ST2 UNITED ST3 1995 65218129 -UNITED ST2 UNITED ST3 1996 59648598 -UNITED ST2 UNITED ST3 1997 65429453 -UNITED ST2 UNITED ST4 1992 91180500 -UNITED ST2 UNITED ST4 1993 56650391 -UNITED ST2 UNITED ST4 1994 55679377 -UNITED ST2 UNITED ST4 1995 82432305 -UNITED ST2 UNITED ST4 1996 57487057 -UNITED ST2 UNITED ST4 1997 57012611 -UNITED ST2 UNITED ST5 1992 57021365 -UNITED ST2 UNITED ST5 1993 37406845 -UNITED ST2 UNITED ST5 1994 37640317 -UNITED ST2 UNITED ST5 1995 43382417 -UNITED ST2 UNITED ST5 1996 47028976 -UNITED ST2 UNITED ST5 1997 40662886 -UNITED ST2 UNITED ST6 1992 27813739 -UNITED ST2 UNITED ST6 1993 26482226 -UNITED ST2 UNITED ST6 1994 41526105 -UNITED ST2 UNITED ST6 1995 37146288 -UNITED ST2 UNITED ST6 1996 45416649 -UNITED ST2 UNITED ST6 1997 34274467 -UNITED ST2 UNITED ST7 1992 33702779 -UNITED ST2 UNITED ST7 1993 64185014 -UNITED ST2 UNITED ST7 1994 22182758 -UNITED ST2 UNITED ST7 1995 60000208 -UNITED ST2 UNITED ST7 1996 80468477 -UNITED ST2 UNITED ST7 1997 76483802 -UNITED ST2 UNITED ST8 1992 36046449 -UNITED ST2 UNITED ST8 1993 33445446 -UNITED ST2 UNITED ST8 1994 33767774 -UNITED ST2 UNITED ST8 1995 30504189 -UNITED ST2 UNITED ST8 1996 46865113 -UNITED ST2 UNITED ST8 1997 46391909 -UNITED ST2 UNITED ST9 1992 83705618 -UNITED ST2 UNITED ST9 1993 48188507 -UNITED ST2 UNITED ST9 1994 69671054 -UNITED ST2 UNITED ST9 1995 65055178 -UNITED ST2 UNITED ST9 1996 53491415 -UNITED ST2 UNITED ST9 1997 69543984 -UNITED ST3 UNITED ST0 1992 62441656 -UNITED ST3 UNITED ST0 1993 66626926 -UNITED ST3 UNITED ST0 1994 65522727 -UNITED ST3 UNITED ST0 1995 84462733 -UNITED ST3 UNITED ST0 1996 60226840 -UNITED ST3 UNITED ST0 1997 54050172 -UNITED ST3 UNITED ST1 1992 73637490 -UNITED ST3 UNITED ST1 1993 78230278 -UNITED ST3 UNITED ST1 1994 85523653 -UNITED ST3 UNITED ST1 1995 64762815 -UNITED ST3 UNITED ST1 1996 74746699 -UNITED ST3 UNITED ST1 1997 54150683 -UNITED ST3 UNITED ST2 1992 1097208 -UNITED ST3 UNITED ST2 1993 21559073 -UNITED ST3 UNITED ST2 1994 9995234 -UNITED ST3 UNITED ST2 1995 12886829 -UNITED ST3 UNITED ST2 1996 4311800 -UNITED ST3 UNITED ST2 1997 7891878 -UNITED ST3 UNITED ST3 1992 45087216 -UNITED ST3 UNITED ST3 1993 21777144 -UNITED ST3 UNITED ST3 1994 45627153 -UNITED ST3 UNITED ST3 1995 48158089 -UNITED ST3 UNITED ST3 1996 49263796 -UNITED ST3 UNITED ST3 1997 48385233 -UNITED ST3 UNITED ST4 1992 63144772 -UNITED ST3 UNITED ST4 1993 92493920 -UNITED ST3 UNITED ST4 1994 63412365 -UNITED ST3 UNITED ST4 1995 30422346 -UNITED ST3 UNITED ST4 1996 54838259 -UNITED ST3 UNITED ST4 1997 58356141 -UNITED ST3 UNITED ST5 1992 64239048 -UNITED ST3 UNITED ST5 1993 55907305 -UNITED ST3 UNITED ST5 1994 26197875 -UNITED ST3 UNITED ST5 1995 40057815 -UNITED ST3 UNITED ST5 1996 33717439 -UNITED ST3 UNITED ST5 1997 32811963 -UNITED ST3 UNITED ST6 1992 38798715 -UNITED ST3 UNITED ST6 1993 58584808 -UNITED ST3 UNITED ST6 1994 36290257 -UNITED ST3 UNITED ST6 1995 16738530 -UNITED ST3 UNITED ST6 1996 41881628 -UNITED ST3 UNITED ST6 1997 36075844 -UNITED ST3 UNITED ST7 1992 39028579 -UNITED ST3 UNITED ST7 1993 50135448 -UNITED ST3 UNITED ST7 1994 40935131 -UNITED ST3 UNITED ST7 1995 51579548 -UNITED ST3 UNITED ST7 1996 53987218 -UNITED ST3 UNITED ST7 1997 61095300 -UNITED ST3 UNITED ST8 1992 41275764 -UNITED ST3 UNITED ST8 1993 42251848 -UNITED ST3 UNITED ST8 1994 15936051 -UNITED ST3 UNITED ST8 1995 22505239 -UNITED ST3 UNITED ST8 1996 33924732 -UNITED ST3 UNITED ST8 1997 29102220 -UNITED ST3 UNITED ST9 1992 86142048 -UNITED ST3 UNITED ST9 1993 54762596 -UNITED ST3 UNITED ST9 1994 97476185 -UNITED ST3 UNITED ST9 1995 37364623 -UNITED ST3 UNITED ST9 1996 49612304 -UNITED ST3 UNITED ST9 1997 51370308 -UNITED ST4 UNITED ST0 1992 94644242 -UNITED ST4 UNITED ST0 1993 86918458 -UNITED ST4 UNITED ST0 1994 73719340 -UNITED ST4 UNITED ST0 1995 46142899 -UNITED ST4 UNITED ST0 1996 106897563 -UNITED ST4 UNITED ST0 1997 69875804 -UNITED ST4 UNITED ST1 1992 102027455 -UNITED ST4 UNITED ST1 1993 88521423 -UNITED ST4 UNITED ST1 1994 72219557 -UNITED ST4 UNITED ST1 1995 90878653 -UNITED ST4 UNITED ST1 1996 72819867 -UNITED ST4 UNITED ST1 1997 84611188 -UNITED ST4 UNITED ST2 1992 12061845 -UNITED ST4 UNITED ST2 1993 27012678 -UNITED ST4 UNITED ST2 1994 23035892 -UNITED ST4 UNITED ST2 1995 22787003 -UNITED ST4 UNITED ST2 1996 12451867 -UNITED ST4 UNITED ST2 1997 2848157 -UNITED ST4 UNITED ST3 1992 69001787 -UNITED ST4 UNITED ST3 1993 33129412 -UNITED ST4 UNITED ST3 1994 31599433 -UNITED ST4 UNITED ST3 1995 37356396 -UNITED ST4 UNITED ST3 1996 46460940 -UNITED ST4 UNITED ST3 1997 59634153 -UNITED ST4 UNITED ST4 1992 70111335 -UNITED ST4 UNITED ST4 1993 98150465 -UNITED ST4 UNITED ST4 1994 76837381 -UNITED ST4 UNITED ST4 1995 74587071 -UNITED ST4 UNITED ST4 1996 32787995 -UNITED ST4 UNITED ST4 1997 62532346 -UNITED ST4 UNITED ST5 1992 65057872 -UNITED ST4 UNITED ST5 1993 29119594 -UNITED ST4 UNITED ST5 1994 59397028 -UNITED ST4 UNITED ST5 1995 62081505 -UNITED ST4 UNITED ST5 1996 42808412 -UNITED ST4 UNITED ST5 1997 31995868 -UNITED ST4 UNITED ST6 1992 81886374 -UNITED ST4 UNITED ST6 1993 30699255 -UNITED ST4 UNITED ST6 1994 58633482 -UNITED ST4 UNITED ST6 1995 38109524 -UNITED ST4 UNITED ST6 1996 62189328 -UNITED ST4 UNITED ST6 1997 46236849 -UNITED ST4 UNITED ST7 1992 62529376 -UNITED ST4 UNITED ST7 1993 88172198 -UNITED ST4 UNITED ST7 1994 76954140 -UNITED ST4 UNITED ST7 1995 69264407 -UNITED ST4 UNITED ST7 1996 83677814 -UNITED ST4 UNITED ST7 1997 84606996 -UNITED ST4 UNITED ST8 1992 21928442 -UNITED ST4 UNITED ST8 1993 41657582 -UNITED ST4 UNITED ST8 1994 20685352 -UNITED ST4 UNITED ST8 1995 23053236 -UNITED ST4 UNITED ST8 1996 53035662 -UNITED ST4 UNITED ST8 1997 32599736 -UNITED ST4 UNITED ST9 1992 67837970 -UNITED ST4 UNITED ST9 1993 73296959 -UNITED ST4 UNITED ST9 1994 83822393 -UNITED ST4 UNITED ST9 1995 99490068 -UNITED ST4 UNITED ST9 1996 54134840 -UNITED ST4 UNITED ST9 1997 79697927 -UNITED ST5 UNITED ST0 1992 90432300 -UNITED ST5 UNITED ST0 1993 106822597 -UNITED ST5 UNITED ST0 1994 78648057 -UNITED ST5 UNITED ST0 1995 77615746 -UNITED ST5 UNITED ST0 1996 104501979 -UNITED ST5 UNITED ST0 1997 121457766 -UNITED ST5 UNITED ST1 1992 59945287 -UNITED ST5 UNITED ST1 1993 71454623 -UNITED ST5 UNITED ST1 1994 73508418 -UNITED ST5 UNITED ST1 1995 119793669 -UNITED ST5 UNITED ST1 1996 65346389 -UNITED ST5 UNITED ST1 1997 106900161 -UNITED ST5 UNITED ST2 1992 20495817 -UNITED ST5 UNITED ST2 1993 20610242 -UNITED ST5 UNITED ST2 1994 7162796 -UNITED ST5 UNITED ST2 1995 14951023 -UNITED ST5 UNITED ST2 1996 8544233 -UNITED ST5 UNITED ST2 1997 13863438 -UNITED ST5 UNITED ST3 1992 53963553 -UNITED ST5 UNITED ST3 1993 34031811 -UNITED ST5 UNITED ST3 1994 65334911 -UNITED ST5 UNITED ST3 1995 78099130 -UNITED ST5 UNITED ST3 1996 32919926 -UNITED ST5 UNITED ST3 1997 56789932 -UNITED ST5 UNITED ST4 1992 57625627 -UNITED ST5 UNITED ST4 1993 63415639 -UNITED ST5 UNITED ST4 1994 51799593 -UNITED ST5 UNITED ST4 1995 81016761 -UNITED ST5 UNITED ST4 1996 33424859 -UNITED ST5 UNITED ST4 1997 103432927 -UNITED ST5 UNITED ST5 1992 29279568 -UNITED ST5 UNITED ST5 1993 50357189 -UNITED ST5 UNITED ST5 1994 56235213 -UNITED ST5 UNITED ST5 1995 50394007 -UNITED ST5 UNITED ST5 1996 43020722 -UNITED ST5 UNITED ST5 1997 32568888 -UNITED ST5 UNITED ST6 1992 62628093 -UNITED ST5 UNITED ST6 1993 25357679 -UNITED ST5 UNITED ST6 1994 38658451 -UNITED ST5 UNITED ST6 1995 30204737 -UNITED ST5 UNITED ST6 1996 38332044 -UNITED ST5 UNITED ST6 1997 49030016 -UNITED ST5 UNITED ST7 1992 68333651 -UNITED ST5 UNITED ST7 1993 52745538 -UNITED ST5 UNITED ST7 1994 89089961 -UNITED ST5 UNITED ST7 1995 53740759 -UNITED ST5 UNITED ST7 1996 40946330 -UNITED ST5 UNITED ST7 1997 54534315 -UNITED ST5 UNITED ST8 1992 33878053 -UNITED ST5 UNITED ST8 1993 74359279 -UNITED ST5 UNITED ST8 1994 18931891 -UNITED ST5 UNITED ST8 1995 13346932 -UNITED ST5 UNITED ST8 1996 32415682 -UNITED ST5 UNITED ST8 1997 32910946 -UNITED ST5 UNITED ST9 1992 50191091 -UNITED ST5 UNITED ST9 1993 76286539 -UNITED ST5 UNITED ST9 1994 53367331 -UNITED ST5 UNITED ST9 1995 62364897 -UNITED ST5 UNITED ST9 1996 72614114 -UNITED ST5 UNITED ST9 1997 97392640 -UNITED ST6 UNITED ST0 1992 78696956 -UNITED ST6 UNITED ST0 1993 68247539 -UNITED ST6 UNITED ST0 1994 53728853 -UNITED ST6 UNITED ST0 1995 91718471 -UNITED ST6 UNITED ST0 1996 132286763 -UNITED ST6 UNITED ST0 1997 55824919 -UNITED ST6 UNITED ST1 1992 69408688 -UNITED ST6 UNITED ST1 1993 83345939 -UNITED ST6 UNITED ST1 1994 96937147 -UNITED ST6 UNITED ST1 1995 38328352 -UNITED ST6 UNITED ST1 1996 89705870 -UNITED ST6 UNITED ST1 1997 55117709 -UNITED ST6 UNITED ST2 1992 11820698 -UNITED ST6 UNITED ST2 1993 18814367 -UNITED ST6 UNITED ST2 1994 25809000 -UNITED ST6 UNITED ST2 1995 11363551 -UNITED ST6 UNITED ST2 1996 20447847 -UNITED ST6 UNITED ST2 1997 14746680 -UNITED ST6 UNITED ST3 1992 78778218 -UNITED ST6 UNITED ST3 1993 83417161 -UNITED ST6 UNITED ST3 1994 56123238 -UNITED ST6 UNITED ST3 1995 67513062 -UNITED ST6 UNITED ST3 1996 82168229 -UNITED ST6 UNITED ST3 1997 69725506 -UNITED ST6 UNITED ST4 1992 71086944 -UNITED ST6 UNITED ST4 1993 69646913 -UNITED ST6 UNITED ST4 1994 59829989 -UNITED ST6 UNITED ST4 1995 47697241 -UNITED ST6 UNITED ST4 1996 69296005 -UNITED ST6 UNITED ST4 1997 57215022 -UNITED ST6 UNITED ST5 1992 85882966 -UNITED ST6 UNITED ST5 1993 34229082 -UNITED ST6 UNITED ST5 1994 66461534 -UNITED ST6 UNITED ST5 1995 45428152 -UNITED ST6 UNITED ST5 1996 46030104 -UNITED ST6 UNITED ST5 1997 37571788 -UNITED ST6 UNITED ST6 1992 18137105 -UNITED ST6 UNITED ST6 1993 33050663 -UNITED ST6 UNITED ST6 1994 46199722 -UNITED ST6 UNITED ST6 1995 50954677 -UNITED ST6 UNITED ST6 1996 42923032 -UNITED ST6 UNITED ST6 1997 24983291 -UNITED ST6 UNITED ST7 1992 87658267 -UNITED ST6 UNITED ST7 1993 78197294 -UNITED ST6 UNITED ST7 1994 37057808 -UNITED ST6 UNITED ST7 1995 23833977 -UNITED ST6 UNITED ST7 1996 57090619 -UNITED ST6 UNITED ST7 1997 49795849 -UNITED ST6 UNITED ST8 1992 42463690 -UNITED ST6 UNITED ST8 1993 36456741 -UNITED ST6 UNITED ST8 1994 26592996 -UNITED ST6 UNITED ST8 1995 41973920 -UNITED ST6 UNITED ST8 1996 52136784 -UNITED ST6 UNITED ST8 1997 37988445 -UNITED ST6 UNITED ST9 1992 40591721 -UNITED ST6 UNITED ST9 1993 54900942 -UNITED ST6 UNITED ST9 1994 94683309 -UNITED ST6 UNITED ST9 1995 63457105 -UNITED ST6 UNITED ST9 1996 63256182 -UNITED ST6 UNITED ST9 1997 58231263 -UNITED ST7 UNITED ST0 1992 96078010 -UNITED ST7 UNITED ST0 1993 109788024 -UNITED ST7 UNITED ST0 1994 82265536 -UNITED ST7 UNITED ST0 1995 98031295 -UNITED ST7 UNITED ST0 1996 75602445 -UNITED ST7 UNITED ST0 1997 69354312 -UNITED ST7 UNITED ST1 1992 50167208 -UNITED ST7 UNITED ST1 1993 34391159 -UNITED ST7 UNITED ST1 1994 75320799 -UNITED ST7 UNITED ST1 1995 79964362 -UNITED ST7 UNITED ST1 1996 48784175 -UNITED ST7 UNITED ST1 1997 101326940 -UNITED ST7 UNITED ST2 1992 4943204 -UNITED ST7 UNITED ST2 1993 20219304 -UNITED ST7 UNITED ST2 1994 19655081 -UNITED ST7 UNITED ST2 1995 3973565 -UNITED ST7 UNITED ST2 1996 8390470 -UNITED ST7 UNITED ST2 1997 31409590 -UNITED ST7 UNITED ST3 1992 83765666 -UNITED ST7 UNITED ST3 1993 39523861 -UNITED ST7 UNITED ST3 1994 29424840 -UNITED ST7 UNITED ST3 1995 43699734 -UNITED ST7 UNITED ST3 1996 52356886 -UNITED ST7 UNITED ST3 1997 35168922 -UNITED ST7 UNITED ST4 1992 64900888 -UNITED ST7 UNITED ST4 1993 61656613 -UNITED ST7 UNITED ST4 1994 64778715 -UNITED ST7 UNITED ST4 1995 49899545 -UNITED ST7 UNITED ST4 1996 77941011 -UNITED ST7 UNITED ST4 1997 60123494 -UNITED ST7 UNITED ST5 1992 57566802 -UNITED ST7 UNITED ST5 1993 44763692 -UNITED ST7 UNITED ST5 1994 45895364 -UNITED ST7 UNITED ST5 1995 39015293 -UNITED ST7 UNITED ST5 1996 32784224 -UNITED ST7 UNITED ST5 1997 36889736 -UNITED ST7 UNITED ST6 1992 65513205 -UNITED ST7 UNITED ST6 1993 27100502 -UNITED ST7 UNITED ST6 1994 24631937 -UNITED ST7 UNITED ST6 1995 65020671 -UNITED ST7 UNITED ST6 1996 36535628 -UNITED ST7 UNITED ST6 1997 27539931 -UNITED ST7 UNITED ST7 1992 60121386 -UNITED ST7 UNITED ST7 1993 68109820 -UNITED ST7 UNITED ST7 1994 46446442 -UNITED ST7 UNITED ST7 1995 86571624 -UNITED ST7 UNITED ST7 1996 74040644 -UNITED ST7 UNITED ST7 1997 42566603 -UNITED ST7 UNITED ST8 1992 11895702 -UNITED ST7 UNITED ST8 1993 43902628 -UNITED ST7 UNITED ST8 1994 42776467 -UNITED ST7 UNITED ST8 1995 26444490 -UNITED ST7 UNITED ST8 1996 24348851 -UNITED ST7 UNITED ST8 1997 45476266 -UNITED ST7 UNITED ST9 1992 60326042 -UNITED ST7 UNITED ST9 1993 80310701 -UNITED ST7 UNITED ST9 1994 50144085 -UNITED ST7 UNITED ST9 1995 75317709 -UNITED ST7 UNITED ST9 1996 25028091 -UNITED ST7 UNITED ST9 1997 55039970 -UNITED ST8 UNITED ST0 1992 44368839 -UNITED ST8 UNITED ST0 1993 88356371 -UNITED ST8 UNITED ST0 1994 99220789 -UNITED ST8 UNITED ST0 1995 55231664 -UNITED ST8 UNITED ST0 1996 39845427 -UNITED ST8 UNITED ST0 1997 89434845 -UNITED ST8 UNITED ST1 1992 35680752 -UNITED ST8 UNITED ST1 1993 44415434 -UNITED ST8 UNITED ST1 1994 40152096 -UNITED ST8 UNITED ST1 1995 34217780 -UNITED ST8 UNITED ST1 1996 78971756 -UNITED ST8 UNITED ST1 1997 43284139 -UNITED ST8 UNITED ST2 1992 10971561 -UNITED ST8 UNITED ST2 1993 9984366 -UNITED ST8 UNITED ST2 1994 3789626 -UNITED ST8 UNITED ST2 1995 18282990 -UNITED ST8 UNITED ST2 1996 7028334 -UNITED ST8 UNITED ST2 1997 12053706 -UNITED ST8 UNITED ST3 1992 30949324 -UNITED ST8 UNITED ST3 1993 54049874 -UNITED ST8 UNITED ST3 1994 55203841 -UNITED ST8 UNITED ST3 1995 29349938 -UNITED ST8 UNITED ST3 1996 39535940 -UNITED ST8 UNITED ST3 1997 66918394 -UNITED ST8 UNITED ST4 1992 46757019 -UNITED ST8 UNITED ST4 1993 51419356 -UNITED ST8 UNITED ST4 1994 62141681 -UNITED ST8 UNITED ST4 1995 19287311 -UNITED ST8 UNITED ST4 1996 51033054 -UNITED ST8 UNITED ST4 1997 36600584 -UNITED ST8 UNITED ST5 1992 35187610 -UNITED ST8 UNITED ST5 1993 52237837 -UNITED ST8 UNITED ST5 1994 44553056 -UNITED ST8 UNITED ST5 1995 44994863 -UNITED ST8 UNITED ST5 1996 43480473 -UNITED ST8 UNITED ST5 1997 26673743 -UNITED ST8 UNITED ST6 1992 60540440 -UNITED ST8 UNITED ST6 1993 54633062 -UNITED ST8 UNITED ST6 1994 36776569 -UNITED ST8 UNITED ST6 1995 58990404 -UNITED ST8 UNITED ST6 1996 59746655 -UNITED ST8 UNITED ST6 1997 20073000 -UNITED ST8 UNITED ST7 1992 41897901 -UNITED ST8 UNITED ST7 1993 36902218 -UNITED ST8 UNITED ST7 1994 37600687 -UNITED ST8 UNITED ST7 1995 30918202 -UNITED ST8 UNITED ST7 1996 57523925 -UNITED ST8 UNITED ST7 1997 40878429 -UNITED ST8 UNITED ST8 1992 29309108 -UNITED ST8 UNITED ST8 1993 38790235 -UNITED ST8 UNITED ST8 1994 10023412 -UNITED ST8 UNITED ST8 1995 11285867 -UNITED ST8 UNITED ST8 1996 16340603 -UNITED ST8 UNITED ST8 1997 40749400 -UNITED ST8 UNITED ST9 1992 22684471 -UNITED ST8 UNITED ST9 1993 38516361 -UNITED ST8 UNITED ST9 1994 58666906 -UNITED ST8 UNITED ST9 1995 88855089 -UNITED ST8 UNITED ST9 1996 60466691 -UNITED ST8 UNITED ST9 1997 45852796 -UNITED ST9 UNITED ST0 1992 94813294 -UNITED ST9 UNITED ST0 1993 32653305 -UNITED ST9 UNITED ST0 1994 87670573 -UNITED ST9 UNITED ST0 1995 51548621 -UNITED ST9 UNITED ST0 1996 49590701 -UNITED ST9 UNITED ST0 1997 66066791 -UNITED ST9 UNITED ST1 1992 80795302 -UNITED ST9 UNITED ST1 1993 74059419 -UNITED ST9 UNITED ST1 1994 90440948 -UNITED ST9 UNITED ST1 1995 82938967 -UNITED ST9 UNITED ST1 1996 101983827 -UNITED ST9 UNITED ST1 1997 62418406 -UNITED ST9 UNITED ST2 1992 11623177 -UNITED ST9 UNITED ST2 1993 17718390 -UNITED ST9 UNITED ST2 1994 9700989 -UNITED ST9 UNITED ST2 1995 12653362 -UNITED ST9 UNITED ST2 1996 8108035 -UNITED ST9 UNITED ST2 1997 2661652 -UNITED ST9 UNITED ST3 1992 47681165 -UNITED ST9 UNITED ST3 1993 46943647 -UNITED ST9 UNITED ST3 1994 35846500 -UNITED ST9 UNITED ST3 1995 17539102 -UNITED ST9 UNITED ST3 1996 47021018 -UNITED ST9 UNITED ST3 1997 46121724 -UNITED ST9 UNITED ST4 1992 61489513 -UNITED ST9 UNITED ST4 1993 30644687 -UNITED ST9 UNITED ST4 1994 43910761 -UNITED ST9 UNITED ST4 1995 72525686 -UNITED ST9 UNITED ST4 1996 69616614 -UNITED ST9 UNITED ST4 1997 31765478 -UNITED ST9 UNITED ST5 1992 28075004 -UNITED ST9 UNITED ST5 1993 39685269 -UNITED ST9 UNITED ST5 1994 37146726 -UNITED ST9 UNITED ST5 1995 36009924 -UNITED ST9 UNITED ST5 1996 52825922 -UNITED ST9 UNITED ST5 1997 56091920 -UNITED ST9 UNITED ST6 1992 15879568 -UNITED ST9 UNITED ST6 1993 43867160 -UNITED ST9 UNITED ST6 1994 29212927 -UNITED ST9 UNITED ST6 1995 43482494 -UNITED ST9 UNITED ST6 1996 48685063 -UNITED ST9 UNITED ST6 1997 37596882 -UNITED ST9 UNITED ST7 1992 56883359 -UNITED ST9 UNITED ST7 1993 66625483 -UNITED ST9 UNITED ST7 1994 31586173 -UNITED ST9 UNITED ST7 1995 46464589 -UNITED ST9 UNITED ST7 1996 48798759 -UNITED ST9 UNITED ST7 1997 69052630 -UNITED ST9 UNITED ST8 1992 21386100 -UNITED ST9 UNITED ST8 1993 37184696 -UNITED ST9 UNITED ST8 1994 39278808 -UNITED ST9 UNITED ST8 1995 21292527 -UNITED ST9 UNITED ST8 1996 36112301 -UNITED ST9 UNITED ST8 1997 31272385 -UNITED ST9 UNITED ST9 1992 36048343 -UNITED ST9 UNITED ST9 1993 63260763 -UNITED ST9 UNITED ST9 1994 51980849 -UNITED ST9 UNITED ST9 1995 76515958 -UNITED ST9 UNITED ST9 1996 77301740 -UNITED ST9 UNITED ST9 1997 106222795 +UNITED ST0 UNITED ST0 1992 18293852 +UNITED ST0 UNITED ST3 1992 3201624 +UNITED ST0 UNITED ST7 1992 7137641 +UNITED ST1 UNITED ST0 1992 10878084 +UNITED ST1 UNITED ST2 1992 8004700 +UNITED ST1 UNITED ST3 1992 13938002 +UNITED ST1 UNITED ST6 1992 11000283 +UNITED ST1 UNITED ST7 1992 17410800 +UNITED ST1 UNITED ST9 1992 25644597 +UNITED ST2 UNITED ST0 1992 23943154 +UNITED ST2 UNITED ST2 1992 8313714 +UNITED ST2 UNITED ST3 1992 15331073 +UNITED ST2 UNITED ST7 1992 7759164 +UNITED ST2 UNITED ST9 1992 12343455 +UNITED ST3 UNITED ST3 1992 4080199 +UNITED ST3 UNITED ST9 1992 2614811 +UNITED ST4 UNITED ST0 1992 34626982 +UNITED ST4 UNITED ST2 1992 11696334 +UNITED ST4 UNITED ST3 1992 29767238 +UNITED ST4 UNITED ST6 1992 11369008 +UNITED ST4 UNITED ST7 1992 10151573 +UNITED ST4 UNITED ST9 1992 21189183 +UNITED ST5 UNITED ST0 1992 12301234 +UNITED ST5 UNITED ST2 1992 9917834 +UNITED ST5 UNITED ST3 1992 5862031 +UNITED ST5 UNITED ST6 1992 12398029 +UNITED ST5 UNITED ST7 1992 3936271 +UNITED ST5 UNITED ST9 1992 14448179 +UNITED ST6 UNITED ST0 1992 11900889 +UNITED ST6 UNITED ST2 1992 291566 +UNITED ST6 UNITED ST3 1992 1873819 +UNITED ST6 UNITED ST6 1992 8685228 +UNITED ST6 UNITED ST9 1992 6597261 +UNITED ST7 UNITED ST0 1992 1882015 +UNITED ST7 UNITED ST2 1992 6125476 +UNITED ST7 UNITED ST3 1992 17996772 +UNITED ST7 UNITED ST6 1992 6058017 +UNITED ST7 UNITED ST7 1992 9715656 +UNITED ST7 UNITED ST9 1992 5403152 +UNITED ST8 UNITED ST0 1992 3574169 +UNITED ST8 UNITED ST7 1992 2373825 +UNITED ST8 UNITED ST9 1992 5690491 +UNITED ST9 UNITED ST0 1992 4816370 +UNITED ST9 UNITED ST2 1992 12370917 +UNITED ST9 UNITED ST3 1992 17863433 +UNITED ST9 UNITED ST6 1992 2066609 +UNITED ST9 UNITED ST7 1992 6703890 +UNITED ST9 UNITED ST9 1992 4234523 -- !query3_3before -- -UNITED KI1 UNITED KI1 1992 46735995 -UNITED KI1 UNITED KI1 1993 45669535 -UNITED KI1 UNITED KI1 1994 34868441 -UNITED KI1 UNITED KI1 1995 35881186 -UNITED KI1 UNITED KI1 1996 59923037 -UNITED KI1 UNITED KI1 1997 70343133 -UNITED KI1 UNITED KI5 1992 21917846 -UNITED KI1 UNITED KI5 1993 21444206 -UNITED KI1 UNITED KI5 1994 21221780 -UNITED KI1 UNITED KI5 1995 17176510 -UNITED KI1 UNITED KI5 1996 13700254 -UNITED KI1 UNITED KI5 1997 16410668 -UNITED KI5 UNITED KI1 1992 36277055 -UNITED KI5 UNITED KI1 1993 61017607 -UNITED KI5 UNITED KI1 1994 36282163 -UNITED KI5 UNITED KI1 1995 82955896 -UNITED KI5 UNITED KI1 1996 65767049 -UNITED KI5 UNITED KI1 1997 64978359 -UNITED KI5 UNITED KI5 1992 25355267 -UNITED KI5 UNITED KI5 1993 34099392 -UNITED KI5 UNITED KI5 1994 34507284 -UNITED KI5 UNITED KI5 1995 20539805 -UNITED KI5 UNITED KI5 1996 46077342 -UNITED KI5 UNITED KI5 1997 27332027 +UNITED KI5 UNITED KI1 1992 4397192 -- !query3_3_after -- -UNITED KI1 UNITED KI1 1992 46735995 -UNITED KI1 UNITED KI1 1993 45669535 -UNITED KI1 UNITED KI1 1994 34868441 -UNITED KI1 UNITED KI1 1995 35881186 -UNITED KI1 UNITED KI1 1996 59923037 -UNITED KI1 UNITED KI1 1997 70343133 -UNITED KI1 UNITED KI5 1992 21917846 -UNITED KI1 UNITED KI5 1993 21444206 -UNITED KI1 UNITED KI5 1994 21221780 -UNITED KI1 UNITED KI5 1995 17176510 -UNITED KI1 UNITED KI5 1996 13700254 -UNITED KI1 UNITED KI5 1997 16410668 -UNITED KI5 UNITED KI1 1992 36277055 -UNITED KI5 UNITED KI1 1993 61017607 -UNITED KI5 UNITED KI1 1994 36282163 -UNITED KI5 UNITED KI1 1995 82955896 -UNITED KI5 UNITED KI1 1996 65767049 -UNITED KI5 UNITED KI1 1997 64978359 -UNITED KI5 UNITED KI5 1992 25355267 -UNITED KI5 UNITED KI5 1993 34099392 -UNITED KI5 UNITED KI5 1994 34507284 -UNITED KI5 UNITED KI5 1995 20539805 -UNITED KI5 UNITED KI5 1996 46077342 -UNITED KI5 UNITED KI5 1997 27332027 +UNITED KI5 UNITED KI1 1992 4397192 -- !query3_4before -- -UNITED KI1 UNITED KI5 1997 1870070 -UNITED KI5 UNITED KI1 1997 9117846 -UNITED KI5 UNITED KI5 1997 6203801 +UNITED KI5 UNITED KI1 1992 3856441 -- !query3_4_after -- -UNITED KI1 UNITED KI5 1997 1870070 -UNITED KI5 UNITED KI1 1997 9117846 -UNITED KI5 UNITED KI5 1997 6203801 +UNITED KI5 UNITED KI1 1992 3856441 -- !query4_1before -- -1992 ARGENTINA 9658964469 -1992 BRAZIL 9226983055 -1992 CANADA 9643176787 -1992 PERU 9410676597 -1992 UNITED STATES 9849427653 -1993 ARGENTINA 9976332853 -1993 BRAZIL 9468799229 -1993 CANADA 9897302420 -1993 PERU 9309445836 -1993 UNITED STATES 10003985086 -1994 ARGENTINA 9940305215 -1994 BRAZIL 9348651677 -1994 CANADA 9582647596 -1994 PERU 9295265013 -1994 UNITED STATES 9519880425 -1995 ARGENTINA 10143841380 -1995 BRAZIL 9156077350 -1995 CANADA 9562612160 -1995 PERU 9778087211 -1995 UNITED STATES 9310565244 -1996 ARGENTINA 10001927895 -1996 BRAZIL 9168485151 -1996 CANADA 10061604203 -1996 PERU 9355135674 -1996 UNITED STATES 9769712174 -1997 ARGENTINA 9854560261 -1997 BRAZIL 9121571047 -1997 CANADA 10097371778 -1997 PERU 9315525917 -1997 UNITED STATES 10506723879 -1998 ARGENTINA 5834240407 -1998 BRAZIL 5356398095 -1998 CANADA 5423211196 -1998 PERU 5726185970 -1998 UNITED STATES 6009462019 +1992 ARGENTINA 533196600 +1992 BRAZIL 684224630 +1992 CANADA 532686194 +1992 PERU 586223155 +1992 UNITED STATES 682387184 -- !query4_1_after -- -1992 ARGENTINA 9658964469 -1992 BRAZIL 9226983055 -1992 CANADA 9643176787 -1992 PERU 9410676597 -1992 UNITED STATES 9849427653 -1993 ARGENTINA 9976332853 -1993 BRAZIL 9468799229 -1993 CANADA 9897302420 -1993 PERU 9309445836 -1993 UNITED STATES 10003985086 -1994 ARGENTINA 9940305215 -1994 BRAZIL 9348651677 -1994 CANADA 9582647596 -1994 PERU 9295265013 -1994 UNITED STATES 9519880425 -1995 ARGENTINA 10143841380 -1995 BRAZIL 9156077350 -1995 CANADA 9562612160 -1995 PERU 9778087211 -1995 UNITED STATES 9310565244 -1996 ARGENTINA 10001927895 -1996 BRAZIL 9168485151 -1996 CANADA 10061604203 -1996 PERU 9355135674 -1996 UNITED STATES 9769712174 -1997 ARGENTINA 9854560261 -1997 BRAZIL 9121571047 -1997 CANADA 10097371778 -1997 PERU 9315525917 -1997 UNITED STATES 10506723879 -1998 ARGENTINA 5834240407 -1998 BRAZIL 5356398095 -1998 CANADA 5423211196 -1998 PERU 5726185970 -1998 UNITED STATES 6009462019 +1992 ARGENTINA 533196600 +1992 BRAZIL 684224630 +1992 CANADA 532686194 +1992 PERU 586223155 +1992 UNITED STATES 682387184 -- !query4_2before -- -1997 ARGENTINA MFGR#11 907071566 -1997 ARGENTINA MFGR#12 924115562 -1997 ARGENTINA MFGR#13 972861821 -1997 ARGENTINA MFGR#14 975410345 -1997 ARGENTINA MFGR#15 938867375 -1997 ARGENTINA MFGR#21 1014782574 -1997 ARGENTINA MFGR#22 873016783 -1997 ARGENTINA MFGR#23 1030357302 -1997 ARGENTINA MFGR#24 893460579 -1997 ARGENTINA MFGR#25 1006311403 -1997 BRAZIL MFGR#11 1073219328 -1997 BRAZIL MFGR#12 989858833 -1997 BRAZIL MFGR#13 1128480379 -1997 BRAZIL MFGR#14 1194256722 -1997 BRAZIL MFGR#15 1094419124 -1997 BRAZIL MFGR#21 910026832 -1997 BRAZIL MFGR#22 993142048 -1997 BRAZIL MFGR#23 1107672874 -1997 BRAZIL MFGR#24 1058013649 -1997 BRAZIL MFGR#25 994733764 -1997 CANADA MFGR#11 854725020 -1997 CANADA MFGR#12 938718164 -1997 CANADA MFGR#13 959265890 -1997 CANADA MFGR#14 1002812450 -1997 CANADA MFGR#15 848183013 -1997 CANADA MFGR#21 999805272 -1997 CANADA MFGR#22 778419763 -1997 CANADA MFGR#23 928359645 -1997 CANADA MFGR#24 849895128 -1997 CANADA MFGR#25 904587965 -1997 PERU MFGR#11 1100242877 -1997 PERU MFGR#12 994365350 -1997 PERU MFGR#13 847486105 -1997 PERU MFGR#14 947769683 -1997 PERU MFGR#15 999395678 -1997 PERU MFGR#21 867923394 -1997 PERU MFGR#22 988747459 -1997 PERU MFGR#23 1066645086 -1997 PERU MFGR#24 935665725 -1997 PERU MFGR#25 981454129 -1997 UNITED STATES MFGR#11 1046706048 -1997 UNITED STATES MFGR#12 909213709 -1997 UNITED STATES MFGR#13 1096278906 -1997 UNITED STATES MFGR#14 934282111 -1997 UNITED STATES MFGR#15 962760843 -1997 UNITED STATES MFGR#21 1000676474 -1997 UNITED STATES MFGR#22 1076947615 -1997 UNITED STATES MFGR#23 937288102 -1997 UNITED STATES MFGR#24 1003386363 -1997 UNITED STATES MFGR#25 1053666052 -1998 ARGENTINA MFGR#11 567612227 -1998 ARGENTINA MFGR#12 527025042 -1998 ARGENTINA MFGR#13 582791792 -1998 ARGENTINA MFGR#14 523726368 -1998 ARGENTINA MFGR#15 522078267 -1998 ARGENTINA MFGR#21 504712685 -1998 ARGENTINA MFGR#22 506061736 -1998 ARGENTINA MFGR#23 560479801 -1998 ARGENTINA MFGR#24 524579118 -1998 ARGENTINA MFGR#25 547840211 -1998 BRAZIL MFGR#11 638578488 -1998 BRAZIL MFGR#12 646312681 -1998 BRAZIL MFGR#13 655161772 -1998 BRAZIL MFGR#14 552676170 -1998 BRAZIL MFGR#15 663812709 -1998 BRAZIL MFGR#21 668822448 -1998 BRAZIL MFGR#22 591791918 -1998 BRAZIL MFGR#23 690648877 -1998 BRAZIL MFGR#24 562362220 -1998 BRAZIL MFGR#25 704182376 -1998 CANADA MFGR#11 509086125 -1998 CANADA MFGR#12 488089772 -1998 CANADA MFGR#13 486533297 -1998 CANADA MFGR#14 527837227 -1998 CANADA MFGR#15 535869299 -1998 CANADA MFGR#21 455868651 -1998 CANADA MFGR#22 594277308 -1998 CANADA MFGR#23 574125070 -1998 CANADA MFGR#24 508530424 -1998 CANADA MFGR#25 547757992 -1998 PERU MFGR#11 567838547 -1998 PERU MFGR#12 540544757 -1998 PERU MFGR#13 591331883 -1998 PERU MFGR#14 481335064 -1998 PERU MFGR#15 570246138 -1998 PERU MFGR#21 533733330 -1998 PERU MFGR#22 527790656 -1998 PERU MFGR#23 636393221 -1998 PERU MFGR#24 589075262 -1998 PERU MFGR#25 543251115 -1998 UNITED STATES MFGR#11 556276232 -1998 UNITED STATES MFGR#12 612385982 -1998 UNITED STATES MFGR#13 622413927 -1998 UNITED STATES MFGR#14 555006887 -1998 UNITED STATES MFGR#15 525119569 -1998 UNITED STATES MFGR#21 563406836 -1998 UNITED STATES MFGR#22 580478735 -1998 UNITED STATES MFGR#23 656080465 -1998 UNITED STATES MFGR#24 538445058 -1998 UNITED STATES MFGR#25 589111952 +1992 ARGENTINA MFGR#11 70016467 +1992 ARGENTINA MFGR#12 57130735 +1992 ARGENTINA MFGR#13 89600962 +1992 ARGENTINA MFGR#14 54217587 +1992 ARGENTINA MFGR#15 45034625 +1992 ARGENTINA MFGR#21 43792906 +1992 ARGENTINA MFGR#22 49248625 +1992 ARGENTINA MFGR#23 79718572 +1992 ARGENTINA MFGR#24 51057067 +1992 ARGENTINA MFGR#25 47431621 +1992 BRAZIL MFGR#11 49035302 +1992 BRAZIL MFGR#12 45469986 +1992 BRAZIL MFGR#13 68191869 +1992 BRAZIL MFGR#14 54781608 +1992 BRAZIL MFGR#15 40204144 +1992 BRAZIL MFGR#21 54385184 +1992 BRAZIL MFGR#22 73332346 +1992 BRAZIL MFGR#23 47288876 +1992 BRAZIL MFGR#24 61293205 +1992 BRAZIL MFGR#25 77335065 +1992 CANADA MFGR#11 20381804 +1992 CANADA MFGR#12 30863219 +1992 CANADA MFGR#13 52567971 +1992 CANADA MFGR#14 65603736 +1992 CANADA MFGR#15 49669956 +1992 CANADA MFGR#21 45613232 +1992 CANADA MFGR#22 32289959 +1992 CANADA MFGR#23 23226684 +1992 CANADA MFGR#24 49284224 +1992 CANADA MFGR#25 44418525 +1992 PERU MFGR#11 60890044 +1992 PERU MFGR#12 57092894 +1992 PERU MFGR#13 54461960 +1992 PERU MFGR#14 39217023 +1992 PERU MFGR#15 21170135 +1992 PERU MFGR#21 33391866 +1992 PERU MFGR#22 59295631 +1992 PERU MFGR#23 69486455 +1992 PERU MFGR#24 34232078 +1992 PERU MFGR#25 45525878 +1992 UNITED STATES MFGR#11 91427459 +1992 UNITED STATES MFGR#12 100744805 +1992 UNITED STATES MFGR#13 101592664 +1992 UNITED STATES MFGR#14 121042999 +1992 UNITED STATES MFGR#15 99396680 +1992 UNITED STATES MFGR#21 72692399 +1992 UNITED STATES MFGR#22 92929921 +1992 UNITED STATES MFGR#23 101853653 +1992 UNITED STATES MFGR#24 96327671 +1992 UNITED STATES MFGR#25 93459486 -- !query4_2_after -- -1997 ARGENTINA MFGR#11 907071566 -1997 ARGENTINA MFGR#12 924115562 -1997 ARGENTINA MFGR#13 972861821 -1997 ARGENTINA MFGR#14 975410345 -1997 ARGENTINA MFGR#15 938867375 -1997 ARGENTINA MFGR#21 1014782574 -1997 ARGENTINA MFGR#22 873016783 -1997 ARGENTINA MFGR#23 1030357302 -1997 ARGENTINA MFGR#24 893460579 -1997 ARGENTINA MFGR#25 1006311403 -1997 BRAZIL MFGR#11 1073219328 -1997 BRAZIL MFGR#12 989858833 -1997 BRAZIL MFGR#13 1128480379 -1997 BRAZIL MFGR#14 1194256722 -1997 BRAZIL MFGR#15 1094419124 -1997 BRAZIL MFGR#21 910026832 -1997 BRAZIL MFGR#22 993142048 -1997 BRAZIL MFGR#23 1107672874 -1997 BRAZIL MFGR#24 1058013649 -1997 BRAZIL MFGR#25 994733764 -1997 CANADA MFGR#11 854725020 -1997 CANADA MFGR#12 938718164 -1997 CANADA MFGR#13 959265890 -1997 CANADA MFGR#14 1002812450 -1997 CANADA MFGR#15 848183013 -1997 CANADA MFGR#21 999805272 -1997 CANADA MFGR#22 778419763 -1997 CANADA MFGR#23 928359645 -1997 CANADA MFGR#24 849895128 -1997 CANADA MFGR#25 904587965 -1997 PERU MFGR#11 1100242877 -1997 PERU MFGR#12 994365350 -1997 PERU MFGR#13 847486105 -1997 PERU MFGR#14 947769683 -1997 PERU MFGR#15 999395678 -1997 PERU MFGR#21 867923394 -1997 PERU MFGR#22 988747459 -1997 PERU MFGR#23 1066645086 -1997 PERU MFGR#24 935665725 -1997 PERU MFGR#25 981454129 -1997 UNITED STATES MFGR#11 1046706048 -1997 UNITED STATES MFGR#12 909213709 -1997 UNITED STATES MFGR#13 1096278906 -1997 UNITED STATES MFGR#14 934282111 -1997 UNITED STATES MFGR#15 962760843 -1997 UNITED STATES MFGR#21 1000676474 -1997 UNITED STATES MFGR#22 1076947615 -1997 UNITED STATES MFGR#23 937288102 -1997 UNITED STATES MFGR#24 1003386363 -1997 UNITED STATES MFGR#25 1053666052 -1998 ARGENTINA MFGR#11 567612227 -1998 ARGENTINA MFGR#12 527025042 -1998 ARGENTINA MFGR#13 582791792 -1998 ARGENTINA MFGR#14 523726368 -1998 ARGENTINA MFGR#15 522078267 -1998 ARGENTINA MFGR#21 504712685 -1998 ARGENTINA MFGR#22 506061736 -1998 ARGENTINA MFGR#23 560479801 -1998 ARGENTINA MFGR#24 524579118 -1998 ARGENTINA MFGR#25 547840211 -1998 BRAZIL MFGR#11 638578488 -1998 BRAZIL MFGR#12 646312681 -1998 BRAZIL MFGR#13 655161772 -1998 BRAZIL MFGR#14 552676170 -1998 BRAZIL MFGR#15 663812709 -1998 BRAZIL MFGR#21 668822448 -1998 BRAZIL MFGR#22 591791918 -1998 BRAZIL MFGR#23 690648877 -1998 BRAZIL MFGR#24 562362220 -1998 BRAZIL MFGR#25 704182376 -1998 CANADA MFGR#11 509086125 -1998 CANADA MFGR#12 488089772 -1998 CANADA MFGR#13 486533297 -1998 CANADA MFGR#14 527837227 -1998 CANADA MFGR#15 535869299 -1998 CANADA MFGR#21 455868651 -1998 CANADA MFGR#22 594277308 -1998 CANADA MFGR#23 574125070 -1998 CANADA MFGR#24 508530424 -1998 CANADA MFGR#25 547757992 -1998 PERU MFGR#11 567838547 -1998 PERU MFGR#12 540544757 -1998 PERU MFGR#13 591331883 -1998 PERU MFGR#14 481335064 -1998 PERU MFGR#15 570246138 -1998 PERU MFGR#21 533733330 -1998 PERU MFGR#22 527790656 -1998 PERU MFGR#23 636393221 -1998 PERU MFGR#24 589075262 -1998 PERU MFGR#25 543251115 -1998 UNITED STATES MFGR#11 556276232 -1998 UNITED STATES MFGR#12 612385982 -1998 UNITED STATES MFGR#13 622413927 -1998 UNITED STATES MFGR#14 555006887 -1998 UNITED STATES MFGR#15 525119569 -1998 UNITED STATES MFGR#21 563406836 -1998 UNITED STATES MFGR#22 580478735 -1998 UNITED STATES MFGR#23 656080465 -1998 UNITED STATES MFGR#24 538445058 -1998 UNITED STATES MFGR#25 589111952 +1992 ARGENTINA MFGR#11 70016467 +1992 ARGENTINA MFGR#12 57130735 +1992 ARGENTINA MFGR#13 89600962 +1992 ARGENTINA MFGR#14 54217587 +1992 ARGENTINA MFGR#15 45034625 +1992 ARGENTINA MFGR#21 43792906 +1992 ARGENTINA MFGR#22 49248625 +1992 ARGENTINA MFGR#23 79718572 +1992 ARGENTINA MFGR#24 51057067 +1992 ARGENTINA MFGR#25 47431621 +1992 BRAZIL MFGR#11 49035302 +1992 BRAZIL MFGR#12 45469986 +1992 BRAZIL MFGR#13 68191869 +1992 BRAZIL MFGR#14 54781608 +1992 BRAZIL MFGR#15 40204144 +1992 BRAZIL MFGR#21 54385184 +1992 BRAZIL MFGR#22 73332346 +1992 BRAZIL MFGR#23 47288876 +1992 BRAZIL MFGR#24 61293205 +1992 BRAZIL MFGR#25 77335065 +1992 CANADA MFGR#11 20381804 +1992 CANADA MFGR#12 30863219 +1992 CANADA MFGR#13 52567971 +1992 CANADA MFGR#14 65603736 +1992 CANADA MFGR#15 49669956 +1992 CANADA MFGR#21 45613232 +1992 CANADA MFGR#22 32289959 +1992 CANADA MFGR#23 23226684 +1992 CANADA MFGR#24 49284224 +1992 CANADA MFGR#25 44418525 +1992 PERU MFGR#11 60890044 +1992 PERU MFGR#12 57092894 +1992 PERU MFGR#13 54461960 +1992 PERU MFGR#14 39217023 +1992 PERU MFGR#15 21170135 +1992 PERU MFGR#21 33391866 +1992 PERU MFGR#22 59295631 +1992 PERU MFGR#23 69486455 +1992 PERU MFGR#24 34232078 +1992 PERU MFGR#25 45525878 +1992 UNITED STATES MFGR#11 91427459 +1992 UNITED STATES MFGR#12 100744805 +1992 UNITED STATES MFGR#13 101592664 +1992 UNITED STATES MFGR#14 121042999 +1992 UNITED STATES MFGR#15 99396680 +1992 UNITED STATES MFGR#21 72692399 +1992 UNITED STATES MFGR#22 92929921 +1992 UNITED STATES MFGR#23 101853653 +1992 UNITED STATES MFGR#24 96327671 +1992 UNITED STATES MFGR#25 93459486 -- !query4_3before -- -1997 UNITED ST0 MFGR#141 15456288 -1997 UNITED ST0 MFGR#1410 32961113 -1997 UNITED ST0 MFGR#1411 36166400 -1997 UNITED ST0 MFGR#1412 18237451 -1997 UNITED ST0 MFGR#1413 40752813 -1997 UNITED ST0 MFGR#1414 24237588 -1997 UNITED ST0 MFGR#1415 6419001 -1997 UNITED ST0 MFGR#1416 15639876 -1997 UNITED ST0 MFGR#1417 15476211 -1997 UNITED ST0 MFGR#1418 8169956 -1997 UNITED ST0 MFGR#1419 18217252 -1997 UNITED ST0 MFGR#142 29361222 -1997 UNITED ST0 MFGR#1420 34587603 -1997 UNITED ST0 MFGR#1421 28560921 -1997 UNITED ST0 MFGR#1422 25154181 -1997 UNITED ST0 MFGR#1423 18144008 -1997 UNITED ST0 MFGR#1424 28064267 -1997 UNITED ST0 MFGR#1425 15255845 -1997 UNITED ST0 MFGR#1426 5610353 -1997 UNITED ST0 MFGR#1427 5750693 -1997 UNITED ST0 MFGR#1428 14081707 -1997 UNITED ST0 MFGR#1429 9024512 -1997 UNITED ST0 MFGR#143 5325193 -1997 UNITED ST0 MFGR#1430 16961739 -1997 UNITED ST0 MFGR#1431 19015253 -1997 UNITED ST0 MFGR#1432 6324917 -1997 UNITED ST0 MFGR#1433 6734461 -1997 UNITED ST0 MFGR#1434 19206393 -1997 UNITED ST0 MFGR#1435 679925 -1997 UNITED ST0 MFGR#1436 13834012 -1997 UNITED ST0 MFGR#1437 23575407 -1997 UNITED ST0 MFGR#1438 24756074 -1997 UNITED ST0 MFGR#1439 19644072 -1997 UNITED ST0 MFGR#144 20285157 -1997 UNITED ST0 MFGR#1440 14068558 -1997 UNITED ST0 MFGR#145 22646138 -1997 UNITED ST0 MFGR#146 16381159 -1997 UNITED ST0 MFGR#147 12267827 -1997 UNITED ST0 MFGR#148 30968614 -1997 UNITED ST0 MFGR#149 8640669 -1997 UNITED ST1 MFGR#141 10152087 -1997 UNITED ST1 MFGR#1410 32583300 -1997 UNITED ST1 MFGR#1411 15690352 -1997 UNITED ST1 MFGR#1412 16671403 -1997 UNITED ST1 MFGR#1413 4942775 -1997 UNITED ST1 MFGR#1414 9391597 -1997 UNITED ST1 MFGR#1415 26937296 -1997 UNITED ST1 MFGR#1416 25782871 -1997 UNITED ST1 MFGR#1417 23349641 -1997 UNITED ST1 MFGR#1418 10934620 -1997 UNITED ST1 MFGR#1419 22673030 -1997 UNITED ST1 MFGR#142 14048276 -1997 UNITED ST1 MFGR#1420 18100795 -1997 UNITED ST1 MFGR#1421 25405678 -1997 UNITED ST1 MFGR#1423 5713800 -1997 UNITED ST1 MFGR#1424 4131923 -1997 UNITED ST1 MFGR#1425 17190425 -1997 UNITED ST1 MFGR#1426 22314044 -1997 UNITED ST1 MFGR#1427 13635057 -1997 UNITED ST1 MFGR#1428 17735811 -1997 UNITED ST1 MFGR#1429 28740911 -1997 UNITED ST1 MFGR#143 27418033 -1997 UNITED ST1 MFGR#1430 18655292 -1997 UNITED ST1 MFGR#1431 11074399 -1997 UNITED ST1 MFGR#1432 7490170 -1997 UNITED ST1 MFGR#1433 28997266 -1997 UNITED ST1 MFGR#1434 4612211 -1997 UNITED ST1 MFGR#1435 16327124 -1997 UNITED ST1 MFGR#1436 17409665 -1997 UNITED ST1 MFGR#1437 33391635 -1997 UNITED ST1 MFGR#1438 37321134 -1997 UNITED ST1 MFGR#1439 10279941 -1997 UNITED ST1 MFGR#144 12304723 -1997 UNITED ST1 MFGR#1440 28921230 -1997 UNITED ST1 MFGR#145 17933684 -1997 UNITED ST1 MFGR#146 8709103 -1997 UNITED ST1 MFGR#147 20145983 -1997 UNITED ST1 MFGR#148 16940400 -1997 UNITED ST1 MFGR#149 10284949 -1997 UNITED ST2 MFGR#1412 6328240 -1997 UNITED ST2 MFGR#1415 5022682 -1997 UNITED ST2 MFGR#1416 4102214 -1997 UNITED ST2 MFGR#1417 5284444 -1997 UNITED ST2 MFGR#1418 2649908 -1997 UNITED ST2 MFGR#1419 4664673 -1997 UNITED ST2 MFGR#142 10784835 -1997 UNITED ST2 MFGR#1423 4095190 -1997 UNITED ST2 MFGR#1424 5376355 -1997 UNITED ST2 MFGR#1425 3949957 -1997 UNITED ST2 MFGR#143 2859829 -1997 UNITED ST2 MFGR#1430 4618266 -1997 UNITED ST2 MFGR#1432 5183060 -1997 UNITED ST2 MFGR#1433 10531371 -1997 UNITED ST2 MFGR#1434 77646 -1997 UNITED ST2 MFGR#1436 6538508 -1997 UNITED ST2 MFGR#1438 12324747 -1997 UNITED ST2 MFGR#144 2002870 -1997 UNITED ST2 MFGR#1440 12247229 -1997 UNITED ST2 MFGR#145 202685 -1997 UNITED ST2 MFGR#148 5998932 -1997 UNITED ST3 MFGR#141 12866083 -1997 UNITED ST3 MFGR#1410 31051703 -1997 UNITED ST3 MFGR#1411 14099543 -1997 UNITED ST3 MFGR#1412 4970551 -1997 UNITED ST3 MFGR#1413 26129762 -1997 UNITED ST3 MFGR#1414 22364787 -1997 UNITED ST3 MFGR#1415 27883428 -1997 UNITED ST3 MFGR#1416 25289630 -1997 UNITED ST3 MFGR#1417 5524810 -1997 UNITED ST3 MFGR#1419 3616608 -1997 UNITED ST3 MFGR#142 10776059 -1997 UNITED ST3 MFGR#1420 12958851 -1997 UNITED ST3 MFGR#1421 5815494 -1997 UNITED ST3 MFGR#1422 17694270 -1997 UNITED ST3 MFGR#1423 10285142 -1997 UNITED ST3 MFGR#1424 21555317 -1997 UNITED ST3 MFGR#1425 5214517 -1997 UNITED ST3 MFGR#1426 16810700 -1997 UNITED ST3 MFGR#1427 10086430 -1997 UNITED ST3 MFGR#1428 7733716 -1997 UNITED ST3 MFGR#1429 11922430 -1997 UNITED ST3 MFGR#1430 20105292 -1997 UNITED ST3 MFGR#1431 14716979 -1997 UNITED ST3 MFGR#1432 15779247 -1997 UNITED ST3 MFGR#1433 15308773 -1997 UNITED ST3 MFGR#1434 8515037 -1997 UNITED ST3 MFGR#1435 25486899 -1997 UNITED ST3 MFGR#1436 12739448 -1997 UNITED ST3 MFGR#1437 11249180 -1997 UNITED ST3 MFGR#1438 4156824 -1997 UNITED ST3 MFGR#1439 18358277 -1997 UNITED ST3 MFGR#144 10097718 -1997 UNITED ST3 MFGR#1440 22920494 -1997 UNITED ST3 MFGR#145 20285859 -1997 UNITED ST3 MFGR#146 18406423 -1997 UNITED ST3 MFGR#147 17295990 -1997 UNITED ST3 MFGR#148 26127145 -1997 UNITED ST3 MFGR#149 19895538 -1997 UNITED ST4 MFGR#141 8135477 -1997 UNITED ST4 MFGR#1410 5503247 -1997 UNITED ST4 MFGR#1411 6096435 -1997 UNITED ST4 MFGR#1412 19976043 -1997 UNITED ST4 MFGR#1413 22581282 -1997 UNITED ST4 MFGR#1414 10101379 -1997 UNITED ST4 MFGR#1415 8877444 -1997 UNITED ST4 MFGR#1416 17943398 -1997 UNITED ST4 MFGR#1417 9224501 -1997 UNITED ST4 MFGR#1418 16255176 -1997 UNITED ST4 MFGR#1419 683069 -1997 UNITED ST4 MFGR#142 23512944 -1997 UNITED ST4 MFGR#1420 27954262 -1997 UNITED ST4 MFGR#1421 54209185 -1997 UNITED ST4 MFGR#1422 1173957 -1997 UNITED ST4 MFGR#1423 20410710 -1997 UNITED ST4 MFGR#1424 2889270 -1997 UNITED ST4 MFGR#1425 13361796 -1997 UNITED ST4 MFGR#1426 18678819 -1997 UNITED ST4 MFGR#1427 18206066 -1997 UNITED ST4 MFGR#1428 18153974 -1997 UNITED ST4 MFGR#1429 12545076 -1997 UNITED ST4 MFGR#143 6158834 -1997 UNITED ST4 MFGR#1430 16128370 -1997 UNITED ST4 MFGR#1431 11153717 -1997 UNITED ST4 MFGR#1432 7745636 -1997 UNITED ST4 MFGR#1433 15983634 -1997 UNITED ST4 MFGR#1434 3662246 -1997 UNITED ST4 MFGR#1435 3060180 -1997 UNITED ST4 MFGR#1436 11524592 -1997 UNITED ST4 MFGR#1437 36824174 -1997 UNITED ST4 MFGR#1438 8226666 -1997 UNITED ST4 MFGR#1439 9872470 -1997 UNITED ST4 MFGR#1440 401207 -1997 UNITED ST4 MFGR#145 11698312 -1997 UNITED ST4 MFGR#146 18259956 -1997 UNITED ST4 MFGR#147 10256110 -1997 UNITED ST4 MFGR#148 15892628 -1997 UNITED ST4 MFGR#149 4322491 -1997 UNITED ST5 MFGR#141 8534275 -1997 UNITED ST5 MFGR#1411 2427261 -1997 UNITED ST5 MFGR#1412 9032422 -1997 UNITED ST5 MFGR#1413 6343588 -1997 UNITED ST5 MFGR#1414 2961025 -1997 UNITED ST5 MFGR#1415 21186568 -1997 UNITED ST5 MFGR#1416 9354080 -1997 UNITED ST5 MFGR#1417 11786246 -1997 UNITED ST5 MFGR#1418 15297245 -1997 UNITED ST5 MFGR#1420 15787608 -1997 UNITED ST5 MFGR#1421 17858301 -1997 UNITED ST5 MFGR#1422 14080987 -1997 UNITED ST5 MFGR#1423 12535186 -1997 UNITED ST5 MFGR#1424 3221960 -1997 UNITED ST5 MFGR#1425 8950317 -1997 UNITED ST5 MFGR#1426 13161486 -1997 UNITED ST5 MFGR#1427 6149737 -1997 UNITED ST5 MFGR#1428 6796520 -1997 UNITED ST5 MFGR#1429 26026300 -1997 UNITED ST5 MFGR#143 6712572 -1997 UNITED ST5 MFGR#1430 16092131 -1997 UNITED ST5 MFGR#1431 3425291 -1997 UNITED ST5 MFGR#1432 14630175 -1997 UNITED ST5 MFGR#1433 4967585 -1997 UNITED ST5 MFGR#1434 6716261 -1997 UNITED ST5 MFGR#1435 6290790 -1997 UNITED ST5 MFGR#1436 6787699 -1997 UNITED ST5 MFGR#1437 6045168 -1997 UNITED ST5 MFGR#1438 1082877 -1997 UNITED ST5 MFGR#1439 5268611 -1997 UNITED ST5 MFGR#144 8900840 -1997 UNITED ST5 MFGR#1440 2644336 -1997 UNITED ST5 MFGR#145 18054844 -1997 UNITED ST5 MFGR#146 4893668 -1997 UNITED ST5 MFGR#147 9615891 -1997 UNITED ST5 MFGR#148 1426124 -1997 UNITED ST5 MFGR#149 7320726 -1997 UNITED ST6 MFGR#141 10051337 -1997 UNITED ST6 MFGR#1410 23352963 -1997 UNITED ST6 MFGR#1411 1537303 -1997 UNITED ST6 MFGR#1412 4332646 -1997 UNITED ST6 MFGR#1413 14686752 -1997 UNITED ST6 MFGR#1414 7226409 -1997 UNITED ST6 MFGR#1415 15062686 -1997 UNITED ST6 MFGR#1416 8606480 -1997 UNITED ST6 MFGR#1417 7662761 -1997 UNITED ST6 MFGR#1418 1810860 -1997 UNITED ST6 MFGR#1419 20179568 -1997 UNITED ST6 MFGR#142 15506072 -1997 UNITED ST6 MFGR#1420 3844271 -1997 UNITED ST6 MFGR#1421 10608709 -1997 UNITED ST6 MFGR#1422 16083699 -1997 UNITED ST6 MFGR#1423 6789571 -1997 UNITED ST6 MFGR#1424 17631950 -1997 UNITED ST6 MFGR#1425 8477221 -1997 UNITED ST6 MFGR#1426 16847029 -1997 UNITED ST6 MFGR#1428 17314796 -1997 UNITED ST6 MFGR#1429 19175871 -1997 UNITED ST6 MFGR#143 8775848 -1997 UNITED ST6 MFGR#1431 5319063 -1997 UNITED ST6 MFGR#1432 7875606 -1997 UNITED ST6 MFGR#1433 9305264 -1997 UNITED ST6 MFGR#1434 8050404 -1997 UNITED ST6 MFGR#1435 17585836 -1997 UNITED ST6 MFGR#1436 13745758 -1997 UNITED ST6 MFGR#1438 14536324 -1997 UNITED ST6 MFGR#1439 11451329 -1997 UNITED ST6 MFGR#144 10794837 -1997 UNITED ST6 MFGR#1440 5719238 -1997 UNITED ST6 MFGR#145 25449058 -1997 UNITED ST6 MFGR#146 17821996 -1997 UNITED ST6 MFGR#147 18922333 -1997 UNITED ST6 MFGR#148 10825718 -1997 UNITED ST6 MFGR#149 14950868 -1997 UNITED ST7 MFGR#141 8291476 -1997 UNITED ST7 MFGR#1410 4622953 -1997 UNITED ST7 MFGR#1411 14885507 -1997 UNITED ST7 MFGR#1412 19011375 -1997 UNITED ST7 MFGR#1413 31178370 -1997 UNITED ST7 MFGR#1414 16429613 -1997 UNITED ST7 MFGR#1415 3091832 -1997 UNITED ST7 MFGR#1416 17008243 -1997 UNITED ST7 MFGR#1417 15609283 -1997 UNITED ST7 MFGR#1418 5059394 -1997 UNITED ST7 MFGR#1419 7234721 -1997 UNITED ST7 MFGR#142 6898938 -1997 UNITED ST7 MFGR#1420 6426590 -1997 UNITED ST7 MFGR#1421 6043221 -1997 UNITED ST7 MFGR#1422 13273021 -1997 UNITED ST7 MFGR#1423 30537249 -1997 UNITED ST7 MFGR#1424 9224296 -1997 UNITED ST7 MFGR#1425 12799069 -1997 UNITED ST7 MFGR#1426 30277891 -1997 UNITED ST7 MFGR#1427 8331493 -1997 UNITED ST7 MFGR#1428 16205423 -1997 UNITED ST7 MFGR#1429 14396627 -1997 UNITED ST7 MFGR#143 10209623 -1997 UNITED ST7 MFGR#1430 16556758 -1997 UNITED ST7 MFGR#1431 15775327 -1997 UNITED ST7 MFGR#1432 24602989 -1997 UNITED ST7 MFGR#1433 22927717 -1997 UNITED ST7 MFGR#1434 9486462 -1997 UNITED ST7 MFGR#1435 14254290 -1997 UNITED ST7 MFGR#1436 8645365 -1997 UNITED ST7 MFGR#1437 19026166 -1997 UNITED ST7 MFGR#1438 11291550 -1997 UNITED ST7 MFGR#1439 9590292 -1997 UNITED ST7 MFGR#144 14110404 -1997 UNITED ST7 MFGR#1440 21877355 -1997 UNITED ST7 MFGR#145 8246717 -1997 UNITED ST7 MFGR#146 9785781 -1997 UNITED ST7 MFGR#147 13412691 -1997 UNITED ST7 MFGR#148 14235939 -1997 UNITED ST7 MFGR#149 23450309 -1997 UNITED ST8 MFGR#141 7323416 -1997 UNITED ST8 MFGR#1410 7191961 -1997 UNITED ST8 MFGR#1411 2425849 -1997 UNITED ST8 MFGR#1412 26725641 -1997 UNITED ST8 MFGR#1413 7915896 -1997 UNITED ST8 MFGR#1414 14359044 -1997 UNITED ST8 MFGR#1415 15834927 -1997 UNITED ST8 MFGR#1416 7464470 -1997 UNITED ST8 MFGR#1417 15274496 -1997 UNITED ST8 MFGR#1418 3261391 -1997 UNITED ST8 MFGR#1419 8020386 -1997 UNITED ST8 MFGR#142 5985550 -1997 UNITED ST8 MFGR#1420 9441461 -1997 UNITED ST8 MFGR#1421 2998735 -1997 UNITED ST8 MFGR#1423 1565415 -1997 UNITED ST8 MFGR#1424 5613094 -1997 UNITED ST8 MFGR#1425 7186734 -1997 UNITED ST8 MFGR#1426 3153117 -1997 UNITED ST8 MFGR#1427 11303043 -1997 UNITED ST8 MFGR#1428 3564003 -1997 UNITED ST8 MFGR#143 6357477 -1997 UNITED ST8 MFGR#1430 6633950 -1997 UNITED ST8 MFGR#1431 867285 -1997 UNITED ST8 MFGR#1432 9962884 -1997 UNITED ST8 MFGR#1433 13708011 -1997 UNITED ST8 MFGR#1434 7357067 -1997 UNITED ST8 MFGR#1435 11993453 -1997 UNITED ST8 MFGR#1436 7231938 -1997 UNITED ST8 MFGR#1437 18632056 -1997 UNITED ST8 MFGR#1438 3609384 -1997 UNITED ST8 MFGR#144 4822734 -1997 UNITED ST8 MFGR#1440 2521917 -1997 UNITED ST8 MFGR#145 11683385 -1997 UNITED ST8 MFGR#146 3350415 -1997 UNITED ST8 MFGR#147 8425393 -1997 UNITED ST8 MFGR#148 15005341 -1997 UNITED ST8 MFGR#149 9018101 -1997 UNITED ST9 MFGR#141 21838850 -1997 UNITED ST9 MFGR#1410 8455387 -1997 UNITED ST9 MFGR#1411 27984689 -1997 UNITED ST9 MFGR#1412 26491401 -1997 UNITED ST9 MFGR#1413 30237705 -1997 UNITED ST9 MFGR#1414 8957476 -1997 UNITED ST9 MFGR#1415 35391583 -1997 UNITED ST9 MFGR#1416 8923782 -1997 UNITED ST9 MFGR#1417 3210984 -1997 UNITED ST9 MFGR#1418 12102607 -1997 UNITED ST9 MFGR#1419 14468627 -1997 UNITED ST9 MFGR#142 33363582 -1997 UNITED ST9 MFGR#1420 13139731 -1997 UNITED ST9 MFGR#1421 8410650 -1997 UNITED ST9 MFGR#1422 11268410 -1997 UNITED ST9 MFGR#1423 2846614 -1997 UNITED ST9 MFGR#1424 2305872 -1997 UNITED ST9 MFGR#1425 4896060 -1997 UNITED ST9 MFGR#1426 31544772 -1997 UNITED ST9 MFGR#1427 4113085 -1997 UNITED ST9 MFGR#1428 11137081 -1997 UNITED ST9 MFGR#1429 10633230 -1997 UNITED ST9 MFGR#143 11438203 -1997 UNITED ST9 MFGR#1430 2727389 -1997 UNITED ST9 MFGR#1431 10548538 -1997 UNITED ST9 MFGR#1433 26564495 -1997 UNITED ST9 MFGR#1434 11948206 -1997 UNITED ST9 MFGR#1435 2201909 -1997 UNITED ST9 MFGR#1436 10347631 -1997 UNITED ST9 MFGR#1437 40051690 -1997 UNITED ST9 MFGR#1438 2662945 -1997 UNITED ST9 MFGR#1439 18011455 -1997 UNITED ST9 MFGR#144 10202784 -1997 UNITED ST9 MFGR#1440 11123196 -1997 UNITED ST9 MFGR#145 11326624 -1997 UNITED ST9 MFGR#146 27825456 -1997 UNITED ST9 MFGR#147 17579502 -1997 UNITED ST9 MFGR#148 17001116 -1998 UNITED ST0 MFGR#141 12484786 -1998 UNITED ST0 MFGR#1410 5225999 -1998 UNITED ST0 MFGR#1411 2633395 -1998 UNITED ST0 MFGR#1412 9479536 -1998 UNITED ST0 MFGR#1413 18468711 -1998 UNITED ST0 MFGR#1414 19440601 -1998 UNITED ST0 MFGR#1415 16894643 -1998 UNITED ST0 MFGR#1416 7520117 -1998 UNITED ST0 MFGR#1417 464576 -1998 UNITED ST0 MFGR#1418 5171747 -1998 UNITED ST0 MFGR#1419 5970985 -1998 UNITED ST0 MFGR#142 23830409 -1998 UNITED ST0 MFGR#1420 2568835 -1998 UNITED ST0 MFGR#1421 6096442 -1998 UNITED ST0 MFGR#1422 10099456 -1998 UNITED ST0 MFGR#1423 24617746 -1998 UNITED ST0 MFGR#1424 22853888 -1998 UNITED ST0 MFGR#1425 25622112 -1998 UNITED ST0 MFGR#1426 9630701 -1998 UNITED ST0 MFGR#1427 4952625 -1998 UNITED ST0 MFGR#1428 4675471 -1998 UNITED ST0 MFGR#1429 19713527 -1998 UNITED ST0 MFGR#143 4971265 -1998 UNITED ST0 MFGR#1430 13948705 -1998 UNITED ST0 MFGR#1431 2955181 -1998 UNITED ST0 MFGR#1432 17946759 -1998 UNITED ST0 MFGR#1433 23325027 -1998 UNITED ST0 MFGR#1434 8580738 -1998 UNITED ST0 MFGR#1435 6663914 -1998 UNITED ST0 MFGR#1436 13468582 -1998 UNITED ST0 MFGR#1437 9419979 -1998 UNITED ST0 MFGR#1438 16164522 -1998 UNITED ST0 MFGR#1439 8692866 -1998 UNITED ST0 MFGR#144 17338711 -1998 UNITED ST0 MFGR#1440 8513736 -1998 UNITED ST0 MFGR#145 6212569 -1998 UNITED ST0 MFGR#146 2702349 -1998 UNITED ST0 MFGR#149 8844540 -1998 UNITED ST1 MFGR#141 13449110 -1998 UNITED ST1 MFGR#1410 5224626 -1998 UNITED ST1 MFGR#1411 16046399 -1998 UNITED ST1 MFGR#1412 8424392 -1998 UNITED ST1 MFGR#1413 7634982 -1998 UNITED ST1 MFGR#1414 8711782 -1998 UNITED ST1 MFGR#1415 5991201 -1998 UNITED ST1 MFGR#1416 11059645 -1998 UNITED ST1 MFGR#1417 8108087 -1998 UNITED ST1 MFGR#1418 1710875 -1998 UNITED ST1 MFGR#1419 452298 -1998 UNITED ST1 MFGR#142 6186526 -1998 UNITED ST1 MFGR#1421 4766963 -1998 UNITED ST1 MFGR#1422 5556563 -1998 UNITED ST1 MFGR#1423 12110257 -1998 UNITED ST1 MFGR#1424 8924948 -1998 UNITED ST1 MFGR#1425 5684710 -1998 UNITED ST1 MFGR#1426 6844444 -1998 UNITED ST1 MFGR#1427 7625525 -1998 UNITED ST1 MFGR#1428 5227395 -1998 UNITED ST1 MFGR#1429 2855272 -1998 UNITED ST1 MFGR#143 6964811 -1998 UNITED ST1 MFGR#1430 16952380 -1998 UNITED ST1 MFGR#1431 7655883 -1998 UNITED ST1 MFGR#1432 2975960 -1998 UNITED ST1 MFGR#1433 10175594 -1998 UNITED ST1 MFGR#1434 26358848 -1998 UNITED ST1 MFGR#1435 11408623 -1998 UNITED ST1 MFGR#1436 4438242 -1998 UNITED ST1 MFGR#1437 10014906 -1998 UNITED ST1 MFGR#1438 18817171 -1998 UNITED ST1 MFGR#1439 2129973 -1998 UNITED ST1 MFGR#144 9512727 -1998 UNITED ST1 MFGR#1440 20185179 -1998 UNITED ST1 MFGR#145 823605 -1998 UNITED ST1 MFGR#146 9358026 -1998 UNITED ST1 MFGR#147 6552312 -1998 UNITED ST1 MFGR#148 9144461 -1998 UNITED ST1 MFGR#149 2149639 -1998 UNITED ST2 MFGR#1410 8044368 -1998 UNITED ST2 MFGR#1413 727343 -1998 UNITED ST2 MFGR#1414 251088 -1998 UNITED ST2 MFGR#1416 5224696 -1998 UNITED ST2 MFGR#1419 7917142 -1998 UNITED ST2 MFGR#1420 9848986 -1998 UNITED ST2 MFGR#1427 4816976 -1998 UNITED ST2 MFGR#1429 4196069 -1998 UNITED ST2 MFGR#143 10073044 -1998 UNITED ST2 MFGR#1431 877123 -1998 UNITED ST2 MFGR#1433 3740836 -1998 UNITED ST2 MFGR#1434 857691 -1998 UNITED ST2 MFGR#1435 2277870 -1998 UNITED ST2 MFGR#145 4906521 -1998 UNITED ST2 MFGR#149 842348 -1998 UNITED ST3 MFGR#141 4181582 -1998 UNITED ST3 MFGR#1410 3169761 -1998 UNITED ST3 MFGR#1412 6590440 -1998 UNITED ST3 MFGR#1413 2015369 -1998 UNITED ST3 MFGR#1414 4408466 -1998 UNITED ST3 MFGR#1415 3963826 -1998 UNITED ST3 MFGR#1416 3179540 -1998 UNITED ST3 MFGR#1417 11440999 -1998 UNITED ST3 MFGR#1418 8725751 -1998 UNITED ST3 MFGR#1419 10646990 -1998 UNITED ST3 MFGR#142 5056172 -1998 UNITED ST3 MFGR#1420 3583150 -1998 UNITED ST3 MFGR#1421 7085689 -1998 UNITED ST3 MFGR#1422 20858904 -1998 UNITED ST3 MFGR#1423 12825743 -1998 UNITED ST3 MFGR#1424 3533518 -1998 UNITED ST3 MFGR#1425 4938324 -1998 UNITED ST3 MFGR#1426 15164664 -1998 UNITED ST3 MFGR#1427 2619162 -1998 UNITED ST3 MFGR#1428 7723699 -1998 UNITED ST3 MFGR#143 24362893 -1998 UNITED ST3 MFGR#1430 7139650 -1998 UNITED ST3 MFGR#1431 10306394 -1998 UNITED ST3 MFGR#1432 14769004 -1998 UNITED ST3 MFGR#1433 9610545 -1998 UNITED ST3 MFGR#1435 6611760 -1998 UNITED ST3 MFGR#1436 941348 -1998 UNITED ST3 MFGR#1437 4802162 -1998 UNITED ST3 MFGR#1438 5079750 -1998 UNITED ST3 MFGR#1439 5905118 -1998 UNITED ST3 MFGR#144 5436839 -1998 UNITED ST3 MFGR#1440 9815517 -1998 UNITED ST3 MFGR#145 7580271 -1998 UNITED ST3 MFGR#146 14051404 -1998 UNITED ST3 MFGR#147 7786017 -1998 UNITED ST3 MFGR#149 1271813 -1998 UNITED ST4 MFGR#141 26703183 -1998 UNITED ST4 MFGR#1410 18094129 -1998 UNITED ST4 MFGR#1411 615999 -1998 UNITED ST4 MFGR#1412 13988709 -1998 UNITED ST4 MFGR#1413 10115647 -1998 UNITED ST4 MFGR#1414 1586551 -1998 UNITED ST4 MFGR#1415 14357244 -1998 UNITED ST4 MFGR#1416 9430036 -1998 UNITED ST4 MFGR#1417 17361773 -1998 UNITED ST4 MFGR#1418 10187338 -1998 UNITED ST4 MFGR#1419 7116094 -1998 UNITED ST4 MFGR#142 18454502 -1998 UNITED ST4 MFGR#1420 1621340 -1998 UNITED ST4 MFGR#1422 13861291 -1998 UNITED ST4 MFGR#1423 4208006 -1998 UNITED ST4 MFGR#1424 3242576 -1998 UNITED ST4 MFGR#1425 1470642 -1998 UNITED ST4 MFGR#1426 11987205 -1998 UNITED ST4 MFGR#1427 12975399 -1998 UNITED ST4 MFGR#1428 11756155 -1998 UNITED ST4 MFGR#1429 6956191 -1998 UNITED ST4 MFGR#143 11499476 -1998 UNITED ST4 MFGR#1430 4355096 -1998 UNITED ST4 MFGR#1431 20166612 -1998 UNITED ST4 MFGR#1432 8849789 -1998 UNITED ST4 MFGR#1433 10786037 -1998 UNITED ST4 MFGR#1435 13208151 -1998 UNITED ST4 MFGR#1436 4747808 -1998 UNITED ST4 MFGR#1437 12999351 -1998 UNITED ST4 MFGR#1438 17360787 -1998 UNITED ST4 MFGR#1439 2557841 -1998 UNITED ST4 MFGR#144 22030911 -1998 UNITED ST4 MFGR#145 5272280 -1998 UNITED ST4 MFGR#146 4307958 -1998 UNITED ST4 MFGR#147 7995050 -1998 UNITED ST4 MFGR#148 2348889 -1998 UNITED ST5 MFGR#141 8396190 -1998 UNITED ST5 MFGR#1410 17422889 -1998 UNITED ST5 MFGR#1411 764246 -1998 UNITED ST5 MFGR#1413 8051561 -1998 UNITED ST5 MFGR#1414 1500867 -1998 UNITED ST5 MFGR#1416 3874315 -1998 UNITED ST5 MFGR#1417 7550747 -1998 UNITED ST5 MFGR#1418 14241755 -1998 UNITED ST5 MFGR#1419 4893846 -1998 UNITED ST5 MFGR#142 5418299 -1998 UNITED ST5 MFGR#1420 3810727 -1998 UNITED ST5 MFGR#1421 3321336 -1998 UNITED ST5 MFGR#1423 5021100 -1998 UNITED ST5 MFGR#1424 6125645 -1998 UNITED ST5 MFGR#1426 880680 -1998 UNITED ST5 MFGR#1427 1284890 -1998 UNITED ST5 MFGR#1428 3813108 -1998 UNITED ST5 MFGR#1429 9429868 -1998 UNITED ST5 MFGR#143 15699981 -1998 UNITED ST5 MFGR#1430 7977654 -1998 UNITED ST5 MFGR#1431 5545969 -1998 UNITED ST5 MFGR#1432 1803434 -1998 UNITED ST5 MFGR#1435 12957056 -1998 UNITED ST5 MFGR#1436 20666755 -1998 UNITED ST5 MFGR#1437 9536733 -1998 UNITED ST5 MFGR#1439 11784214 -1998 UNITED ST5 MFGR#1440 613089 -1998 UNITED ST5 MFGR#145 8237944 -1998 UNITED ST5 MFGR#147 5370351 -1998 UNITED ST6 MFGR#141 6683977 -1998 UNITED ST6 MFGR#1411 11589040 -1998 UNITED ST6 MFGR#1412 4800954 -1998 UNITED ST6 MFGR#1416 5848432 -1998 UNITED ST6 MFGR#1417 1804324 -1998 UNITED ST6 MFGR#1418 4642727 -1998 UNITED ST6 MFGR#1419 10079621 -1998 UNITED ST6 MFGR#142 2533570 -1998 UNITED ST6 MFGR#1420 9016258 -1998 UNITED ST6 MFGR#1421 7636150 -1998 UNITED ST6 MFGR#1422 1272436 -1998 UNITED ST6 MFGR#1423 9600097 -1998 UNITED ST6 MFGR#1424 7036762 -1998 UNITED ST6 MFGR#1425 10867426 -1998 UNITED ST6 MFGR#1426 7033994 -1998 UNITED ST6 MFGR#1427 6481679 -1998 UNITED ST6 MFGR#1428 6269185 -1998 UNITED ST6 MFGR#1429 12055631 -1998 UNITED ST6 MFGR#143 6500459 -1998 UNITED ST6 MFGR#1430 6816243 -1998 UNITED ST6 MFGR#1432 7752973 -1998 UNITED ST6 MFGR#1436 6292813 -1998 UNITED ST6 MFGR#1437 2334894 -1998 UNITED ST6 MFGR#1438 4459455 -1998 UNITED ST6 MFGR#144 640220 -1998 UNITED ST6 MFGR#1440 748200 -1998 UNITED ST6 MFGR#145 7948794 -1998 UNITED ST6 MFGR#146 2734333 -1998 UNITED ST6 MFGR#147 2995632 -1998 UNITED ST6 MFGR#148 10695287 -1998 UNITED ST6 MFGR#149 1475864 -1998 UNITED ST7 MFGR#141 4816637 -1998 UNITED ST7 MFGR#1410 24393912 -1998 UNITED ST7 MFGR#1411 1283038 -1998 UNITED ST7 MFGR#1412 12607459 -1998 UNITED ST7 MFGR#1413 15421293 -1998 UNITED ST7 MFGR#1414 7681668 -1998 UNITED ST7 MFGR#1415 7138377 -1998 UNITED ST7 MFGR#1416 9246990 -1998 UNITED ST7 MFGR#1417 5242972 -1998 UNITED ST7 MFGR#1419 13548941 -1998 UNITED ST7 MFGR#142 17758762 -1998 UNITED ST7 MFGR#1420 11980437 -1998 UNITED ST7 MFGR#1421 11166952 -1998 UNITED ST7 MFGR#1422 9231883 -1998 UNITED ST7 MFGR#1424 5796389 -1998 UNITED ST7 MFGR#1425 4943582 -1998 UNITED ST7 MFGR#1426 16361100 -1998 UNITED ST7 MFGR#1427 9659296 -1998 UNITED ST7 MFGR#1428 2998020 -1998 UNITED ST7 MFGR#1429 5766768 -1998 UNITED ST7 MFGR#143 5526167 -1998 UNITED ST7 MFGR#1430 4191892 -1998 UNITED ST7 MFGR#1431 12979559 -1998 UNITED ST7 MFGR#1432 5832068 -1998 UNITED ST7 MFGR#1433 11410227 -1998 UNITED ST7 MFGR#1434 10525792 -1998 UNITED ST7 MFGR#1435 5982420 -1998 UNITED ST7 MFGR#1436 7142934 -1998 UNITED ST7 MFGR#1437 7499671 -1998 UNITED ST7 MFGR#1438 9177564 -1998 UNITED ST7 MFGR#1439 1974444 -1998 UNITED ST7 MFGR#144 2428463 -1998 UNITED ST7 MFGR#1440 3300838 -1998 UNITED ST7 MFGR#146 1771645 -1998 UNITED ST7 MFGR#147 4681835 -1998 UNITED ST7 MFGR#148 7124367 -1998 UNITED ST7 MFGR#149 11662970 -1998 UNITED ST8 MFGR#141 12753354 -1998 UNITED ST8 MFGR#1410 1097292 -1998 UNITED ST8 MFGR#1411 4069881 -1998 UNITED ST8 MFGR#1414 9474831 -1998 UNITED ST8 MFGR#1415 6406304 -1998 UNITED ST8 MFGR#1418 2392133 -1998 UNITED ST8 MFGR#1419 2953230 -1998 UNITED ST8 MFGR#142 10702563 -1998 UNITED ST8 MFGR#1420 3719432 -1998 UNITED ST8 MFGR#1421 3854657 -1998 UNITED ST8 MFGR#1422 9711968 -1998 UNITED ST8 MFGR#1424 1102516 -1998 UNITED ST8 MFGR#1426 8937532 -1998 UNITED ST8 MFGR#1427 5255637 -1998 UNITED ST8 MFGR#1428 1695112 -1998 UNITED ST8 MFGR#143 6231349 -1998 UNITED ST8 MFGR#1430 4678664 -1998 UNITED ST8 MFGR#1431 2459741 -1998 UNITED ST8 MFGR#1433 1724339 -1998 UNITED ST8 MFGR#1434 8222328 -1998 UNITED ST8 MFGR#1435 4087165 -1998 UNITED ST8 MFGR#1437 7379288 -1998 UNITED ST8 MFGR#1439 1947163 -1998 UNITED ST8 MFGR#144 21846266 -1998 UNITED ST8 MFGR#146 13768113 -1998 UNITED ST8 MFGR#147 5279643 -1998 UNITED ST8 MFGR#149 15221410 -1998 UNITED ST9 MFGR#141 5544000 -1998 UNITED ST9 MFGR#1410 7971104 -1998 UNITED ST9 MFGR#1411 26426609 -1998 UNITED ST9 MFGR#1412 1701777 -1998 UNITED ST9 MFGR#1413 18786672 -1998 UNITED ST9 MFGR#1414 15787780 -1998 UNITED ST9 MFGR#1415 11214818 -1998 UNITED ST9 MFGR#1416 2076120 -1998 UNITED ST9 MFGR#1417 14310334 -1998 UNITED ST9 MFGR#1418 3558355 -1998 UNITED ST9 MFGR#1419 6134867 -1998 UNITED ST9 MFGR#142 257897 -1998 UNITED ST9 MFGR#1420 7716338 -1998 UNITED ST9 MFGR#1421 10291775 -1998 UNITED ST9 MFGR#1422 6105716 -1998 UNITED ST9 MFGR#1423 11378613 -1998 UNITED ST9 MFGR#1424 1904550 -1998 UNITED ST9 MFGR#1425 18831147 -1998 UNITED ST9 MFGR#1426 18562642 -1998 UNITED ST9 MFGR#1427 12565890 -1998 UNITED ST9 MFGR#1428 10720914 -1998 UNITED ST9 MFGR#1429 21940718 -1998 UNITED ST9 MFGR#143 21480722 -1998 UNITED ST9 MFGR#1430 30935173 -1998 UNITED ST9 MFGR#1431 7793012 -1998 UNITED ST9 MFGR#1432 10026104 -1998 UNITED ST9 MFGR#1433 17874435 -1998 UNITED ST9 MFGR#1434 5493785 -1998 UNITED ST9 MFGR#1435 10738769 -1998 UNITED ST9 MFGR#1436 43504 -1998 UNITED ST9 MFGR#1437 13559646 -1998 UNITED ST9 MFGR#1438 12654258 -1998 UNITED ST9 MFGR#1439 5972923 -1998 UNITED ST9 MFGR#144 15702123 -1998 UNITED ST9 MFGR#1440 6681029 -1998 UNITED ST9 MFGR#145 920718 -1998 UNITED ST9 MFGR#146 10435931 -1998 UNITED ST9 MFGR#147 4759326 -1998 UNITED ST9 MFGR#148 9970848 -1998 UNITED ST9 MFGR#149 7510940 +1992 UNITED ST0 MFGR#141 555641 +1992 UNITED ST0 MFGR#1411 7079182 +1992 UNITED ST0 MFGR#1412 2872721 +1992 UNITED ST0 MFGR#1413 5471501 +1992 UNITED ST0 MFGR#1414 6180491 +1992 UNITED ST0 MFGR#1415 3832532 +1992 UNITED ST0 MFGR#1417 5090445 +1992 UNITED ST0 MFGR#142 4603254 +1992 UNITED ST0 MFGR#1421 717404 +1992 UNITED ST0 MFGR#1422 6204274 +1992 UNITED ST0 MFGR#1424 5518896 +1992 UNITED ST0 MFGR#1425 5763764 +1992 UNITED ST0 MFGR#1427 6180274 +1992 UNITED ST0 MFGR#1432 6428730 +1992 UNITED ST0 MFGR#1433 833533 +1992 UNITED ST0 MFGR#1434 13713612 +1992 UNITED ST0 MFGR#145 13934235 +1992 UNITED ST2 MFGR#1411 758915 +1992 UNITED ST2 MFGR#142 4221010 +1992 UNITED ST2 MFGR#1421 4315917 +1992 UNITED ST2 MFGR#1424 57589 +1992 UNITED ST2 MFGR#1427 1685113 +1992 UNITED ST2 MFGR#143 1865250 +1992 UNITED ST2 MFGR#1430 6454140 +1992 UNITED ST3 MFGR#141 10985538 +1992 UNITED ST3 MFGR#1410 5661337 +1992 UNITED ST3 MFGR#1411 2913347 +1992 UNITED ST3 MFGR#1412 3456611 +1992 UNITED ST3 MFGR#1413 1843964 +1992 UNITED ST3 MFGR#1415 741657 +1992 UNITED ST3 MFGR#1416 12592059 +1992 UNITED ST3 MFGR#1417 343888 +1992 UNITED ST3 MFGR#1419 2221452 +1992 UNITED ST3 MFGR#142 579832 +1992 UNITED ST3 MFGR#1421 2176380 +1992 UNITED ST3 MFGR#1422 1841100 +1992 UNITED ST3 MFGR#1423 121681 +1992 UNITED ST3 MFGR#1425 3025877 +1992 UNITED ST3 MFGR#1426 2812288 +1992 UNITED ST3 MFGR#1427 7624182 +1992 UNITED ST3 MFGR#1428 9484717 +1992 UNITED ST3 MFGR#1429 13290434 +1992 UNITED ST3 MFGR#1430 5263185 +1992 UNITED ST3 MFGR#1431 1057990 +1992 UNITED ST3 MFGR#1433 3172704 +1992 UNITED ST3 MFGR#1436 3292838 +1992 UNITED ST3 MFGR#1438 10257150 +1992 UNITED ST3 MFGR#144 4992624 +1992 UNITED ST3 MFGR#145 4555258 +1992 UNITED ST3 MFGR#146 5155417 +1992 UNITED ST3 MFGR#147 6044237 +1992 UNITED ST3 MFGR#148 12453066 +1992 UNITED ST3 MFGR#149 4283724 +1992 UNITED ST6 MFGR#1413 4420349 +1992 UNITED ST6 MFGR#1415 6201665 +1992 UNITED ST6 MFGR#1416 2588026 +1992 UNITED ST6 MFGR#142 55047 +1992 UNITED ST6 MFGR#1420 1288451 +1992 UNITED ST6 MFGR#1421 2567818 +1992 UNITED ST6 MFGR#1423 583540 +1992 UNITED ST6 MFGR#1426 6446245 +1992 UNITED ST6 MFGR#1429 1693359 +1992 UNITED ST6 MFGR#1430 954778 +1992 UNITED ST6 MFGR#1436 2419392 +1992 UNITED ST6 MFGR#1437 7302587 +1992 UNITED ST6 MFGR#144 8794123 +1992 UNITED ST6 MFGR#1440 3993789 +1992 UNITED ST7 MFGR#141 6582861 +1992 UNITED ST7 MFGR#1411 692197 +1992 UNITED ST7 MFGR#1412 169740 +1992 UNITED ST7 MFGR#1413 4359125 +1992 UNITED ST7 MFGR#1420 2500664 +1992 UNITED ST7 MFGR#1424 5152200 +1992 UNITED ST7 MFGR#1426 8490217 +1992 UNITED ST7 MFGR#143 5620388 +1992 UNITED ST7 MFGR#1431 1235687 +1992 UNITED ST7 MFGR#1432 6829944 +1992 UNITED ST7 MFGR#1440 6253049 +1992 UNITED ST7 MFGR#145 3732260 +1992 UNITED ST7 MFGR#147 2421233 +1992 UNITED ST7 MFGR#149 1866816 +1992 UNITED ST9 MFGR#141 147955 +1992 UNITED ST9 MFGR#1413 5392214 +1992 UNITED ST9 MFGR#1414 6319659 +1992 UNITED ST9 MFGR#1415 1084942 +1992 UNITED ST9 MFGR#1416 9714488 +1992 UNITED ST9 MFGR#1421 5856389 +1992 UNITED ST9 MFGR#1422 5439287 +1992 UNITED ST9 MFGR#1423 7040305 +1992 UNITED ST9 MFGR#1426 6595331 +1992 UNITED ST9 MFGR#1428 446309 +1992 UNITED ST9 MFGR#1429 3993195 +1992 UNITED ST9 MFGR#143 2320022 +1992 UNITED ST9 MFGR#1433 8938527 +1992 UNITED ST9 MFGR#1434 3280337 +1992 UNITED ST9 MFGR#1435 11735187 +1992 UNITED ST9 MFGR#1438 2940085 +1992 UNITED ST9 MFGR#145 2145463 +1992 UNITED ST9 MFGR#147 5798259 +1992 UNITED ST9 MFGR#148 4292580 -- !query4_3_after -- -1997 UNITED ST0 MFGR#141 15456288 -1997 UNITED ST0 MFGR#1410 32961113 -1997 UNITED ST0 MFGR#1411 36166400 -1997 UNITED ST0 MFGR#1412 18237451 -1997 UNITED ST0 MFGR#1413 40752813 -1997 UNITED ST0 MFGR#1414 24237588 -1997 UNITED ST0 MFGR#1415 6419001 -1997 UNITED ST0 MFGR#1416 15639876 -1997 UNITED ST0 MFGR#1417 15476211 -1997 UNITED ST0 MFGR#1418 8169956 -1997 UNITED ST0 MFGR#1419 18217252 -1997 UNITED ST0 MFGR#142 29361222 -1997 UNITED ST0 MFGR#1420 34587603 -1997 UNITED ST0 MFGR#1421 28560921 -1997 UNITED ST0 MFGR#1422 25154181 -1997 UNITED ST0 MFGR#1423 18144008 -1997 UNITED ST0 MFGR#1424 28064267 -1997 UNITED ST0 MFGR#1425 15255845 -1997 UNITED ST0 MFGR#1426 5610353 -1997 UNITED ST0 MFGR#1427 5750693 -1997 UNITED ST0 MFGR#1428 14081707 -1997 UNITED ST0 MFGR#1429 9024512 -1997 UNITED ST0 MFGR#143 5325193 -1997 UNITED ST0 MFGR#1430 16961739 -1997 UNITED ST0 MFGR#1431 19015253 -1997 UNITED ST0 MFGR#1432 6324917 -1997 UNITED ST0 MFGR#1433 6734461 -1997 UNITED ST0 MFGR#1434 19206393 -1997 UNITED ST0 MFGR#1435 679925 -1997 UNITED ST0 MFGR#1436 13834012 -1997 UNITED ST0 MFGR#1437 23575407 -1997 UNITED ST0 MFGR#1438 24756074 -1997 UNITED ST0 MFGR#1439 19644072 -1997 UNITED ST0 MFGR#144 20285157 -1997 UNITED ST0 MFGR#1440 14068558 -1997 UNITED ST0 MFGR#145 22646138 -1997 UNITED ST0 MFGR#146 16381159 -1997 UNITED ST0 MFGR#147 12267827 -1997 UNITED ST0 MFGR#148 30968614 -1997 UNITED ST0 MFGR#149 8640669 -1997 UNITED ST1 MFGR#141 10152087 -1997 UNITED ST1 MFGR#1410 32583300 -1997 UNITED ST1 MFGR#1411 15690352 -1997 UNITED ST1 MFGR#1412 16671403 -1997 UNITED ST1 MFGR#1413 4942775 -1997 UNITED ST1 MFGR#1414 9391597 -1997 UNITED ST1 MFGR#1415 26937296 -1997 UNITED ST1 MFGR#1416 25782871 -1997 UNITED ST1 MFGR#1417 23349641 -1997 UNITED ST1 MFGR#1418 10934620 -1997 UNITED ST1 MFGR#1419 22673030 -1997 UNITED ST1 MFGR#142 14048276 -1997 UNITED ST1 MFGR#1420 18100795 -1997 UNITED ST1 MFGR#1421 25405678 -1997 UNITED ST1 MFGR#1423 5713800 -1997 UNITED ST1 MFGR#1424 4131923 -1997 UNITED ST1 MFGR#1425 17190425 -1997 UNITED ST1 MFGR#1426 22314044 -1997 UNITED ST1 MFGR#1427 13635057 -1997 UNITED ST1 MFGR#1428 17735811 -1997 UNITED ST1 MFGR#1429 28740911 -1997 UNITED ST1 MFGR#143 27418033 -1997 UNITED ST1 MFGR#1430 18655292 -1997 UNITED ST1 MFGR#1431 11074399 -1997 UNITED ST1 MFGR#1432 7490170 -1997 UNITED ST1 MFGR#1433 28997266 -1997 UNITED ST1 MFGR#1434 4612211 -1997 UNITED ST1 MFGR#1435 16327124 -1997 UNITED ST1 MFGR#1436 17409665 -1997 UNITED ST1 MFGR#1437 33391635 -1997 UNITED ST1 MFGR#1438 37321134 -1997 UNITED ST1 MFGR#1439 10279941 -1997 UNITED ST1 MFGR#144 12304723 -1997 UNITED ST1 MFGR#1440 28921230 -1997 UNITED ST1 MFGR#145 17933684 -1997 UNITED ST1 MFGR#146 8709103 -1997 UNITED ST1 MFGR#147 20145983 -1997 UNITED ST1 MFGR#148 16940400 -1997 UNITED ST1 MFGR#149 10284949 -1997 UNITED ST2 MFGR#1412 6328240 -1997 UNITED ST2 MFGR#1415 5022682 -1997 UNITED ST2 MFGR#1416 4102214 -1997 UNITED ST2 MFGR#1417 5284444 -1997 UNITED ST2 MFGR#1418 2649908 -1997 UNITED ST2 MFGR#1419 4664673 -1997 UNITED ST2 MFGR#142 10784835 -1997 UNITED ST2 MFGR#1423 4095190 -1997 UNITED ST2 MFGR#1424 5376355 -1997 UNITED ST2 MFGR#1425 3949957 -1997 UNITED ST2 MFGR#143 2859829 -1997 UNITED ST2 MFGR#1430 4618266 -1997 UNITED ST2 MFGR#1432 5183060 -1997 UNITED ST2 MFGR#1433 10531371 -1997 UNITED ST2 MFGR#1434 77646 -1997 UNITED ST2 MFGR#1436 6538508 -1997 UNITED ST2 MFGR#1438 12324747 -1997 UNITED ST2 MFGR#144 2002870 -1997 UNITED ST2 MFGR#1440 12247229 -1997 UNITED ST2 MFGR#145 202685 -1997 UNITED ST2 MFGR#148 5998932 -1997 UNITED ST3 MFGR#141 12866083 -1997 UNITED ST3 MFGR#1410 31051703 -1997 UNITED ST3 MFGR#1411 14099543 -1997 UNITED ST3 MFGR#1412 4970551 -1997 UNITED ST3 MFGR#1413 26129762 -1997 UNITED ST3 MFGR#1414 22364787 -1997 UNITED ST3 MFGR#1415 27883428 -1997 UNITED ST3 MFGR#1416 25289630 -1997 UNITED ST3 MFGR#1417 5524810 -1997 UNITED ST3 MFGR#1419 3616608 -1997 UNITED ST3 MFGR#142 10776059 -1997 UNITED ST3 MFGR#1420 12958851 -1997 UNITED ST3 MFGR#1421 5815494 -1997 UNITED ST3 MFGR#1422 17694270 -1997 UNITED ST3 MFGR#1423 10285142 -1997 UNITED ST3 MFGR#1424 21555317 -1997 UNITED ST3 MFGR#1425 5214517 -1997 UNITED ST3 MFGR#1426 16810700 -1997 UNITED ST3 MFGR#1427 10086430 -1997 UNITED ST3 MFGR#1428 7733716 -1997 UNITED ST3 MFGR#1429 11922430 -1997 UNITED ST3 MFGR#1430 20105292 -1997 UNITED ST3 MFGR#1431 14716979 -1997 UNITED ST3 MFGR#1432 15779247 -1997 UNITED ST3 MFGR#1433 15308773 -1997 UNITED ST3 MFGR#1434 8515037 -1997 UNITED ST3 MFGR#1435 25486899 -1997 UNITED ST3 MFGR#1436 12739448 -1997 UNITED ST3 MFGR#1437 11249180 -1997 UNITED ST3 MFGR#1438 4156824 -1997 UNITED ST3 MFGR#1439 18358277 -1997 UNITED ST3 MFGR#144 10097718 -1997 UNITED ST3 MFGR#1440 22920494 -1997 UNITED ST3 MFGR#145 20285859 -1997 UNITED ST3 MFGR#146 18406423 -1997 UNITED ST3 MFGR#147 17295990 -1997 UNITED ST3 MFGR#148 26127145 -1997 UNITED ST3 MFGR#149 19895538 -1997 UNITED ST4 MFGR#141 8135477 -1997 UNITED ST4 MFGR#1410 5503247 -1997 UNITED ST4 MFGR#1411 6096435 -1997 UNITED ST4 MFGR#1412 19976043 -1997 UNITED ST4 MFGR#1413 22581282 -1997 UNITED ST4 MFGR#1414 10101379 -1997 UNITED ST4 MFGR#1415 8877444 -1997 UNITED ST4 MFGR#1416 17943398 -1997 UNITED ST4 MFGR#1417 9224501 -1997 UNITED ST4 MFGR#1418 16255176 -1997 UNITED ST4 MFGR#1419 683069 -1997 UNITED ST4 MFGR#142 23512944 -1997 UNITED ST4 MFGR#1420 27954262 -1997 UNITED ST4 MFGR#1421 54209185 -1997 UNITED ST4 MFGR#1422 1173957 -1997 UNITED ST4 MFGR#1423 20410710 -1997 UNITED ST4 MFGR#1424 2889270 -1997 UNITED ST4 MFGR#1425 13361796 -1997 UNITED ST4 MFGR#1426 18678819 -1997 UNITED ST4 MFGR#1427 18206066 -1997 UNITED ST4 MFGR#1428 18153974 -1997 UNITED ST4 MFGR#1429 12545076 -1997 UNITED ST4 MFGR#143 6158834 -1997 UNITED ST4 MFGR#1430 16128370 -1997 UNITED ST4 MFGR#1431 11153717 -1997 UNITED ST4 MFGR#1432 7745636 -1997 UNITED ST4 MFGR#1433 15983634 -1997 UNITED ST4 MFGR#1434 3662246 -1997 UNITED ST4 MFGR#1435 3060180 -1997 UNITED ST4 MFGR#1436 11524592 -1997 UNITED ST4 MFGR#1437 36824174 -1997 UNITED ST4 MFGR#1438 8226666 -1997 UNITED ST4 MFGR#1439 9872470 -1997 UNITED ST4 MFGR#1440 401207 -1997 UNITED ST4 MFGR#145 11698312 -1997 UNITED ST4 MFGR#146 18259956 -1997 UNITED ST4 MFGR#147 10256110 -1997 UNITED ST4 MFGR#148 15892628 -1997 UNITED ST4 MFGR#149 4322491 -1997 UNITED ST5 MFGR#141 8534275 -1997 UNITED ST5 MFGR#1411 2427261 -1997 UNITED ST5 MFGR#1412 9032422 -1997 UNITED ST5 MFGR#1413 6343588 -1997 UNITED ST5 MFGR#1414 2961025 -1997 UNITED ST5 MFGR#1415 21186568 -1997 UNITED ST5 MFGR#1416 9354080 -1997 UNITED ST5 MFGR#1417 11786246 -1997 UNITED ST5 MFGR#1418 15297245 -1997 UNITED ST5 MFGR#1420 15787608 -1997 UNITED ST5 MFGR#1421 17858301 -1997 UNITED ST5 MFGR#1422 14080987 -1997 UNITED ST5 MFGR#1423 12535186 -1997 UNITED ST5 MFGR#1424 3221960 -1997 UNITED ST5 MFGR#1425 8950317 -1997 UNITED ST5 MFGR#1426 13161486 -1997 UNITED ST5 MFGR#1427 6149737 -1997 UNITED ST5 MFGR#1428 6796520 -1997 UNITED ST5 MFGR#1429 26026300 -1997 UNITED ST5 MFGR#143 6712572 -1997 UNITED ST5 MFGR#1430 16092131 -1997 UNITED ST5 MFGR#1431 3425291 -1997 UNITED ST5 MFGR#1432 14630175 -1997 UNITED ST5 MFGR#1433 4967585 -1997 UNITED ST5 MFGR#1434 6716261 -1997 UNITED ST5 MFGR#1435 6290790 -1997 UNITED ST5 MFGR#1436 6787699 -1997 UNITED ST5 MFGR#1437 6045168 -1997 UNITED ST5 MFGR#1438 1082877 -1997 UNITED ST5 MFGR#1439 5268611 -1997 UNITED ST5 MFGR#144 8900840 -1997 UNITED ST5 MFGR#1440 2644336 -1997 UNITED ST5 MFGR#145 18054844 -1997 UNITED ST5 MFGR#146 4893668 -1997 UNITED ST5 MFGR#147 9615891 -1997 UNITED ST5 MFGR#148 1426124 -1997 UNITED ST5 MFGR#149 7320726 -1997 UNITED ST6 MFGR#141 10051337 -1997 UNITED ST6 MFGR#1410 23352963 -1997 UNITED ST6 MFGR#1411 1537303 -1997 UNITED ST6 MFGR#1412 4332646 -1997 UNITED ST6 MFGR#1413 14686752 -1997 UNITED ST6 MFGR#1414 7226409 -1997 UNITED ST6 MFGR#1415 15062686 -1997 UNITED ST6 MFGR#1416 8606480 -1997 UNITED ST6 MFGR#1417 7662761 -1997 UNITED ST6 MFGR#1418 1810860 -1997 UNITED ST6 MFGR#1419 20179568 -1997 UNITED ST6 MFGR#142 15506072 -1997 UNITED ST6 MFGR#1420 3844271 -1997 UNITED ST6 MFGR#1421 10608709 -1997 UNITED ST6 MFGR#1422 16083699 -1997 UNITED ST6 MFGR#1423 6789571 -1997 UNITED ST6 MFGR#1424 17631950 -1997 UNITED ST6 MFGR#1425 8477221 -1997 UNITED ST6 MFGR#1426 16847029 -1997 UNITED ST6 MFGR#1428 17314796 -1997 UNITED ST6 MFGR#1429 19175871 -1997 UNITED ST6 MFGR#143 8775848 -1997 UNITED ST6 MFGR#1431 5319063 -1997 UNITED ST6 MFGR#1432 7875606 -1997 UNITED ST6 MFGR#1433 9305264 -1997 UNITED ST6 MFGR#1434 8050404 -1997 UNITED ST6 MFGR#1435 17585836 -1997 UNITED ST6 MFGR#1436 13745758 -1997 UNITED ST6 MFGR#1438 14536324 -1997 UNITED ST6 MFGR#1439 11451329 -1997 UNITED ST6 MFGR#144 10794837 -1997 UNITED ST6 MFGR#1440 5719238 -1997 UNITED ST6 MFGR#145 25449058 -1997 UNITED ST6 MFGR#146 17821996 -1997 UNITED ST6 MFGR#147 18922333 -1997 UNITED ST6 MFGR#148 10825718 -1997 UNITED ST6 MFGR#149 14950868 -1997 UNITED ST7 MFGR#141 8291476 -1997 UNITED ST7 MFGR#1410 4622953 -1997 UNITED ST7 MFGR#1411 14885507 -1997 UNITED ST7 MFGR#1412 19011375 -1997 UNITED ST7 MFGR#1413 31178370 -1997 UNITED ST7 MFGR#1414 16429613 -1997 UNITED ST7 MFGR#1415 3091832 -1997 UNITED ST7 MFGR#1416 17008243 -1997 UNITED ST7 MFGR#1417 15609283 -1997 UNITED ST7 MFGR#1418 5059394 -1997 UNITED ST7 MFGR#1419 7234721 -1997 UNITED ST7 MFGR#142 6898938 -1997 UNITED ST7 MFGR#1420 6426590 -1997 UNITED ST7 MFGR#1421 6043221 -1997 UNITED ST7 MFGR#1422 13273021 -1997 UNITED ST7 MFGR#1423 30537249 -1997 UNITED ST7 MFGR#1424 9224296 -1997 UNITED ST7 MFGR#1425 12799069 -1997 UNITED ST7 MFGR#1426 30277891 -1997 UNITED ST7 MFGR#1427 8331493 -1997 UNITED ST7 MFGR#1428 16205423 -1997 UNITED ST7 MFGR#1429 14396627 -1997 UNITED ST7 MFGR#143 10209623 -1997 UNITED ST7 MFGR#1430 16556758 -1997 UNITED ST7 MFGR#1431 15775327 -1997 UNITED ST7 MFGR#1432 24602989 -1997 UNITED ST7 MFGR#1433 22927717 -1997 UNITED ST7 MFGR#1434 9486462 -1997 UNITED ST7 MFGR#1435 14254290 -1997 UNITED ST7 MFGR#1436 8645365 -1997 UNITED ST7 MFGR#1437 19026166 -1997 UNITED ST7 MFGR#1438 11291550 -1997 UNITED ST7 MFGR#1439 9590292 -1997 UNITED ST7 MFGR#144 14110404 -1997 UNITED ST7 MFGR#1440 21877355 -1997 UNITED ST7 MFGR#145 8246717 -1997 UNITED ST7 MFGR#146 9785781 -1997 UNITED ST7 MFGR#147 13412691 -1997 UNITED ST7 MFGR#148 14235939 -1997 UNITED ST7 MFGR#149 23450309 -1997 UNITED ST8 MFGR#141 7323416 -1997 UNITED ST8 MFGR#1410 7191961 -1997 UNITED ST8 MFGR#1411 2425849 -1997 UNITED ST8 MFGR#1412 26725641 -1997 UNITED ST8 MFGR#1413 7915896 -1997 UNITED ST8 MFGR#1414 14359044 -1997 UNITED ST8 MFGR#1415 15834927 -1997 UNITED ST8 MFGR#1416 7464470 -1997 UNITED ST8 MFGR#1417 15274496 -1997 UNITED ST8 MFGR#1418 3261391 -1997 UNITED ST8 MFGR#1419 8020386 -1997 UNITED ST8 MFGR#142 5985550 -1997 UNITED ST8 MFGR#1420 9441461 -1997 UNITED ST8 MFGR#1421 2998735 -1997 UNITED ST8 MFGR#1423 1565415 -1997 UNITED ST8 MFGR#1424 5613094 -1997 UNITED ST8 MFGR#1425 7186734 -1997 UNITED ST8 MFGR#1426 3153117 -1997 UNITED ST8 MFGR#1427 11303043 -1997 UNITED ST8 MFGR#1428 3564003 -1997 UNITED ST8 MFGR#143 6357477 -1997 UNITED ST8 MFGR#1430 6633950 -1997 UNITED ST8 MFGR#1431 867285 -1997 UNITED ST8 MFGR#1432 9962884 -1997 UNITED ST8 MFGR#1433 13708011 -1997 UNITED ST8 MFGR#1434 7357067 -1997 UNITED ST8 MFGR#1435 11993453 -1997 UNITED ST8 MFGR#1436 7231938 -1997 UNITED ST8 MFGR#1437 18632056 -1997 UNITED ST8 MFGR#1438 3609384 -1997 UNITED ST8 MFGR#144 4822734 -1997 UNITED ST8 MFGR#1440 2521917 -1997 UNITED ST8 MFGR#145 11683385 -1997 UNITED ST8 MFGR#146 3350415 -1997 UNITED ST8 MFGR#147 8425393 -1997 UNITED ST8 MFGR#148 15005341 -1997 UNITED ST8 MFGR#149 9018101 -1997 UNITED ST9 MFGR#141 21838850 -1997 UNITED ST9 MFGR#1410 8455387 -1997 UNITED ST9 MFGR#1411 27984689 -1997 UNITED ST9 MFGR#1412 26491401 -1997 UNITED ST9 MFGR#1413 30237705 -1997 UNITED ST9 MFGR#1414 8957476 -1997 UNITED ST9 MFGR#1415 35391583 -1997 UNITED ST9 MFGR#1416 8923782 -1997 UNITED ST9 MFGR#1417 3210984 -1997 UNITED ST9 MFGR#1418 12102607 -1997 UNITED ST9 MFGR#1419 14468627 -1997 UNITED ST9 MFGR#142 33363582 -1997 UNITED ST9 MFGR#1420 13139731 -1997 UNITED ST9 MFGR#1421 8410650 -1997 UNITED ST9 MFGR#1422 11268410 -1997 UNITED ST9 MFGR#1423 2846614 -1997 UNITED ST9 MFGR#1424 2305872 -1997 UNITED ST9 MFGR#1425 4896060 -1997 UNITED ST9 MFGR#1426 31544772 -1997 UNITED ST9 MFGR#1427 4113085 -1997 UNITED ST9 MFGR#1428 11137081 -1997 UNITED ST9 MFGR#1429 10633230 -1997 UNITED ST9 MFGR#143 11438203 -1997 UNITED ST9 MFGR#1430 2727389 -1997 UNITED ST9 MFGR#1431 10548538 -1997 UNITED ST9 MFGR#1433 26564495 -1997 UNITED ST9 MFGR#1434 11948206 -1997 UNITED ST9 MFGR#1435 2201909 -1997 UNITED ST9 MFGR#1436 10347631 -1997 UNITED ST9 MFGR#1437 40051690 -1997 UNITED ST9 MFGR#1438 2662945 -1997 UNITED ST9 MFGR#1439 18011455 -1997 UNITED ST9 MFGR#144 10202784 -1997 UNITED ST9 MFGR#1440 11123196 -1997 UNITED ST9 MFGR#145 11326624 -1997 UNITED ST9 MFGR#146 27825456 -1997 UNITED ST9 MFGR#147 17579502 -1997 UNITED ST9 MFGR#148 17001116 -1998 UNITED ST0 MFGR#141 12484786 -1998 UNITED ST0 MFGR#1410 5225999 -1998 UNITED ST0 MFGR#1411 2633395 -1998 UNITED ST0 MFGR#1412 9479536 -1998 UNITED ST0 MFGR#1413 18468711 -1998 UNITED ST0 MFGR#1414 19440601 -1998 UNITED ST0 MFGR#1415 16894643 -1998 UNITED ST0 MFGR#1416 7520117 -1998 UNITED ST0 MFGR#1417 464576 -1998 UNITED ST0 MFGR#1418 5171747 -1998 UNITED ST0 MFGR#1419 5970985 -1998 UNITED ST0 MFGR#142 23830409 -1998 UNITED ST0 MFGR#1420 2568835 -1998 UNITED ST0 MFGR#1421 6096442 -1998 UNITED ST0 MFGR#1422 10099456 -1998 UNITED ST0 MFGR#1423 24617746 -1998 UNITED ST0 MFGR#1424 22853888 -1998 UNITED ST0 MFGR#1425 25622112 -1998 UNITED ST0 MFGR#1426 9630701 -1998 UNITED ST0 MFGR#1427 4952625 -1998 UNITED ST0 MFGR#1428 4675471 -1998 UNITED ST0 MFGR#1429 19713527 -1998 UNITED ST0 MFGR#143 4971265 -1998 UNITED ST0 MFGR#1430 13948705 -1998 UNITED ST0 MFGR#1431 2955181 -1998 UNITED ST0 MFGR#1432 17946759 -1998 UNITED ST0 MFGR#1433 23325027 -1998 UNITED ST0 MFGR#1434 8580738 -1998 UNITED ST0 MFGR#1435 6663914 -1998 UNITED ST0 MFGR#1436 13468582 -1998 UNITED ST0 MFGR#1437 9419979 -1998 UNITED ST0 MFGR#1438 16164522 -1998 UNITED ST0 MFGR#1439 8692866 -1998 UNITED ST0 MFGR#144 17338711 -1998 UNITED ST0 MFGR#1440 8513736 -1998 UNITED ST0 MFGR#145 6212569 -1998 UNITED ST0 MFGR#146 2702349 -1998 UNITED ST0 MFGR#149 8844540 -1998 UNITED ST1 MFGR#141 13449110 -1998 UNITED ST1 MFGR#1410 5224626 -1998 UNITED ST1 MFGR#1411 16046399 -1998 UNITED ST1 MFGR#1412 8424392 -1998 UNITED ST1 MFGR#1413 7634982 -1998 UNITED ST1 MFGR#1414 8711782 -1998 UNITED ST1 MFGR#1415 5991201 -1998 UNITED ST1 MFGR#1416 11059645 -1998 UNITED ST1 MFGR#1417 8108087 -1998 UNITED ST1 MFGR#1418 1710875 -1998 UNITED ST1 MFGR#1419 452298 -1998 UNITED ST1 MFGR#142 6186526 -1998 UNITED ST1 MFGR#1421 4766963 -1998 UNITED ST1 MFGR#1422 5556563 -1998 UNITED ST1 MFGR#1423 12110257 -1998 UNITED ST1 MFGR#1424 8924948 -1998 UNITED ST1 MFGR#1425 5684710 -1998 UNITED ST1 MFGR#1426 6844444 -1998 UNITED ST1 MFGR#1427 7625525 -1998 UNITED ST1 MFGR#1428 5227395 -1998 UNITED ST1 MFGR#1429 2855272 -1998 UNITED ST1 MFGR#143 6964811 -1998 UNITED ST1 MFGR#1430 16952380 -1998 UNITED ST1 MFGR#1431 7655883 -1998 UNITED ST1 MFGR#1432 2975960 -1998 UNITED ST1 MFGR#1433 10175594 -1998 UNITED ST1 MFGR#1434 26358848 -1998 UNITED ST1 MFGR#1435 11408623 -1998 UNITED ST1 MFGR#1436 4438242 -1998 UNITED ST1 MFGR#1437 10014906 -1998 UNITED ST1 MFGR#1438 18817171 -1998 UNITED ST1 MFGR#1439 2129973 -1998 UNITED ST1 MFGR#144 9512727 -1998 UNITED ST1 MFGR#1440 20185179 -1998 UNITED ST1 MFGR#145 823605 -1998 UNITED ST1 MFGR#146 9358026 -1998 UNITED ST1 MFGR#147 6552312 -1998 UNITED ST1 MFGR#148 9144461 -1998 UNITED ST1 MFGR#149 2149639 -1998 UNITED ST2 MFGR#1410 8044368 -1998 UNITED ST2 MFGR#1413 727343 -1998 UNITED ST2 MFGR#1414 251088 -1998 UNITED ST2 MFGR#1416 5224696 -1998 UNITED ST2 MFGR#1419 7917142 -1998 UNITED ST2 MFGR#1420 9848986 -1998 UNITED ST2 MFGR#1427 4816976 -1998 UNITED ST2 MFGR#1429 4196069 -1998 UNITED ST2 MFGR#143 10073044 -1998 UNITED ST2 MFGR#1431 877123 -1998 UNITED ST2 MFGR#1433 3740836 -1998 UNITED ST2 MFGR#1434 857691 -1998 UNITED ST2 MFGR#1435 2277870 -1998 UNITED ST2 MFGR#145 4906521 -1998 UNITED ST2 MFGR#149 842348 -1998 UNITED ST3 MFGR#141 4181582 -1998 UNITED ST3 MFGR#1410 3169761 -1998 UNITED ST3 MFGR#1412 6590440 -1998 UNITED ST3 MFGR#1413 2015369 -1998 UNITED ST3 MFGR#1414 4408466 -1998 UNITED ST3 MFGR#1415 3963826 -1998 UNITED ST3 MFGR#1416 3179540 -1998 UNITED ST3 MFGR#1417 11440999 -1998 UNITED ST3 MFGR#1418 8725751 -1998 UNITED ST3 MFGR#1419 10646990 -1998 UNITED ST3 MFGR#142 5056172 -1998 UNITED ST3 MFGR#1420 3583150 -1998 UNITED ST3 MFGR#1421 7085689 -1998 UNITED ST3 MFGR#1422 20858904 -1998 UNITED ST3 MFGR#1423 12825743 -1998 UNITED ST3 MFGR#1424 3533518 -1998 UNITED ST3 MFGR#1425 4938324 -1998 UNITED ST3 MFGR#1426 15164664 -1998 UNITED ST3 MFGR#1427 2619162 -1998 UNITED ST3 MFGR#1428 7723699 -1998 UNITED ST3 MFGR#143 24362893 -1998 UNITED ST3 MFGR#1430 7139650 -1998 UNITED ST3 MFGR#1431 10306394 -1998 UNITED ST3 MFGR#1432 14769004 -1998 UNITED ST3 MFGR#1433 9610545 -1998 UNITED ST3 MFGR#1435 6611760 -1998 UNITED ST3 MFGR#1436 941348 -1998 UNITED ST3 MFGR#1437 4802162 -1998 UNITED ST3 MFGR#1438 5079750 -1998 UNITED ST3 MFGR#1439 5905118 -1998 UNITED ST3 MFGR#144 5436839 -1998 UNITED ST3 MFGR#1440 9815517 -1998 UNITED ST3 MFGR#145 7580271 -1998 UNITED ST3 MFGR#146 14051404 -1998 UNITED ST3 MFGR#147 7786017 -1998 UNITED ST3 MFGR#149 1271813 -1998 UNITED ST4 MFGR#141 26703183 -1998 UNITED ST4 MFGR#1410 18094129 -1998 UNITED ST4 MFGR#1411 615999 -1998 UNITED ST4 MFGR#1412 13988709 -1998 UNITED ST4 MFGR#1413 10115647 -1998 UNITED ST4 MFGR#1414 1586551 -1998 UNITED ST4 MFGR#1415 14357244 -1998 UNITED ST4 MFGR#1416 9430036 -1998 UNITED ST4 MFGR#1417 17361773 -1998 UNITED ST4 MFGR#1418 10187338 -1998 UNITED ST4 MFGR#1419 7116094 -1998 UNITED ST4 MFGR#142 18454502 -1998 UNITED ST4 MFGR#1420 1621340 -1998 UNITED ST4 MFGR#1422 13861291 -1998 UNITED ST4 MFGR#1423 4208006 -1998 UNITED ST4 MFGR#1424 3242576 -1998 UNITED ST4 MFGR#1425 1470642 -1998 UNITED ST4 MFGR#1426 11987205 -1998 UNITED ST4 MFGR#1427 12975399 -1998 UNITED ST4 MFGR#1428 11756155 -1998 UNITED ST4 MFGR#1429 6956191 -1998 UNITED ST4 MFGR#143 11499476 -1998 UNITED ST4 MFGR#1430 4355096 -1998 UNITED ST4 MFGR#1431 20166612 -1998 UNITED ST4 MFGR#1432 8849789 -1998 UNITED ST4 MFGR#1433 10786037 -1998 UNITED ST4 MFGR#1435 13208151 -1998 UNITED ST4 MFGR#1436 4747808 -1998 UNITED ST4 MFGR#1437 12999351 -1998 UNITED ST4 MFGR#1438 17360787 -1998 UNITED ST4 MFGR#1439 2557841 -1998 UNITED ST4 MFGR#144 22030911 -1998 UNITED ST4 MFGR#145 5272280 -1998 UNITED ST4 MFGR#146 4307958 -1998 UNITED ST4 MFGR#147 7995050 -1998 UNITED ST4 MFGR#148 2348889 -1998 UNITED ST5 MFGR#141 8396190 -1998 UNITED ST5 MFGR#1410 17422889 -1998 UNITED ST5 MFGR#1411 764246 -1998 UNITED ST5 MFGR#1413 8051561 -1998 UNITED ST5 MFGR#1414 1500867 -1998 UNITED ST5 MFGR#1416 3874315 -1998 UNITED ST5 MFGR#1417 7550747 -1998 UNITED ST5 MFGR#1418 14241755 -1998 UNITED ST5 MFGR#1419 4893846 -1998 UNITED ST5 MFGR#142 5418299 -1998 UNITED ST5 MFGR#1420 3810727 -1998 UNITED ST5 MFGR#1421 3321336 -1998 UNITED ST5 MFGR#1423 5021100 -1998 UNITED ST5 MFGR#1424 6125645 -1998 UNITED ST5 MFGR#1426 880680 -1998 UNITED ST5 MFGR#1427 1284890 -1998 UNITED ST5 MFGR#1428 3813108 -1998 UNITED ST5 MFGR#1429 9429868 -1998 UNITED ST5 MFGR#143 15699981 -1998 UNITED ST5 MFGR#1430 7977654 -1998 UNITED ST5 MFGR#1431 5545969 -1998 UNITED ST5 MFGR#1432 1803434 -1998 UNITED ST5 MFGR#1435 12957056 -1998 UNITED ST5 MFGR#1436 20666755 -1998 UNITED ST5 MFGR#1437 9536733 -1998 UNITED ST5 MFGR#1439 11784214 -1998 UNITED ST5 MFGR#1440 613089 -1998 UNITED ST5 MFGR#145 8237944 -1998 UNITED ST5 MFGR#147 5370351 -1998 UNITED ST6 MFGR#141 6683977 -1998 UNITED ST6 MFGR#1411 11589040 -1998 UNITED ST6 MFGR#1412 4800954 -1998 UNITED ST6 MFGR#1416 5848432 -1998 UNITED ST6 MFGR#1417 1804324 -1998 UNITED ST6 MFGR#1418 4642727 -1998 UNITED ST6 MFGR#1419 10079621 -1998 UNITED ST6 MFGR#142 2533570 -1998 UNITED ST6 MFGR#1420 9016258 -1998 UNITED ST6 MFGR#1421 7636150 -1998 UNITED ST6 MFGR#1422 1272436 -1998 UNITED ST6 MFGR#1423 9600097 -1998 UNITED ST6 MFGR#1424 7036762 -1998 UNITED ST6 MFGR#1425 10867426 -1998 UNITED ST6 MFGR#1426 7033994 -1998 UNITED ST6 MFGR#1427 6481679 -1998 UNITED ST6 MFGR#1428 6269185 -1998 UNITED ST6 MFGR#1429 12055631 -1998 UNITED ST6 MFGR#143 6500459 -1998 UNITED ST6 MFGR#1430 6816243 -1998 UNITED ST6 MFGR#1432 7752973 -1998 UNITED ST6 MFGR#1436 6292813 -1998 UNITED ST6 MFGR#1437 2334894 -1998 UNITED ST6 MFGR#1438 4459455 -1998 UNITED ST6 MFGR#144 640220 -1998 UNITED ST6 MFGR#1440 748200 -1998 UNITED ST6 MFGR#145 7948794 -1998 UNITED ST6 MFGR#146 2734333 -1998 UNITED ST6 MFGR#147 2995632 -1998 UNITED ST6 MFGR#148 10695287 -1998 UNITED ST6 MFGR#149 1475864 -1998 UNITED ST7 MFGR#141 4816637 -1998 UNITED ST7 MFGR#1410 24393912 -1998 UNITED ST7 MFGR#1411 1283038 -1998 UNITED ST7 MFGR#1412 12607459 -1998 UNITED ST7 MFGR#1413 15421293 -1998 UNITED ST7 MFGR#1414 7681668 -1998 UNITED ST7 MFGR#1415 7138377 -1998 UNITED ST7 MFGR#1416 9246990 -1998 UNITED ST7 MFGR#1417 5242972 -1998 UNITED ST7 MFGR#1419 13548941 -1998 UNITED ST7 MFGR#142 17758762 -1998 UNITED ST7 MFGR#1420 11980437 -1998 UNITED ST7 MFGR#1421 11166952 -1998 UNITED ST7 MFGR#1422 9231883 -1998 UNITED ST7 MFGR#1424 5796389 -1998 UNITED ST7 MFGR#1425 4943582 -1998 UNITED ST7 MFGR#1426 16361100 -1998 UNITED ST7 MFGR#1427 9659296 -1998 UNITED ST7 MFGR#1428 2998020 -1998 UNITED ST7 MFGR#1429 5766768 -1998 UNITED ST7 MFGR#143 5526167 -1998 UNITED ST7 MFGR#1430 4191892 -1998 UNITED ST7 MFGR#1431 12979559 -1998 UNITED ST7 MFGR#1432 5832068 -1998 UNITED ST7 MFGR#1433 11410227 -1998 UNITED ST7 MFGR#1434 10525792 -1998 UNITED ST7 MFGR#1435 5982420 -1998 UNITED ST7 MFGR#1436 7142934 -1998 UNITED ST7 MFGR#1437 7499671 -1998 UNITED ST7 MFGR#1438 9177564 -1998 UNITED ST7 MFGR#1439 1974444 -1998 UNITED ST7 MFGR#144 2428463 -1998 UNITED ST7 MFGR#1440 3300838 -1998 UNITED ST7 MFGR#146 1771645 -1998 UNITED ST7 MFGR#147 4681835 -1998 UNITED ST7 MFGR#148 7124367 -1998 UNITED ST7 MFGR#149 11662970 -1998 UNITED ST8 MFGR#141 12753354 -1998 UNITED ST8 MFGR#1410 1097292 -1998 UNITED ST8 MFGR#1411 4069881 -1998 UNITED ST8 MFGR#1414 9474831 -1998 UNITED ST8 MFGR#1415 6406304 -1998 UNITED ST8 MFGR#1418 2392133 -1998 UNITED ST8 MFGR#1419 2953230 -1998 UNITED ST8 MFGR#142 10702563 -1998 UNITED ST8 MFGR#1420 3719432 -1998 UNITED ST8 MFGR#1421 3854657 -1998 UNITED ST8 MFGR#1422 9711968 -1998 UNITED ST8 MFGR#1424 1102516 -1998 UNITED ST8 MFGR#1426 8937532 -1998 UNITED ST8 MFGR#1427 5255637 -1998 UNITED ST8 MFGR#1428 1695112 -1998 UNITED ST8 MFGR#143 6231349 -1998 UNITED ST8 MFGR#1430 4678664 -1998 UNITED ST8 MFGR#1431 2459741 -1998 UNITED ST8 MFGR#1433 1724339 -1998 UNITED ST8 MFGR#1434 8222328 -1998 UNITED ST8 MFGR#1435 4087165 -1998 UNITED ST8 MFGR#1437 7379288 -1998 UNITED ST8 MFGR#1439 1947163 -1998 UNITED ST8 MFGR#144 21846266 -1998 UNITED ST8 MFGR#146 13768113 -1998 UNITED ST8 MFGR#147 5279643 -1998 UNITED ST8 MFGR#149 15221410 -1998 UNITED ST9 MFGR#141 5544000 -1998 UNITED ST9 MFGR#1410 7971104 -1998 UNITED ST9 MFGR#1411 26426609 -1998 UNITED ST9 MFGR#1412 1701777 -1998 UNITED ST9 MFGR#1413 18786672 -1998 UNITED ST9 MFGR#1414 15787780 -1998 UNITED ST9 MFGR#1415 11214818 -1998 UNITED ST9 MFGR#1416 2076120 -1998 UNITED ST9 MFGR#1417 14310334 -1998 UNITED ST9 MFGR#1418 3558355 -1998 UNITED ST9 MFGR#1419 6134867 -1998 UNITED ST9 MFGR#142 257897 -1998 UNITED ST9 MFGR#1420 7716338 -1998 UNITED ST9 MFGR#1421 10291775 -1998 UNITED ST9 MFGR#1422 6105716 -1998 UNITED ST9 MFGR#1423 11378613 -1998 UNITED ST9 MFGR#1424 1904550 -1998 UNITED ST9 MFGR#1425 18831147 -1998 UNITED ST9 MFGR#1426 18562642 -1998 UNITED ST9 MFGR#1427 12565890 -1998 UNITED ST9 MFGR#1428 10720914 -1998 UNITED ST9 MFGR#1429 21940718 -1998 UNITED ST9 MFGR#143 21480722 -1998 UNITED ST9 MFGR#1430 30935173 -1998 UNITED ST9 MFGR#1431 7793012 -1998 UNITED ST9 MFGR#1432 10026104 -1998 UNITED ST9 MFGR#1433 17874435 -1998 UNITED ST9 MFGR#1434 5493785 -1998 UNITED ST9 MFGR#1435 10738769 -1998 UNITED ST9 MFGR#1436 43504 -1998 UNITED ST9 MFGR#1437 13559646 -1998 UNITED ST9 MFGR#1438 12654258 -1998 UNITED ST9 MFGR#1439 5972923 -1998 UNITED ST9 MFGR#144 15702123 -1998 UNITED ST9 MFGR#1440 6681029 -1998 UNITED ST9 MFGR#145 920718 -1998 UNITED ST9 MFGR#146 10435931 -1998 UNITED ST9 MFGR#147 4759326 -1998 UNITED ST9 MFGR#148 9970848 -1998 UNITED ST9 MFGR#149 7510940 +1992 UNITED ST0 MFGR#141 555641 +1992 UNITED ST0 MFGR#1411 7079182 +1992 UNITED ST0 MFGR#1412 2872721 +1992 UNITED ST0 MFGR#1413 5471501 +1992 UNITED ST0 MFGR#1414 6180491 +1992 UNITED ST0 MFGR#1415 3832532 +1992 UNITED ST0 MFGR#1417 5090445 +1992 UNITED ST0 MFGR#142 4603254 +1992 UNITED ST0 MFGR#1421 717404 +1992 UNITED ST0 MFGR#1422 6204274 +1992 UNITED ST0 MFGR#1424 5518896 +1992 UNITED ST0 MFGR#1425 5763764 +1992 UNITED ST0 MFGR#1427 6180274 +1992 UNITED ST0 MFGR#1432 6428730 +1992 UNITED ST0 MFGR#1433 833533 +1992 UNITED ST0 MFGR#1434 13713612 +1992 UNITED ST0 MFGR#145 13934235 +1992 UNITED ST2 MFGR#1411 758915 +1992 UNITED ST2 MFGR#142 4221010 +1992 UNITED ST2 MFGR#1421 4315917 +1992 UNITED ST2 MFGR#1424 57589 +1992 UNITED ST2 MFGR#1427 1685113 +1992 UNITED ST2 MFGR#143 1865250 +1992 UNITED ST2 MFGR#1430 6454140 +1992 UNITED ST3 MFGR#141 10985538 +1992 UNITED ST3 MFGR#1410 5661337 +1992 UNITED ST3 MFGR#1411 2913347 +1992 UNITED ST3 MFGR#1412 3456611 +1992 UNITED ST3 MFGR#1413 1843964 +1992 UNITED ST3 MFGR#1415 741657 +1992 UNITED ST3 MFGR#1416 12592059 +1992 UNITED ST3 MFGR#1417 343888 +1992 UNITED ST3 MFGR#1419 2221452 +1992 UNITED ST3 MFGR#142 579832 +1992 UNITED ST3 MFGR#1421 2176380 +1992 UNITED ST3 MFGR#1422 1841100 +1992 UNITED ST3 MFGR#1423 121681 +1992 UNITED ST3 MFGR#1425 3025877 +1992 UNITED ST3 MFGR#1426 2812288 +1992 UNITED ST3 MFGR#1427 7624182 +1992 UNITED ST3 MFGR#1428 9484717 +1992 UNITED ST3 MFGR#1429 13290434 +1992 UNITED ST3 MFGR#1430 5263185 +1992 UNITED ST3 MFGR#1431 1057990 +1992 UNITED ST3 MFGR#1433 3172704 +1992 UNITED ST3 MFGR#1436 3292838 +1992 UNITED ST3 MFGR#1438 10257150 +1992 UNITED ST3 MFGR#144 4992624 +1992 UNITED ST3 MFGR#145 4555258 +1992 UNITED ST3 MFGR#146 5155417 +1992 UNITED ST3 MFGR#147 6044237 +1992 UNITED ST3 MFGR#148 12453066 +1992 UNITED ST3 MFGR#149 4283724 +1992 UNITED ST6 MFGR#1413 4420349 +1992 UNITED ST6 MFGR#1415 6201665 +1992 UNITED ST6 MFGR#1416 2588026 +1992 UNITED ST6 MFGR#142 55047 +1992 UNITED ST6 MFGR#1420 1288451 +1992 UNITED ST6 MFGR#1421 2567818 +1992 UNITED ST6 MFGR#1423 583540 +1992 UNITED ST6 MFGR#1426 6446245 +1992 UNITED ST6 MFGR#1429 1693359 +1992 UNITED ST6 MFGR#1430 954778 +1992 UNITED ST6 MFGR#1436 2419392 +1992 UNITED ST6 MFGR#1437 7302587 +1992 UNITED ST6 MFGR#144 8794123 +1992 UNITED ST6 MFGR#1440 3993789 +1992 UNITED ST7 MFGR#141 6582861 +1992 UNITED ST7 MFGR#1411 692197 +1992 UNITED ST7 MFGR#1412 169740 +1992 UNITED ST7 MFGR#1413 4359125 +1992 UNITED ST7 MFGR#1420 2500664 +1992 UNITED ST7 MFGR#1424 5152200 +1992 UNITED ST7 MFGR#1426 8490217 +1992 UNITED ST7 MFGR#143 5620388 +1992 UNITED ST7 MFGR#1431 1235687 +1992 UNITED ST7 MFGR#1432 6829944 +1992 UNITED ST7 MFGR#1440 6253049 +1992 UNITED ST7 MFGR#145 3732260 +1992 UNITED ST7 MFGR#147 2421233 +1992 UNITED ST7 MFGR#149 1866816 +1992 UNITED ST9 MFGR#141 147955 +1992 UNITED ST9 MFGR#1413 5392214 +1992 UNITED ST9 MFGR#1414 6319659 +1992 UNITED ST9 MFGR#1415 1084942 +1992 UNITED ST9 MFGR#1416 9714488 +1992 UNITED ST9 MFGR#1421 5856389 +1992 UNITED ST9 MFGR#1422 5439287 +1992 UNITED ST9 MFGR#1423 7040305 +1992 UNITED ST9 MFGR#1426 6595331 +1992 UNITED ST9 MFGR#1428 446309 +1992 UNITED ST9 MFGR#1429 3993195 +1992 UNITED ST9 MFGR#143 2320022 +1992 UNITED ST9 MFGR#1433 8938527 +1992 UNITED ST9 MFGR#1434 3280337 +1992 UNITED ST9 MFGR#1435 11735187 +1992 UNITED ST9 MFGR#1438 2940085 +1992 UNITED ST9 MFGR#145 2145463 +1992 UNITED ST9 MFGR#147 5798259 +1992 UNITED ST9 MFGR#148 4292580 diff --git a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query4.out b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query4.out index 8f376d296af9e4e..1946f8020ded90a 100644 --- a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query4.out +++ b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query4.out @@ -56,7 +56,10 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------------filter((t_w_firstyear.dyear = 1999) and (t_w_firstyear.sale_type = 'w') and (t_w_firstyear.year_total > 0.000000)) ----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) ------------------PhysicalProject ---------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t_s_firstyear.customer_id = t_c_secyear.customer_id)) otherCondition=((if((year_total > 0.000000), (cast(year_total as DECIMALV3(38, 16)) / year_total), NULL) > if((year_total > 0.000000), (cast(year_total as DECIMALV3(38, 16)) / year_total), NULL))) +--------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((t_s_firstyear.customer_id = t_c_secyear.customer_id)) otherCondition=((if((year_total > 0.000000), (cast(year_total as DECIMALV3(38, 16)) / year_total), NULL) > if((year_total > 0.000000), (cast(year_total as DECIMALV3(38, 16)) / year_total), NULL))) +----------------------PhysicalProject +------------------------filter((t_c_secyear.dyear = 2000) and (t_c_secyear.sale_type = 'c')) +--------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) ----------------------PhysicalProject ------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t_s_firstyear.customer_id = t_c_firstyear.customer_id)) otherCondition=() --------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((t_s_secyear.customer_id = t_s_firstyear.customer_id)) otherCondition=() @@ -69,7 +72,4 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------------------PhysicalProject ----------------------------filter((t_c_firstyear.dyear = 1999) and (t_c_firstyear.sale_type = 'c') and (t_c_firstyear.year_total > 0.000000)) ------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) -----------------------PhysicalProject -------------------------filter((t_c_secyear.dyear = 2000) and (t_c_secyear.sale_type = 'c')) ---------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query11.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query11.out index 531ca06c5a80bb8..01b1d2752e59b51 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query11.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query11.out @@ -40,10 +40,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------------filter((t_w_secyear.dyear = 2002) and (t_w_secyear.sale_type = 'w')) ------------------PhysicalCteConsumer ( cteId=CTEId#0 ) --------------PhysicalProject -----------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((t_s_firstyear.customer_id = t_w_firstyear.customer_id)) otherCondition=() -------------------PhysicalProject ---------------------filter((t_w_firstyear.dyear = 2001) and (t_w_firstyear.sale_type = 'w') and (t_w_firstyear.year_total > 0.00)) -----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +----------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t_s_firstyear.customer_id = t_w_firstyear.customer_id)) otherCondition=() ------------------hashJoin[INNER_JOIN shuffle] hashCondition=((t_s_secyear.customer_id = t_s_firstyear.customer_id)) otherCondition=() --------------------PhysicalProject ----------------------filter((t_s_secyear.dyear = 2002) and (t_s_secyear.sale_type = 's')) @@ -51,4 +48,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------------PhysicalProject ----------------------filter((t_s_firstyear.dyear = 2001) and (t_s_firstyear.sale_type = 's') and (t_s_firstyear.year_total > 0.00)) ------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +------------------PhysicalProject +--------------------filter((t_w_firstyear.dyear = 2001) and (t_w_firstyear.sale_type = 'w') and (t_w_firstyear.year_total > 0.00)) +----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query4.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query4.out index 3d35d56d81d4e1d..63f30b978f4d686 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query4.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query4.out @@ -51,20 +51,14 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------------filter((t_w_secyear.dyear = 2000) and (t_w_secyear.sale_type = 'w')) ------------------PhysicalCteConsumer ( cteId=CTEId#0 ) --------------PhysicalProject -----------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((t_s_firstyear.customer_id = t_w_firstyear.customer_id)) otherCondition=() -------------------PhysicalProject ---------------------filter((t_w_firstyear.dyear = 1999) and (t_w_firstyear.sale_type = 'w') and (t_w_firstyear.year_total > 0.000000)) -----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +----------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t_s_firstyear.customer_id = t_w_firstyear.customer_id)) otherCondition=() ------------------PhysicalProject --------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((t_s_firstyear.customer_id = t_c_secyear.customer_id)) otherCondition=((if((year_total > 0.000000), (cast(year_total as DECIMALV3(38, 16)) / year_total), NULL) > if((year_total > 0.000000), (cast(year_total as DECIMALV3(38, 16)) / year_total), NULL))) ----------------------PhysicalProject ------------------------filter((t_c_secyear.dyear = 2000) and (t_c_secyear.sale_type = 'c')) --------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) ----------------------PhysicalProject -------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((t_s_firstyear.customer_id = t_c_firstyear.customer_id)) otherCondition=() ---------------------------PhysicalProject -----------------------------filter((t_c_firstyear.dyear = 1999) and (t_c_firstyear.sale_type = 'c') and (t_c_firstyear.year_total > 0.000000)) -------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t_s_firstyear.customer_id = t_c_firstyear.customer_id)) otherCondition=() --------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((t_s_secyear.customer_id = t_s_firstyear.customer_id)) otherCondition=() ----------------------------PhysicalProject ------------------------------filter((t_s_secyear.dyear = 2000) and (t_s_secyear.sale_type = 's')) @@ -72,4 +66,10 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------------------------PhysicalProject ------------------------------filter((t_s_firstyear.dyear = 1999) and (t_s_firstyear.sale_type = 's') and (t_s_firstyear.year_total > 0.000000)) --------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +--------------------------PhysicalProject +----------------------------filter((t_c_firstyear.dyear = 1999) and (t_c_firstyear.sale_type = 'c') and (t_c_firstyear.year_total > 0.000000)) +------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +------------------PhysicalProject +--------------------filter((t_w_firstyear.dyear = 1999) and (t_w_firstyear.sale_type = 'w') and (t_w_firstyear.year_total > 0.000000)) +----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query11.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query11.out index fa8d314e00019dd..b44acb095192100 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query11.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query11.out @@ -40,10 +40,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------------filter((t_w_secyear.dyear = 2002) and (t_w_secyear.sale_type = 'w')) ------------------PhysicalCteConsumer ( cteId=CTEId#0 ) --------------PhysicalProject -----------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((t_s_firstyear.customer_id = t_w_firstyear.customer_id)) otherCondition=() -------------------PhysicalProject ---------------------filter((t_w_firstyear.dyear = 2001) and (t_w_firstyear.sale_type = 'w') and (t_w_firstyear.year_total > 0.00)) -----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +----------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t_s_firstyear.customer_id = t_w_firstyear.customer_id)) otherCondition=() ------------------hashJoin[INNER_JOIN shuffle] hashCondition=((t_s_secyear.customer_id = t_s_firstyear.customer_id)) otherCondition=() --------------------PhysicalProject ----------------------filter((t_s_secyear.dyear = 2002) and (t_s_secyear.sale_type = 's')) @@ -51,4 +48,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------------PhysicalProject ----------------------filter((t_s_firstyear.dyear = 2001) and (t_s_firstyear.sale_type = 's') and (t_s_firstyear.year_total > 0.00)) ------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +------------------PhysicalProject +--------------------filter((t_w_firstyear.dyear = 2001) and (t_w_firstyear.sale_type = 'w') and (t_w_firstyear.year_total > 0.00)) +----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query4.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query4.out index 27803aec4d8f64e..2ab0ebfe7fb09c0 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query4.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query4.out @@ -51,20 +51,14 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------------filter((t_w_secyear.dyear = 2000) and (t_w_secyear.sale_type = 'w')) ------------------PhysicalCteConsumer ( cteId=CTEId#0 ) --------------PhysicalProject -----------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((t_s_firstyear.customer_id = t_w_firstyear.customer_id)) otherCondition=() -------------------PhysicalProject ---------------------filter((t_w_firstyear.dyear = 1999) and (t_w_firstyear.sale_type = 'w') and (t_w_firstyear.year_total > 0.000000)) -----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +----------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t_s_firstyear.customer_id = t_w_firstyear.customer_id)) otherCondition=() ------------------PhysicalProject --------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((t_s_firstyear.customer_id = t_c_secyear.customer_id)) otherCondition=((if((year_total > 0.000000), (cast(year_total as DECIMALV3(38, 16)) / year_total), NULL) > if((year_total > 0.000000), (cast(year_total as DECIMALV3(38, 16)) / year_total), NULL))) ----------------------PhysicalProject ------------------------filter((t_c_secyear.dyear = 2000) and (t_c_secyear.sale_type = 'c')) --------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) ----------------------PhysicalProject -------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((t_s_firstyear.customer_id = t_c_firstyear.customer_id)) otherCondition=() ---------------------------PhysicalProject -----------------------------filter((t_c_firstyear.dyear = 1999) and (t_c_firstyear.sale_type = 'c') and (t_c_firstyear.year_total > 0.000000)) -------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t_s_firstyear.customer_id = t_c_firstyear.customer_id)) otherCondition=() --------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((t_s_secyear.customer_id = t_s_firstyear.customer_id)) otherCondition=() ----------------------------PhysicalProject ------------------------------filter((t_s_secyear.dyear = 2000) and (t_s_secyear.sale_type = 's')) @@ -72,4 +66,10 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------------------------PhysicalProject ------------------------------filter((t_s_firstyear.dyear = 1999) and (t_s_firstyear.sale_type = 's') and (t_s_firstyear.year_total > 0.000000)) --------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +--------------------------PhysicalProject +----------------------------filter((t_c_firstyear.dyear = 1999) and (t_c_firstyear.sale_type = 'c') and (t_c_firstyear.year_total > 0.000000)) +------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +------------------PhysicalProject +--------------------filter((t_w_firstyear.dyear = 1999) and (t_w_firstyear.sale_type = 'w') and (t_w_firstyear.year_total > 0.000000)) +----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query4.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query4.out index a6b03c32eee3df7..61bce070b34aaae 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query4.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query4.out @@ -56,7 +56,10 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------------filter((t_w_firstyear.dyear = 1999) and (t_w_firstyear.sale_type = 'w') and (t_w_firstyear.year_total > 0.000000)) ----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) ------------------PhysicalProject ---------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t_s_firstyear.customer_id = t_c_secyear.customer_id)) otherCondition=((if((year_total > 0.000000), (cast(year_total as DECIMALV3(38, 16)) / year_total), NULL) > if((year_total > 0.000000), (cast(year_total as DECIMALV3(38, 16)) / year_total), NULL))) +--------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((t_s_firstyear.customer_id = t_c_secyear.customer_id)) otherCondition=((if((year_total > 0.000000), (cast(year_total as DECIMALV3(38, 16)) / year_total), NULL) > if((year_total > 0.000000), (cast(year_total as DECIMALV3(38, 16)) / year_total), NULL))) +----------------------PhysicalProject +------------------------filter((t_c_secyear.dyear = 2000) and (t_c_secyear.sale_type = 'c')) +--------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) ----------------------PhysicalProject ------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t_s_firstyear.customer_id = t_c_firstyear.customer_id)) otherCondition=() --------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((t_s_secyear.customer_id = t_s_firstyear.customer_id)) otherCondition=() @@ -69,7 +72,4 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------------------PhysicalProject ----------------------------filter((t_c_firstyear.dyear = 1999) and (t_c_firstyear.sale_type = 'c') and (t_c_firstyear.year_total > 0.000000)) ------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) -----------------------PhysicalProject -------------------------filter((t_c_secyear.dyear = 2000) and (t_c_secyear.sale_type = 'c')) ---------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query4.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query4.out index 8f376d296af9e4e..1946f8020ded90a 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query4.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query4.out @@ -56,7 +56,10 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------------filter((t_w_firstyear.dyear = 1999) and (t_w_firstyear.sale_type = 'w') and (t_w_firstyear.year_total > 0.000000)) ----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) ------------------PhysicalProject ---------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t_s_firstyear.customer_id = t_c_secyear.customer_id)) otherCondition=((if((year_total > 0.000000), (cast(year_total as DECIMALV3(38, 16)) / year_total), NULL) > if((year_total > 0.000000), (cast(year_total as DECIMALV3(38, 16)) / year_total), NULL))) +--------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((t_s_firstyear.customer_id = t_c_secyear.customer_id)) otherCondition=((if((year_total > 0.000000), (cast(year_total as DECIMALV3(38, 16)) / year_total), NULL) > if((year_total > 0.000000), (cast(year_total as DECIMALV3(38, 16)) / year_total), NULL))) +----------------------PhysicalProject +------------------------filter((t_c_secyear.dyear = 2000) and (t_c_secyear.sale_type = 'c')) +--------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) ----------------------PhysicalProject ------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t_s_firstyear.customer_id = t_c_firstyear.customer_id)) otherCondition=() --------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((t_s_secyear.customer_id = t_s_firstyear.customer_id)) otherCondition=() @@ -69,7 +72,4 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------------------PhysicalProject ----------------------------filter((t_c_firstyear.dyear = 1999) and (t_c_firstyear.sale_type = 'c') and (t_c_firstyear.year_total > 0.000000)) ------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) -----------------------PhysicalProject -------------------------filter((t_c_secyear.dyear = 2000) and (t_c_secyear.sale_type = 'c')) ---------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query11.out b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query11.out index 531ca06c5a80bb8..01b1d2752e59b51 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query11.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query11.out @@ -40,10 +40,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------------filter((t_w_secyear.dyear = 2002) and (t_w_secyear.sale_type = 'w')) ------------------PhysicalCteConsumer ( cteId=CTEId#0 ) --------------PhysicalProject -----------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((t_s_firstyear.customer_id = t_w_firstyear.customer_id)) otherCondition=() -------------------PhysicalProject ---------------------filter((t_w_firstyear.dyear = 2001) and (t_w_firstyear.sale_type = 'w') and (t_w_firstyear.year_total > 0.00)) -----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +----------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t_s_firstyear.customer_id = t_w_firstyear.customer_id)) otherCondition=() ------------------hashJoin[INNER_JOIN shuffle] hashCondition=((t_s_secyear.customer_id = t_s_firstyear.customer_id)) otherCondition=() --------------------PhysicalProject ----------------------filter((t_s_secyear.dyear = 2002) and (t_s_secyear.sale_type = 's')) @@ -51,4 +48,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------------PhysicalProject ----------------------filter((t_s_firstyear.dyear = 2001) and (t_s_firstyear.sale_type = 's') and (t_s_firstyear.year_total > 0.00)) ------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +------------------PhysicalProject +--------------------filter((t_w_firstyear.dyear = 2001) and (t_w_firstyear.sale_type = 'w') and (t_w_firstyear.year_total > 0.00)) +----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query4.out b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query4.out index 3d35d56d81d4e1d..63f30b978f4d686 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query4.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query4.out @@ -51,20 +51,14 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------------filter((t_w_secyear.dyear = 2000) and (t_w_secyear.sale_type = 'w')) ------------------PhysicalCteConsumer ( cteId=CTEId#0 ) --------------PhysicalProject -----------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((t_s_firstyear.customer_id = t_w_firstyear.customer_id)) otherCondition=() -------------------PhysicalProject ---------------------filter((t_w_firstyear.dyear = 1999) and (t_w_firstyear.sale_type = 'w') and (t_w_firstyear.year_total > 0.000000)) -----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +----------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t_s_firstyear.customer_id = t_w_firstyear.customer_id)) otherCondition=() ------------------PhysicalProject --------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((t_s_firstyear.customer_id = t_c_secyear.customer_id)) otherCondition=((if((year_total > 0.000000), (cast(year_total as DECIMALV3(38, 16)) / year_total), NULL) > if((year_total > 0.000000), (cast(year_total as DECIMALV3(38, 16)) / year_total), NULL))) ----------------------PhysicalProject ------------------------filter((t_c_secyear.dyear = 2000) and (t_c_secyear.sale_type = 'c')) --------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) ----------------------PhysicalProject -------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((t_s_firstyear.customer_id = t_c_firstyear.customer_id)) otherCondition=() ---------------------------PhysicalProject -----------------------------filter((t_c_firstyear.dyear = 1999) and (t_c_firstyear.sale_type = 'c') and (t_c_firstyear.year_total > 0.000000)) -------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t_s_firstyear.customer_id = t_c_firstyear.customer_id)) otherCondition=() --------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((t_s_secyear.customer_id = t_s_firstyear.customer_id)) otherCondition=() ----------------------------PhysicalProject ------------------------------filter((t_s_secyear.dyear = 2000) and (t_s_secyear.sale_type = 's')) @@ -72,4 +66,10 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------------------------PhysicalProject ------------------------------filter((t_s_firstyear.dyear = 1999) and (t_s_firstyear.sale_type = 's') and (t_s_firstyear.year_total > 0.000000)) --------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +--------------------------PhysicalProject +----------------------------filter((t_c_firstyear.dyear = 1999) and (t_c_firstyear.sale_type = 'c') and (t_c_firstyear.year_total > 0.000000)) +------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +------------------PhysicalProject +--------------------filter((t_w_firstyear.dyear = 1999) and (t_w_firstyear.sale_type = 'w') and (t_w_firstyear.year_total > 0.000000)) +----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query11.out b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query11.out index fa8d314e00019dd..b44acb095192100 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query11.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query11.out @@ -40,10 +40,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------------filter((t_w_secyear.dyear = 2002) and (t_w_secyear.sale_type = 'w')) ------------------PhysicalCteConsumer ( cteId=CTEId#0 ) --------------PhysicalProject -----------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((t_s_firstyear.customer_id = t_w_firstyear.customer_id)) otherCondition=() -------------------PhysicalProject ---------------------filter((t_w_firstyear.dyear = 2001) and (t_w_firstyear.sale_type = 'w') and (t_w_firstyear.year_total > 0.00)) -----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +----------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t_s_firstyear.customer_id = t_w_firstyear.customer_id)) otherCondition=() ------------------hashJoin[INNER_JOIN shuffle] hashCondition=((t_s_secyear.customer_id = t_s_firstyear.customer_id)) otherCondition=() --------------------PhysicalProject ----------------------filter((t_s_secyear.dyear = 2002) and (t_s_secyear.sale_type = 's')) @@ -51,4 +48,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------------PhysicalProject ----------------------filter((t_s_firstyear.dyear = 2001) and (t_s_firstyear.sale_type = 's') and (t_s_firstyear.year_total > 0.00)) ------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +------------------PhysicalProject +--------------------filter((t_w_firstyear.dyear = 2001) and (t_w_firstyear.sale_type = 'w') and (t_w_firstyear.year_total > 0.00)) +----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query4.out b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query4.out index 27803aec4d8f64e..2ab0ebfe7fb09c0 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query4.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query4.out @@ -51,20 +51,14 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------------filter((t_w_secyear.dyear = 2000) and (t_w_secyear.sale_type = 'w')) ------------------PhysicalCteConsumer ( cteId=CTEId#0 ) --------------PhysicalProject -----------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((t_s_firstyear.customer_id = t_w_firstyear.customer_id)) otherCondition=() -------------------PhysicalProject ---------------------filter((t_w_firstyear.dyear = 1999) and (t_w_firstyear.sale_type = 'w') and (t_w_firstyear.year_total > 0.000000)) -----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +----------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t_s_firstyear.customer_id = t_w_firstyear.customer_id)) otherCondition=() ------------------PhysicalProject --------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((t_s_firstyear.customer_id = t_c_secyear.customer_id)) otherCondition=((if((year_total > 0.000000), (cast(year_total as DECIMALV3(38, 16)) / year_total), NULL) > if((year_total > 0.000000), (cast(year_total as DECIMALV3(38, 16)) / year_total), NULL))) ----------------------PhysicalProject ------------------------filter((t_c_secyear.dyear = 2000) and (t_c_secyear.sale_type = 'c')) --------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) ----------------------PhysicalProject -------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((t_s_firstyear.customer_id = t_c_firstyear.customer_id)) otherCondition=() ---------------------------PhysicalProject -----------------------------filter((t_c_firstyear.dyear = 1999) and (t_c_firstyear.sale_type = 'c') and (t_c_firstyear.year_total > 0.000000)) -------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t_s_firstyear.customer_id = t_c_firstyear.customer_id)) otherCondition=() --------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((t_s_secyear.customer_id = t_s_firstyear.customer_id)) otherCondition=() ----------------------------PhysicalProject ------------------------------filter((t_s_secyear.dyear = 2000) and (t_s_secyear.sale_type = 's')) @@ -72,4 +66,10 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------------------------PhysicalProject ------------------------------filter((t_s_firstyear.dyear = 1999) and (t_s_firstyear.sale_type = 's') and (t_s_firstyear.year_total > 0.000000)) --------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +--------------------------PhysicalProject +----------------------------filter((t_c_firstyear.dyear = 1999) and (t_c_firstyear.sale_type = 'c') and (t_c_firstyear.year_total > 0.000000)) +------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +------------------PhysicalProject +--------------------filter((t_w_firstyear.dyear = 1999) and (t_w_firstyear.sale_type = 'w') and (t_w_firstyear.year_total > 0.000000)) +----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query4.out b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query4.out index a6b03c32eee3df7..61bce070b34aaae 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query4.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query4.out @@ -56,7 +56,10 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------------filter((t_w_firstyear.dyear = 1999) and (t_w_firstyear.sale_type = 'w') and (t_w_firstyear.year_total > 0.000000)) ----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) ------------------PhysicalProject ---------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t_s_firstyear.customer_id = t_c_secyear.customer_id)) otherCondition=((if((year_total > 0.000000), (cast(year_total as DECIMALV3(38, 16)) / year_total), NULL) > if((year_total > 0.000000), (cast(year_total as DECIMALV3(38, 16)) / year_total), NULL))) +--------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((t_s_firstyear.customer_id = t_c_secyear.customer_id)) otherCondition=((if((year_total > 0.000000), (cast(year_total as DECIMALV3(38, 16)) / year_total), NULL) > if((year_total > 0.000000), (cast(year_total as DECIMALV3(38, 16)) / year_total), NULL))) +----------------------PhysicalProject +------------------------filter((t_c_secyear.dyear = 2000) and (t_c_secyear.sale_type = 'c')) +--------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) ----------------------PhysicalProject ------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t_s_firstyear.customer_id = t_c_firstyear.customer_id)) otherCondition=() --------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((t_s_secyear.customer_id = t_s_firstyear.customer_id)) otherCondition=() @@ -69,7 +72,4 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------------------PhysicalProject ----------------------------filter((t_c_firstyear.dyear = 1999) and (t_c_firstyear.sale_type = 'c') and (t_c_firstyear.year_total > 0.000000)) ------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) -----------------------PhysicalProject -------------------------filter((t_c_secyear.dyear = 2000) and (t_c_secyear.sale_type = 'c')) ---------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query4.out b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query4.out index 8f376d296af9e4e..1946f8020ded90a 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query4.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query4.out @@ -56,7 +56,10 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------------filter((t_w_firstyear.dyear = 1999) and (t_w_firstyear.sale_type = 'w') and (t_w_firstyear.year_total > 0.000000)) ----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) ------------------PhysicalProject ---------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t_s_firstyear.customer_id = t_c_secyear.customer_id)) otherCondition=((if((year_total > 0.000000), (cast(year_total as DECIMALV3(38, 16)) / year_total), NULL) > if((year_total > 0.000000), (cast(year_total as DECIMALV3(38, 16)) / year_total), NULL))) +--------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((t_s_firstyear.customer_id = t_c_secyear.customer_id)) otherCondition=((if((year_total > 0.000000), (cast(year_total as DECIMALV3(38, 16)) / year_total), NULL) > if((year_total > 0.000000), (cast(year_total as DECIMALV3(38, 16)) / year_total), NULL))) +----------------------PhysicalProject +------------------------filter((t_c_secyear.dyear = 2000) and (t_c_secyear.sale_type = 'c')) +--------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) ----------------------PhysicalProject ------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t_s_firstyear.customer_id = t_c_firstyear.customer_id)) otherCondition=() --------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((t_s_secyear.customer_id = t_s_firstyear.customer_id)) otherCondition=() @@ -69,7 +72,4 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------------------PhysicalProject ----------------------------filter((t_c_firstyear.dyear = 1999) and (t_c_firstyear.sale_type = 'c') and (t_c_firstyear.year_total > 0.000000)) ------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) -----------------------PhysicalProject -------------------------filter((t_c_secyear.dyear = 2000) and (t_c_secyear.sale_type = 'c')) ---------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query4.out b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query4.out index 8f376d296af9e4e..1946f8020ded90a 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query4.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query4.out @@ -56,7 +56,10 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------------filter((t_w_firstyear.dyear = 1999) and (t_w_firstyear.sale_type = 'w') and (t_w_firstyear.year_total > 0.000000)) ----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) ------------------PhysicalProject ---------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t_s_firstyear.customer_id = t_c_secyear.customer_id)) otherCondition=((if((year_total > 0.000000), (cast(year_total as DECIMALV3(38, 16)) / year_total), NULL) > if((year_total > 0.000000), (cast(year_total as DECIMALV3(38, 16)) / year_total), NULL))) +--------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((t_s_firstyear.customer_id = t_c_secyear.customer_id)) otherCondition=((if((year_total > 0.000000), (cast(year_total as DECIMALV3(38, 16)) / year_total), NULL) > if((year_total > 0.000000), (cast(year_total as DECIMALV3(38, 16)) / year_total), NULL))) +----------------------PhysicalProject +------------------------filter((t_c_secyear.dyear = 2000) and (t_c_secyear.sale_type = 'c')) +--------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) ----------------------PhysicalProject ------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t_s_firstyear.customer_id = t_c_firstyear.customer_id)) otherCondition=() --------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((t_s_secyear.customer_id = t_s_firstyear.customer_id)) otherCondition=() @@ -69,7 +72,4 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------------------PhysicalProject ----------------------------filter((t_c_firstyear.dyear = 1999) and (t_c_firstyear.sale_type = 'c') and (t_c_firstyear.year_total > 0.000000)) ------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) -----------------------PhysicalProject -------------------------filter((t_c_secyear.dyear = 2000) and (t_c_secyear.sale_type = 'c')) ---------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/data/partition_p0/auto_partition/test_auto_dynamic.out b/regression-test/data/partition_p0/auto_partition/test_auto_dynamic.out new file mode 100644 index 000000000000000..0b588c49759ad49 --- /dev/null +++ b/regression-test/data/partition_p0/auto_partition/test_auto_dynamic.out @@ -0,0 +1,6 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql_dynamic_auto -- +2024-01-01T00:00 +2900-01-01T00:00 +3000-01-01T00:00 + diff --git a/regression-test/data/partition_p0/auto_partition/test_auto_partition_behavior.out b/regression-test/data/partition_p0/auto_partition/test_auto_partition_behavior.out index 27d77c500f2864a..133a86cddab91dc 100644 --- a/regression-test/data/partition_p0/auto_partition/test_auto_partition_behavior.out +++ b/regression-test/data/partition_p0/auto_partition/test_auto_partition_behavior.out @@ -114,8 +114,3 @@ Xxx -- !sql_non_order3 -- 3 2013-12-12T00:00 --- !sql_dynamic_auto -- -2024-01-01T00:00 -2900-01-01T00:00 -3000-01-01T00:00 - diff --git a/regression-test/data/point_query_p0/test_point_query.out b/regression-test/data/point_query_p0/test_point_query.out index 5a4e0b6617826f0..1cc4142e39f3061 100644 --- a/regression-test/data/point_query_p0/test_point_query.out +++ b/regression-test/data/point_query_p0/test_point_query.out @@ -152,3 +152,11 @@ -- !sql -- 0 1111111 +-- !sql -- +10 20 aabc value + +-- !sql -- + +-- !sql -- +-10 20 aabc update val + diff --git a/regression-test/data/query_p0/sql_functions/encryption_digest/test_encryption_function.out b/regression-test/data/query_p0/sql_functions/encryption_digest/test_encryption_function.out index 9e920596f894cf8..721412dc0364c14 100644 --- a/regression-test/data/query_p0/sql_functions/encryption_digest/test_encryption_function.out +++ b/regression-test/data/query_p0/sql_functions/encryption_digest/test_encryption_function.out @@ -11,33 +11,6 @@ text -- !sql -- \N --- !sql -- -wr2JEDVXzL9+2XtRhgIloA== - --- !sql -- -wr2JEDVXzL9+2XtRhgIloA== - --- !sql -- -text - --- !sql -- -\N - --- !sql -- -wr2JEDVXzL9+2XtRhgIloA== - --- !sql -- -BO2vxHeUcw5BQQalSBbo1w== - --- !sql -- -text - --- !sql -- -\N - --- !sql -- -text - -- !sql -- BO2vxHeUcw5BQQalSBbo1w== @@ -54,25 +27,7 @@ text text -- !sql -- -wr2JEDVXzL9+2XtRhgIloA== - --- !sql -- -tsmK1HzbpnEdR2//WhO+MA== - --- !sql -- -ciacXDLHMNG7CD9Kws8png== - --- !sql -- -text - --- !sql -- -\N - --- !sql -- -text - --- !sql -- -text +3dym0E7/+1zbrLIaBVNHSw== -- !sql -- tsmK1HzbpnEdR2//WhO+MA== @@ -93,7 +48,7 @@ text text -- !sql -- -aDjwRflBrDjhBZIOFNw3Tg== +FSYstvOmH2cXy7B/072Mug== -- !sql -- 1Y4NGIukSbv9OrkZnRD1bQ== @@ -101,27 +56,12 @@ aDjwRflBrDjhBZIOFNw3Tg== -- !sql -- G5POcFAJwiZHeTtN6DjInQ== --- !sql -- -text - --- !sql -- -text - -- !sql -- \N -- !sql -- text --- !sql -- -1Y4NGIukSbv9OrkZnRD1bQ== - --- !sql -- -G5POcFAJwiZHeTtN6DjInQ== - --- !sql -- -text - -- !sql -- \N diff --git a/regression-test/data/query_p0/sql_functions/math_functions/test_math_unary_always_nullable.out b/regression-test/data/query_p0/sql_functions/math_functions/test_math_unary_always_nullable.out new file mode 100644 index 000000000000000..0a190f0bd6b2f93 --- /dev/null +++ b/regression-test/data/query_p0/sql_functions/math_functions/test_math_unary_always_nullable.out @@ -0,0 +1,95 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !acos_1 -- +\N true + +-- !acos_2 -- +\N true + +-- !acos_3 -- +\N true 0 +\N true 1 +\N true 2 +\N true 3 +\N true 4 +\N true 5 +\N true 6 +\N true 7 +\N true 8 +\N true 9 + +-- !asin_1 -- +\N true + +-- !asin_2 -- +\N true + +-- !asin_3 -- +\N true 0 +\N true 1 +\N true 2 +\N true 3 +\N true 4 +\N true 5 +\N true 6 +\N true 7 +\N true 8 +\N true 9 + +-- !sqrt_1 -- +\N true + +-- !sqrt_2 -- +\N true + +-- !sqrt_3 -- +\N true 0 +\N true 1 +\N true 2 +\N true 3 +\N true 4 +\N true 5 +\N true 6 +\N true 7 +\N true 8 +\N true 9 + +-- !acos_tbl_1 -- +1 \N true +2 \N true +3 1.5707963267948966 false +4 \N true +5 \N true +6 \N true +7 \N true +8 \N true + +-- !asin_tbl_1 -- +1 \N true +2 \N true +3 0.0 false +4 \N true +5 \N true +6 \N true +7 \N true +8 \N true + +-- !sqrt_tbl_1 -- +1 1.0488088481701516 false +2 \N true +3 0.0 false +4 \N true +5 \N true +6 \N true +7 \N true +8 \N true + +-- !dsqrt_tbl_1 -- +1 1.0488088481701516 false +2 \N true +3 0.0 false +4 \N true +5 \N true +6 \N true +7 \N true +8 \N true + diff --git a/regression-test/data/unique_with_mow_p0/partial_update/partial_update_delete.csv b/regression-test/data/unique_with_mow_p0/partial_update/partial_update_delete.csv index 5f5fbe759f10b98..e22b5b747a60e7d 100644 --- a/regression-test/data/unique_with_mow_p0/partial_update/partial_update_delete.csv +++ b/regression-test/data/unique_with_mow_p0/partial_update/partial_update_delete.csv @@ -1,3 +1,4 @@ 1 2 -3 \ No newline at end of file +3 +7 \ No newline at end of file diff --git a/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_delete.out b/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_delete.out index e1623d424601815..7a639b8d6e29dbc 100644 --- a/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_delete.out +++ b/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_delete.out @@ -1,21 +1,21 @@ -- This file is automatically generated. You should know what you did if you want to edit this --- !sql -- +-- !sql1 -- 1 1 1 1 1 2 2 2 2 2 3 3 3 3 3 4 4 4 4 4 5 5 5 5 5 --- !sql -- +-- !sql1 -- 2 2 2 2 2 4 4 4 4 4 5 5 5 5 5 --- !sql -- +-- !sql1 -- 4 4 4 4 4 5 5 5 5 5 --- !with_delete_sign -- +-- !with_delete_sign1 -- 1 \N \N 0 \N 1 1 1 1 1 1 0 2 \N \N 0 \N 1 @@ -25,44 +25,291 @@ 4 4 4 4 4 0 5 5 5 5 5 0 --- !sql -- +-- !sql2 -- 1 1 1 1 1 2 2 2 2 2 3 3 3 3 3 4 4 4 4 4 5 5 5 5 5 +6 6 6 6 6 --- !sql -- +-- !sql2 -- 4 4 4 4 4 5 5 5 5 5 +6 6 6 6 6 --- !sql -- -1 \N \N \N \N 1 +-- !sql2 -- +5 5 5 5 5 +6 6 6 6 6 + +-- !sql2 -- +1 \N \N 0 \N 1 +1 1 1 1 1 0 +2 \N \N 0 \N 1 +2 2 2 2 2 0 +3 \N \N 0 \N 1 +3 3 3 3 3 0 +4 \N \N 0 \N 1 +4 4 4 4 4 0 +5 5 5 5 5 0 +6 6 6 6 6 0 +7 \N \N 0 \N 1 +8 \N \N 0 \N 1 +9 \N \N 0 \N 1 + +-- !sql3 -- +1 1 1 1 1 +2 2 2 2 2 +3 3 3 3 3 +4 4 4 4 4 +5 5 5 5 5 +6 6 6 6 6 + +-- !sql3 -- +4 4 4 4 4 +5 5 5 5 5 +6 6 6 6 6 + +-- !sql3 -- +5 5 5 5 5 +6 6 6 6 6 + +-- !sql3 -- +1 1 1 1 1 0 1 +1 1 1 1 1 1 1 +2 2 2 2 2 0 2 +2 2 2 2 2 1 2 +3 3 3 3 3 0 3 +3 3 3 3 3 1 3 +4 4 4 4 4 0 4 +4 4 4 4 4 1 4 +5 5 5 5 5 0 5 +6 6 6 6 6 0 6 +7 \N \N 0 \N 1 \N +8 \N \N 0 \N 1 \N +9 \N \N 0 \N 1 \N + +-- !sql4 -- +1 1 1 1 1 +2 2 2 2 2 +3 3 3 3 3 +4 4 4 4 4 +5 5 5 5 5 +6 6 6 6 6 + +-- !sql1 -- +1 1 1 1 1 +2 2 2 2 2 +3 3 3 3 3 +4 4 4 4 4 +5 5 5 5 5 + +-- !sql1 -- +2 2 2 2 2 +4 4 4 4 4 +5 5 5 5 5 + +-- !sql1 -- +4 4 4 4 4 +5 5 5 5 5 + +-- !with_delete_sign1 -- +1 \N \N 0 \N 1 +1 1 1 1 1 0 +2 \N \N 0 \N 1 +2 2 2 2 2 0 +3 \N \N 0 \N 1 +3 3 3 3 3 0 +4 4 4 4 4 0 +5 5 5 5 5 0 + +-- !sql2 -- +1 1 1 1 1 +2 2 2 2 2 +3 3 3 3 3 +4 4 4 4 4 +5 5 5 5 5 +6 6 6 6 6 + +-- !sql2 -- +4 4 4 4 4 +5 5 5 5 5 +6 6 6 6 6 + +-- !sql2 -- +5 5 5 5 5 +6 6 6 6 6 + +-- !sql2 -- +1 \N \N 0 \N 1 +1 1 1 1 1 0 +2 \N \N 0 \N 1 +2 2 2 2 2 0 +3 \N \N 0 \N 1 +3 3 3 3 3 0 +4 \N \N 0 \N 1 +4 4 4 4 4 0 +5 5 5 5 5 0 +6 6 6 6 6 0 +7 \N \N 0 \N 1 +8 \N \N 0 \N 1 +9 \N \N 0 \N 1 + +-- !sql3 -- +1 1 1 1 1 +2 2 2 2 2 +3 3 3 3 3 +4 4 4 4 4 +5 5 5 5 5 +6 6 6 6 6 + +-- !sql3 -- +4 4 4 4 4 +5 5 5 5 5 +6 6 6 6 6 + +-- !sql3 -- +5 5 5 5 5 +6 6 6 6 6 + +-- !sql3 -- +1 1 1 1 1 0 1 +1 1 1 1 1 1 1 +2 2 2 2 2 0 2 +2 2 2 2 2 1 2 +3 3 3 3 3 0 3 +3 3 3 3 3 1 3 +4 4 4 4 4 0 4 +4 4 4 4 4 1 4 +5 5 5 5 5 0 5 +6 6 6 6 6 0 6 +7 \N \N 0 \N 1 \N +8 \N \N 0 \N 1 \N +9 \N \N 0 \N 1 \N + +-- !sql4 -- +1 1 1 1 1 +2 2 2 2 2 +3 3 3 3 3 +4 4 4 4 4 +5 5 5 5 5 +6 6 6 6 6 + +-- !sql1 -- +1 1 1 1 1 +2 2 2 2 2 +3 3 3 3 3 +4 4 4 4 4 +5 5 5 5 5 + +-- !sql1 -- +2 2 2 2 2 +4 4 4 4 4 +5 5 5 5 5 + +-- !sql1 -- +4 4 4 4 4 +5 5 5 5 5 + +-- !with_delete_sign1 -- +1 \N \N 0 \N 1 1 1 1 1 1 0 -2 \N \N \N \N 1 +2 \N \N 0 \N 1 2 2 2 2 2 0 -3 \N \N \N \N 1 +3 \N \N 0 \N 1 3 3 3 3 3 0 4 4 4 4 4 0 5 5 5 5 5 0 --- !sql -- +-- !sql2 -- 1 1 1 1 1 2 2 2 2 2 3 3 3 3 3 4 4 4 4 4 5 5 5 5 5 +6 6 6 6 6 + +-- !sql2 -- +4 4 4 4 4 +5 5 5 5 5 +6 6 6 6 6 + +-- !sql2 -- +5 5 5 5 5 +6 6 6 6 6 --- !sql -- +-- !sql2 -- +1 \N \N 0 \N 1 +1 1 1 1 1 0 +2 \N \N 0 \N 1 +2 2 2 2 2 0 +3 \N \N 0 \N 1 +3 3 3 3 3 0 +4 \N \N 0 \N 1 +4 4 4 4 4 0 +5 5 5 5 5 0 +6 6 6 6 6 0 +7 \N \N 0 \N 1 +8 \N \N 0 \N 1 +9 \N \N 0 \N 1 + +-- !sql3 -- +1 1 1 1 1 2 2 2 2 2 +3 3 3 3 3 4 4 4 4 4 5 5 5 5 5 +6 6 6 6 6 --- !sql -- +-- !sql3 -- 4 4 4 4 4 5 5 5 5 5 +6 6 6 6 6 + +-- !sql3 -- +5 5 5 5 5 +6 6 6 6 6 --- !with_delete_sign -- +-- !sql3 -- +1 1 1 1 1 0 1 +1 1 1 1 1 1 1 +2 2 2 2 2 0 2 +2 2 2 2 2 1 2 +3 3 3 3 3 0 3 +3 3 3 3 3 1 3 +4 4 4 4 4 0 4 +4 4 4 4 4 1 4 +5 5 5 5 5 0 5 +6 6 6 6 6 0 6 +7 \N \N 0 \N 1 \N +8 \N \N 0 \N 1 \N +9 \N \N 0 \N 1 \N + +-- !sql4 -- +1 1 1 1 1 +2 2 2 2 2 +3 3 3 3 3 +4 4 4 4 4 +5 5 5 5 5 +6 6 6 6 6 + +-- !sql1 -- +1 1 1 1 1 +2 2 2 2 2 +3 3 3 3 3 +4 4 4 4 4 +5 5 5 5 5 + +-- !sql1 -- +2 2 2 2 2 +4 4 4 4 4 +5 5 5 5 5 + +-- !sql1 -- +4 4 4 4 4 +5 5 5 5 5 + +-- !with_delete_sign1 -- 1 \N \N 0 \N 1 1 1 1 1 1 0 2 \N \N 0 \N 1 @@ -72,24 +319,75 @@ 4 4 4 4 4 0 5 5 5 5 5 0 --- !sql -- +-- !sql2 -- 1 1 1 1 1 2 2 2 2 2 3 3 3 3 3 4 4 4 4 4 5 5 5 5 5 +6 6 6 6 6 --- !sql -- +-- !sql2 -- 4 4 4 4 4 5 5 5 5 5 +6 6 6 6 6 --- !sql -- -1 \N \N \N \N 1 +-- !sql2 -- +5 5 5 5 5 +6 6 6 6 6 + +-- !sql2 -- +1 \N \N 0 \N 1 1 1 1 1 1 0 -2 \N \N \N \N 1 +2 \N \N 0 \N 1 2 2 2 2 2 0 -3 \N \N \N \N 1 +3 \N \N 0 \N 1 3 3 3 3 3 0 +4 \N \N 0 \N 1 4 4 4 4 4 0 5 5 5 5 5 0 +6 6 6 6 6 0 +7 \N \N 0 \N 1 +8 \N \N 0 \N 1 +9 \N \N 0 \N 1 + +-- !sql3 -- +1 1 1 1 1 +2 2 2 2 2 +3 3 3 3 3 +4 4 4 4 4 +5 5 5 5 5 +6 6 6 6 6 + +-- !sql3 -- +4 4 4 4 4 +5 5 5 5 5 +6 6 6 6 6 + +-- !sql3 -- +5 5 5 5 5 +6 6 6 6 6 + +-- !sql3 -- +1 1 1 1 1 0 1 +1 1 1 1 1 1 1 +2 2 2 2 2 0 2 +2 2 2 2 2 1 2 +3 3 3 3 3 0 3 +3 3 3 3 3 1 3 +4 4 4 4 4 0 4 +4 4 4 4 4 1 4 +5 5 5 5 5 0 5 +6 6 6 6 6 0 6 +7 \N \N 0 \N 1 \N +8 \N \N 0 \N 1 \N +9 \N \N 0 \N 1 \N + +-- !sql4 -- +1 1 1 1 1 +2 2 2 2 2 +3 3 3 3 3 +4 4 4 4 4 +5 5 5 5 5 +6 6 6 6 6 diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy index f5e6e5ab1bab710..1587363dc0cb9df 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy @@ -1394,11 +1394,11 @@ class Suite implements GroovyInterceptable { } boolean enableStoragevault() { + boolean ret = false; if (context.config.metaServiceHttpAddress == null || context.config.metaServiceHttpAddress.isEmpty() || - context.config.metaServiceHttpAddress == null || context.config.metaServiceHttpAddress.isEmpty() || - context.config.instanceId == null || context.config.instanceId.isEmpty() || - context.config.metaServiceToken == null || context.config.metaServiceToken.isEmpty()) { - return false; + context.config.instanceId == null || context.config.instanceId.isEmpty() || + context.config.metaServiceToken == null || context.config.metaServiceToken.isEmpty()) { + return ret; } def getInstanceInfo = { check_func -> httpTest { @@ -1408,7 +1408,6 @@ class Suite implements GroovyInterceptable { check check_func } } - boolean enableStorageVault = false; getInstanceInfo.call() { respCode, body -> String respCodeValue = "${respCode}".toString(); @@ -1417,10 +1416,10 @@ class Suite implements GroovyInterceptable { } def json = parseJson(body) if (json.result.containsKey("enable_storage_vault") && json.result.enable_storage_vault) { - enableStorageVault = true; + ret = true; } } - return enableStorageVault; + return ret; } boolean isGroupCommitMode() { diff --git a/regression-test/suites/compaction/test_single_compaction_with_variant_inverted_index.groovy b/regression-test/suites/compaction/test_single_compaction_with_variant_inverted_index.groovy index 69768659d59e500..79c6322204585dd 100644 --- a/regression-test/suites/compaction/test_single_compaction_with_variant_inverted_index.groovy +++ b/regression-test/suites/compaction/test_single_compaction_with_variant_inverted_index.groovy @@ -210,12 +210,13 @@ suite("test_single_compaction_with_variant_inverted", "p2") { } } - sql """ INSERT INTO ${tableName} VALUES (1, "a", 100, '{"a" : 1234, "point" : 1, "xxxx" : "ddddd"}'); """ - sql """ INSERT INTO ${tableName} VALUES (1, "b", 100, '{"%a" : 1234, "@point" : 1, "[xxxx" : "ddddd"}'); """ - sql """ INSERT INTO ${tableName} VALUES (2, "a", 100, '{"@a" : 1234, "%point" : 1, "]xxxx" : "ddddd"}'); """ - sql """ INSERT INTO ${tableName} VALUES (2, "b", 100, '{"%a" : 1234, "%point" : 1, "{xxxx" : "ddddd"}'); """ - sql """ INSERT INTO ${tableName} VALUES (3, "a", 100, '{"@a" : 1234, "@point" : 1, "}xxxx" : "ddddd"}'); """ - sql """ INSERT INTO ${tableName} VALUES (3, "b", 100, '{"a" : 1234, "point" : 1, "|xxxx" : "ddddd"}'); """ + sql """ INSERT INTO ${tableName} VALUES (1, "b", 100, '{"%a" : 1234, "@point" : 1, "xxxx" : "ddddd"}'); """ + sql """ INSERT INTO ${tableName} VALUES (2, "a", 100, '{"@a" : 1234, "%point" : 1, "xxxx" : "ddddd"}'); """ + sql """ INSERT INTO ${tableName} VALUES (2, "b", 100, '{"%a" : 1234, "%point" : 1, "xxxx" : "ddddd"}'); """ + sql """ INSERT INTO ${tableName} VALUES (3, "a", 100, '{"@a" : 1234, "@point" : 1, "xxxx" : "ddddd"}'); """ + sql """ INSERT INTO ${tableName} VALUES (3, "b", 100, '{"a" : 1234, "point" : 1, "xxxx" : "ddddd"}'); """ + sql """ INSERT INTO ${tableName} VALUES (3, "b", 100, '{"a" : 1234, "point" : 1, "xxxx" : "ddddd"}'); """ + // trigger master be to do full compaction assertTrue(triggerCompaction(backendId_to_backendIP[master_backend_id], backendId_to_backendHttpPort[master_backend_id], @@ -233,7 +234,7 @@ suite("test_single_compaction_with_variant_inverted", "p2") { checkTabletFileCrc.call() qt_sql """ - select count() from ${tableName} where properties MATCH_ANY 'point xxxx'; + select count() from ${tableName} where cast(properties['xxxx'] as string) MATCH_ANY 'ddddd'; """ sql """ DROP TABLE IF EXISTS ${tableName}; """ diff --git a/regression-test/suites/datatype_p0/bitmap/test_bitmap_int.groovy b/regression-test/suites/datatype_p0/bitmap/test_bitmap_int.groovy index c890b27fada609e..38725987d96fc41 100644 --- a/regression-test/suites/datatype_p0/bitmap/test_bitmap_int.groovy +++ b/regression-test/suites/datatype_p0/bitmap/test_bitmap_int.groovy @@ -68,7 +68,7 @@ suite("test_bitmap_int") { test { sql """SELECT case id_bitmap when 1 then 1 else 0 FROM test_bitmap;""" - exception "Syntax error in line 1" + exception "missing 'END' at 'FROM'" } qt_sql64_4 """SELECT id_bitmap FROM test_bitmap WHERE id_bitmap is null LIMIT 20;""" diff --git a/regression-test/suites/datatype_p0/decimalv3/test_decimal256_cast.groovy b/regression-test/suites/datatype_p0/decimalv3/test_decimal256_cast.groovy index ea3001232f1c08c..0132e74010dbcc6 100644 --- a/regression-test/suites/datatype_p0/decimalv3/test_decimal256_cast.groovy +++ b/regression-test/suites/datatype_p0/decimalv3/test_decimal256_cast.groovy @@ -18,6 +18,9 @@ suite("test_decimal256_cast") { sql "set enable_nereids_planner = true;" sql "set enable_decimal256 = true;" + sql """ + set debug_skip_fold_constant=true; + """ qt_decimal256_cast0 """SELECT /*+ SET_VAR(enable_fold_constant_by_be = false) */ cast(999999999999999999999999999999999999999999999999999999999999999999.9999999999 as decimalv3(76,10));""" @@ -41,4 +44,72 @@ suite("test_decimal256_cast") { select cast('0.000000000000000000000000000000000000000000000000000000000000000000000012345678901' as decimalv3(76,0)); """ + sql """ + drop table if exists cast_to_dec256; + """ + sql """ + create table cast_to_dec256 ( + k1 int, + v1 varchar(128) + ) distributed by hash(k1) + properties ( + 'replication_num' = '1' + ); + """ + sql """ + insert into cast_to_dec256 values(9, "999999999999999999999999999999999999999999999999999999999999999999.9999999999"), + (-9, "-999999999999999999999999999999999999999999999999999999999999999999.9999999999"); + """ + qt_decimal256_cast9 """ + select k1, cast(v1 as decimalv3(76,10)) from cast_to_dec256 order by k1, v1; + """ + + sql """ + truncate table cast_to_dec256; + """ + sql """ + insert into cast_to_dec256 values(10, "0.000000000000000000000000000000000000000000000000000000000000000000000012345678901"); + """ + qt_decimal256_cast10 """ + select k1, cast(v1 as decimalv3(76, 0)) from cast_to_dec256 order by k1, v1; + """ + + test { + sql """ + select /*+SET_VAR(enable_fold_constant_by_be = true) */cast(cast("12345678.000000000000000000000000000000001" as decimalv3(76, 60)) as float); + """ + exception "Arithmetic overflow" + } + test { + sql """ + select /*+SET_VAR(enable_fold_constant_by_be = false) */cast(cast("12345678.000000000000000000000000000000001" as decimalv3(76, 60)) as float); + """ + exception "Arithmetic overflow" + } + + sql """ + drop table if exists dec256cast_to_float; + """ + sql """ + create table dec256cast_to_float ( + k1 int, + v1 decimalv3(76, 60) + ) distributed by hash(k1) + properties ( + 'replication_num' = '1' + ); + """ + sql """ + insert into dec256cast_to_float values (1, "12345678.000000000000000000000000000000001"); + """ + test { + sql """ + select cast(v1 as float) from dec256cast_to_float; + """ + exception "Arithmetic overflow" + } + qt_decimal256_cast_to_double_1 """ + select cast(v1 as double) from dec256cast_to_float; + """ + } \ No newline at end of file diff --git a/regression-test/suites/datatype_p0/nested_types/query/test_nestedtypes_insert_into_select.groovy b/regression-test/suites/datatype_p0/nested_types/query/test_nestedtypes_insert_into_select.groovy index 2023ecf641c0f09..633ad98d86f5560 100644 --- a/regression-test/suites/datatype_p0/nested_types/query/test_nestedtypes_insert_into_select.groovy +++ b/regression-test/suites/datatype_p0/nested_types/query/test_nestedtypes_insert_into_select.groovy @@ -32,7 +32,7 @@ suite("test_nestedtypes_insert_into_select", "p0") { test { sql "insert into ast values ('text' , [named_struct('a',1,'b','home'),named_struct('a',2,'b','work')]);" - exception "errCode = 2, detailMessage = Sql parser can't convert the result to array, please check your sql." + exception "mismatched input 'named_struct' expecting" } @@ -50,6 +50,6 @@ suite("test_nestedtypes_insert_into_select", "p0") { test { sql "insert into ast values ('text' , [named_struct('a',1,'b','home'),named_struct('a',2,'b','work')]);" - exception "Sql parser can't convert the result to array" + exception "mismatched input 'named_struct' expecting" } } diff --git a/regression-test/suites/demo_p0/test_action.groovy b/regression-test/suites/demo_p0/test_action.groovy index 1cae0da47e3a0b1..7f0d4d7eb8ba24e 100644 --- a/regression-test/suites/demo_p0/test_action.groovy +++ b/regression-test/suites/demo_p0/test_action.groovy @@ -19,7 +19,7 @@ suite("test_action") { test { sql "abcdefg" // check exception message contains - exception "Syntax error in line 1" + exception "extraneous input 'abcdefg'" } test { diff --git a/regression-test/suites/external_table_p0/hive/ddl/test_hive_ddl.groovy b/regression-test/suites/external_table_p0/hive/ddl/test_hive_ddl.groovy index aa4d8c0b3c6801d..1d39ab9cde55033 100644 --- a/regression-test/suites/external_table_p0/hive/ddl/test_hive_ddl.groovy +++ b/regression-test/suites/external_table_p0/hive/ddl/test_hive_ddl.groovy @@ -362,7 +362,7 @@ suite("test_hive_ddl", "p0,external,hive,external_docker,external_docker_hive") sql """ CREATE TABLE schema_check ENGINE=hive ; """ - exception "AnalysisException, msg: Should contain at least one column in a table" + exception "Should contain at least one column in a table" } sql """ DROP DATABASE IF EXISTS `test_hive_loc` """ } diff --git a/regression-test/suites/external_table_p0/hive/test_external_catalog_hive.groovy b/regression-test/suites/external_table_p0/hive/test_external_catalog_hive.groovy index b920168198acb74..b71c72957e9c02f 100644 --- a/regression-test/suites/external_table_p0/hive/test_external_catalog_hive.groovy +++ b/regression-test/suites/external_table_p0/hive/test_external_catalog_hive.groovy @@ -178,5 +178,9 @@ suite("test_external_catalog_hive", "p0,external,hive,external_docker,external_d """ exception "Failed to init access controller: bound must be positive" } + + // test catalog_meta_cache_statistics + sql """select * from internal.information_schema.catalog_meta_cache_statistics;""" + sql """select * from ${catalog_name}.information_schema.catalog_meta_cache_statistics where catalog_name="${catalog_name}";""" } } diff --git a/regression-test/suites/external_table_p0/hive/test_hive_statistics_p0.groovy b/regression-test/suites/external_table_p0/hive/test_hive_statistics_p0.groovy index 71a43d2aec94215..c11ff176d959213 100644 --- a/regression-test/suites/external_table_p0/hive/test_hive_statistics_p0.groovy +++ b/regression-test/suites/external_table_p0/hive/test_hive_statistics_p0.groovy @@ -232,6 +232,71 @@ suite("test_hive_statistics_p0", "all_types,p0,external,hive,external_docker,ext assertEquals("N/A", result[0][7]) assertEquals("N/A", result[0][8]) + + // Test auto analyze policy + sql """drop stats stats_test1""" + result = sql """show table stats stats_test1""" + assertEquals(1, result.size()) + assertEquals("false", result[0][8]) + + sql """ALTER CATALOG `${catalog_name}` SET PROPERTIES ('enable.auto.analyze'='true')""" + result = sql """show table stats stats_test1""" + assertEquals(1, result.size()) + assertEquals("true", result[0][8]) + + sql """ALTER CATALOG `${catalog_name}` SET PROPERTIES ('enable.auto.analyze'='false')""" + result = sql """show table stats stats_test1""" + assertEquals(1, result.size()) + assertEquals("false", result[0][8]) + + sql """analyze table stats_test1 PROPERTIES("use.auto.analyzer"="true")""" + result = sql """show auto analyze stats_test1""" + assertEquals(0, result.size()) + + sql """ALTER TABLE stats_test1 SET ("auto_analyze_policy" = "enable");""" + result = sql """show table stats stats_test1""" + assertEquals(1, result.size()) + assertEquals("true", result[0][8]) + + sql """analyze table stats_test1 PROPERTIES("use.auto.analyzer"="true")""" + result = sql """show auto analyze stats_test1""" + assertEquals(1, result.size()) + + sql """ALTER CATALOG `${catalog_name}` SET PROPERTIES ('enable.auto.analyze'='true')""" + result = sql """show table stats stats_test1""" + assertEquals(1, result.size()) + assertEquals("true", result[0][8]) + + sql """ALTER CATALOG `${catalog_name}` SET PROPERTIES ('enable.auto.analyze'='false')""" + result = sql """show table stats stats_test1""" + assertEquals(1, result.size()) + assertEquals("true", result[0][8]) + + sql """ALTER TABLE stats_test1 SET ("auto_analyze_policy" = "disable");""" + result = sql """show table stats stats_test1""" + assertEquals(1, result.size()) + assertEquals("false", result[0][8]) + + sql """ALTER CATALOG `${catalog_name}` SET PROPERTIES ('enable.auto.analyze'='true')""" + result = sql """show table stats stats_test1""" + assertEquals(1, result.size()) + assertEquals("false", result[0][8]) + + sql """ALTER CATALOG `${catalog_name}` SET PROPERTIES ('enable.auto.analyze'='false')""" + result = sql """show table stats stats_test1""" + assertEquals(1, result.size()) + assertEquals("false", result[0][8]) + + sql """ALTER TABLE stats_test1 SET ("auto_analyze_policy" = "base_on_catalog");""" + result = sql """show table stats stats_test1""" + assertEquals(1, result.size()) + assertEquals("false", result[0][8]) + + sql """ALTER CATALOG `${catalog_name}` SET PROPERTIES ('enable.auto.analyze'='true')""" + result = sql """show table stats stats_test1""" + assertEquals(1, result.size()) + assertEquals("true", result[0][8]) + sql """drop catalog if exists ${catalog_name}""" } finally { diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_table_stats.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_table_stats.groovy index 064139d22c9deaf..a950f93909e8a82 100644 --- a/regression-test/suites/external_table_p0/iceberg/test_iceberg_table_stats.groovy +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_table_stats.groovy @@ -58,6 +58,10 @@ suite("test_iceberg_table_stats", "p0,external,doris,external_docker,external_do assert_stats("sample_mor_orc", "1000") assert_stats("sample_mor_parquet", "1000") + // test catalog_meta_cache_statistics + sql """select * from information_schema.catalog_meta_cache_statistics;""" + sql """select * from information_schema.catalog_meta_cache_statistics where catalog_name="${catalog_name}";""" + } finally { } } diff --git a/regression-test/suites/external_table_p0/jdbc/test_mariadb_jdbc_catalog.groovy b/regression-test/suites/external_table_p0/jdbc/test_mariadb_jdbc_catalog.groovy index d90ffe3e1cc4065..5281cdd20cc5391 100644 --- a/regression-test/suites/external_table_p0/jdbc/test_mariadb_jdbc_catalog.groovy +++ b/regression-test/suites/external_table_p0/jdbc/test_mariadb_jdbc_catalog.groovy @@ -68,7 +68,7 @@ suite("test_mariadb_jdbc_catalog", "p0,external,mariadb,external_docker,external sql """ insert into internal.${internal_db_name}.${inDorisTable} select id, name from ${ex_tb0}; """ order_qt_in_tb """ select id, name from internal.${internal_db_name}.${inDorisTable} order by id; """ - order_qt_information_schema """ show tables from information_schema; """ + order_qt_information_schema """ show tables from information_schema like "processlist"; """ order_qt_auto_default_t """insert into ${auto_default_t}(name) values('a'); """ order_qt_dt """select * from ${dt}; """ diff --git a/regression-test/suites/external_table_p0/jdbc/test_mysql_jdbc_catalog.groovy b/regression-test/suites/external_table_p0/jdbc/test_mysql_jdbc_catalog.groovy index d30d7fe9150effb..0be7d0ad396c3be 100644 --- a/regression-test/suites/external_table_p0/jdbc/test_mysql_jdbc_catalog.groovy +++ b/regression-test/suites/external_table_p0/jdbc/test_mysql_jdbc_catalog.groovy @@ -168,7 +168,7 @@ suite("test_mysql_jdbc_catalog", "p0,external,mysql,external_docker,external_doc order_qt_ex_tb21_6 """ select `key`, `id` from ${ex_tb21} where `key` = case when id = 1 then 1 else 0 end order by id;""" order_qt_ex_tb21_7 """ select (`key` +1) as k, `id` from ${ex_tb21} having abs(k) = 2 order by id;""" order_qt_ex_tb21_8 """ select `key` as k, `id` from ${ex_tb21} having abs(k) = 2 order by id;""" - order_qt_information_schema """ show tables from information_schema; """ + order_qt_information_schema """ show tables from information_schema like "processlist"; """ order_qt_dt """select * from ${dt}; """ order_qt_dt_null """select * from ${dt_null} order by 1; """ order_qt_test_dz """select * from ${test_zd} order by 1; """ diff --git a/regression-test/suites/external_table_p0/jdbc/test_mysql_jdbc_catalog_nereids.groovy b/regression-test/suites/external_table_p0/jdbc/test_mysql_jdbc_catalog_nereids.groovy index 24aac183eb861b4..e7ac412c0b58836 100644 --- a/regression-test/suites/external_table_p0/jdbc/test_mysql_jdbc_catalog_nereids.groovy +++ b/regression-test/suites/external_table_p0/jdbc/test_mysql_jdbc_catalog_nereids.groovy @@ -113,7 +113,7 @@ suite("test_mysql_jdbc_catalog_nereids", "p0,external,mysql,external_docker,exte order_qt_ex_tb18 """ select * from ${ex_tb18} order by num_tinyint; """ order_qt_ex_tb19 """ select * from ${ex_tb19} order by date_value; """ order_qt_ex_tb20 """ select * from ${ex_tb20} order by decimal_normal; """ - order_qt_information_schema """ show tables from information_schema; """ + order_qt_information_schema """ show tables from information_schema like "processlist"; """ // test insert String uuid1 = UUID.randomUUID().toString(); diff --git a/regression-test/suites/external_table_p0/jdbc/test_mysql_jdbc_driver5_catalog.groovy b/regression-test/suites/external_table_p0/jdbc/test_mysql_jdbc_driver5_catalog.groovy index fd1826d76e22d7e..a74890a56d897a1 100644 --- a/regression-test/suites/external_table_p0/jdbc/test_mysql_jdbc_driver5_catalog.groovy +++ b/regression-test/suites/external_table_p0/jdbc/test_mysql_jdbc_driver5_catalog.groovy @@ -175,7 +175,7 @@ suite("test_mysql_jdbc_driver5_catalog", "p0,external,mysql,external_docker,exte order_qt_ex_tb21_6 """ select `key`, `id` from ${ex_tb21} where `key` = case when id = 1 then 1 else 0 end order by id;""" order_qt_ex_tb21_7 """ select (`key` +1) as k, `id` from ${ex_tb21} having abs(k) = 2 order by id;""" order_qt_ex_tb21_8 """ select `key` as k, `id` from ${ex_tb21} having abs(k) = 2 order by id;""" - order_qt_information_schema """ show tables from information_schema; """ + order_qt_information_schema """ show tables from information_schema like "processlist"; """ order_qt_dt """select * from ${dt}; """ order_qt_dt_null """select * from ${dt_null} order by 1; """ order_qt_test_dz """select * from ${test_zd} order by 1; """ diff --git a/regression-test/suites/insert_p0/group_commit/test_group_commit_replay_wal.groovy b/regression-test/suites/insert_p0/group_commit/test_group_commit_replay_wal.groovy index da7bcadc11a32c2..1dc35f0b9df34a3 100644 --- a/regression-test/suites/insert_p0/group_commit/test_group_commit_replay_wal.groovy +++ b/regression-test/suites/insert_p0/group_commit/test_group_commit_replay_wal.groovy @@ -58,7 +58,6 @@ suite("test_group_commit_replay_wal", "nonConcurrent") { } getRowCount(5) // check wal count is 0 - sleep(5000) } catch (Exception e) { logger.info("failed: " + e.getMessage()) assertTrue(false) @@ -79,7 +78,7 @@ suite("test_group_commit_replay_wal", "nonConcurrent") { time 10000 } getRowCount(5) - sleep(10000) // wal replay but all failed + sleep(4000) // wal replay but all failed getRowCount(5) // check wal count is 1 diff --git a/regression-test/suites/insert_p0/insert_group_commit_into.groovy b/regression-test/suites/insert_p0/insert_group_commit_into.groovy index 31da87ade5f63b9..dbf2bd2e18ef121 100644 --- a/regression-test/suites/insert_p0/insert_group_commit_into.groovy +++ b/regression-test/suites/insert_p0/insert_group_commit_into.groovy @@ -406,10 +406,10 @@ suite("insert_group_commit_into") { "replication_allocation" = "tag.location.default: 1" ); """ sql """DROP MATERIALIZED VIEW IF EXISTS ods_zn_dnt_max1 ON ${table};""" - sql """create materialized view ods_zn_dnt_max1 as + createMV("""create materialized view ods_zn_dnt_max1 as select ordernum,max(dnt) as dnt from ${table} group by ordernum - ORDER BY ordernum;""" + ORDER BY ordernum;""") connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl) { sql """ set group_commit = async_mode; """ if (item == "nereids") { diff --git a/regression-test/suites/insert_p0/transaction/txn_insert_inject_case.groovy b/regression-test/suites/insert_p0/transaction/txn_insert_inject_case.groovy index 5478aeef69c70de..4e33d8a03ce313d 100644 --- a/regression-test/suites/insert_p0/transaction/txn_insert_inject_case.groovy +++ b/regression-test/suites/insert_p0/transaction/txn_insert_inject_case.groovy @@ -108,8 +108,11 @@ suite("txn_insert_inject_case", "nonConcurrent") { sql """ truncate table ${table}_0 """ // 1. publish timeout + def commit_timeout_second_value = getFeConfig("commit_timeout_second") + logger.info("commit_timeout_second_value: ${commit_timeout_second_value}") def backendId_to_params = get_be_param("pending_data_expire_time_sec") try { + setFeConfig('commit_timeout_second', '2') // test be report tablet and expire txns and fe handle it set_be_param.call("pending_data_expire_time_sec", "1") GetDebugPoint().enableDebugPointForAllFEs('PublishVersionDaemon.stop_publish') @@ -131,21 +134,21 @@ suite("txn_insert_inject_case", "nonConcurrent") { def result = sql "SELECT COUNT(*) FROM ${table}_0" rowCount = result[0][0] assertEquals(0, rowCount) - - sleep(10000) + // sleep(10000) } finally { + setFeConfig('commit_timeout_second', commit_timeout_second_value) set_original_be_param("pending_data_expire_time_sec", backendId_to_params) GetDebugPoint().disableDebugPointForAllFEs('PublishVersionDaemon.stop_publish') def rowCount = 0 - for (int i = 0; i < 30; i++) { + for (int i = 0; i < 600; i++) { def result = sql "SELECT COUNT(*) FROM ${table}_0" logger.info("select result: ${result}") rowCount = result[0][0] if (rowCount == 12) { break } - sleep(2000) + sleep(100) } assertEquals(12, rowCount) } @@ -188,10 +191,13 @@ suite("txn_insert_inject_case", "nonConcurrent") { assertTrue(txn_info[0].get("Reason").contains("DebugPoint: DatabaseTransactionMgr.commitTransaction.failed")) // 3. one txn publish failed + def insert_visible_timeout = sql """show variables where variable_name = 'insert_visible_timeout_ms';""" + logger.info("insert_visible_timeout: ${insert_visible_timeout}") sql """ truncate table ${table}_0 """ txn_id = 0 try (Connection conn = DriverManager.getConnection(url, context.config.jdbcUser, context.config.jdbcPassword); Statement statement = conn.createStatement()) { + statement.execute("ADMIN SET FRONTEND CONFIG ('commit_timeout_second' = '2');") statement.execute("begin") statement.execute("insert into ${table}_0 select * from ${table}_1;") txn_id = get_txn_id_from_server_info((((StatementImpl) statement).results).getServerInfo()) @@ -199,23 +205,29 @@ suite("txn_insert_inject_case", "nonConcurrent") { statement.execute("insert into ${table}_0 select * from ${table}_2;") statement.execute("commit") + sql "set insert_visible_timeout_ms = 2000" sql """insert into ${table}_0 values(100, 2.2, "abc", [], [])""" sql """insert into ${table}_1 values(101, 2.2, "abc", [], [])""" sql """insert into ${table}_2 values(102, 2.2, "abc", [], [])""" order_qt_select2 """select * from ${table}_0""" order_qt_select3 """select * from ${table}_1""" order_qt_select4 """select * from ${table}_2""" + } catch (Exception e) { + logger.info("failed", e) + assertTrue(false, "should not reach here") } finally { + setFeConfig('commit_timeout_second', commit_timeout_second_value) + sql "set insert_visible_timeout_ms = ${insert_visible_timeout[0][1]}" GetDebugPoint().disableDebugPointForAllFEs('PublishVersionDaemon.genPublishTask.failed') def rowCount = 0 - for (int i = 0; i < 20; i++) { + for (int i = 0; i < 200; i++) { def result = sql "select count(*) from ${table}_0" logger.info("rowCount: " + result + ", retry: " + i) rowCount = result[0][0] if (rowCount >= 7) { break } - sleep(1000) + sleep(100) } assertEquals(7, rowCount) } diff --git a/regression-test/suites/insert_p0/transaction/txn_insert_with_drop.groovy b/regression-test/suites/insert_p0/transaction/txn_insert_with_drop.groovy index 4807d65e4b70799..a314f3dfa4b6dcc 100644 --- a/regression-test/suites/insert_p0/transaction/txn_insert_with_drop.groovy +++ b/regression-test/suites/insert_p0/transaction/txn_insert_with_drop.groovy @@ -61,7 +61,10 @@ suite("txn_insert_with_drop", "nonConcurrent") { sql """ insert into ${table}_3 values(10, '1', 1), (20, '2', 2) """ sql """ insert into ${table}_4 values(30, '3', 3), (40, '4', 4), (5, '5', 5) """ + def commit_timeout_second_value = getFeConfig("commit_timeout_second") + logger.info("commit_timeout_second_value: ${commit_timeout_second_value}") try { + setFeConfig('commit_timeout_second', '2') // -------------------- drop partition -------------------- // 1. stop publish and txn insert GetDebugPoint().enableDebugPointForAllFEs('PublishVersionDaemon.stop_publish') @@ -129,6 +132,7 @@ suite("txn_insert_with_drop", "nonConcurrent") { } catch (Exception e) { logger.info("failed", e) } finally { + setFeConfig('commit_timeout_second', commit_timeout_second_value) GetDebugPoint().disableDebugPointForAllFEs('PublishVersionDaemon.stop_publish') } } diff --git a/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_dup_keys.groovy b/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_dup_keys.groovy index 1de813a310bdebb..ce6a7e7c6a355cc 100644 --- a/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_dup_keys.groovy +++ b/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_dup_keys.groovy @@ -24,6 +24,7 @@ suite("test_index_compaction_dup_keys", "nonConcurrent") { def backendId_to_backendHttpPort = [:] getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); + sql """ set global enable_match_without_inverted_index = false """ boolean disableAutoCompaction = false def set_be_config = { key, value -> @@ -240,5 +241,6 @@ suite("test_index_compaction_dup_keys", "nonConcurrent") { if (has_update_be_config) { set_be_config.call("inverted_index_compaction_enable", invertedIndexCompactionEnable.toString()) } + sql """ set global enable_match_without_inverted_index = true """ } } diff --git a/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_null.groovy b/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_null.groovy index 8a77c2dbf7d4411..5bf1ec1773b5593 100644 --- a/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_null.groovy +++ b/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_null.groovy @@ -24,6 +24,7 @@ suite("test_index_compaction_null", "nonConcurrent") { def backendId_to_backendHttpPort = [:] getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); + sql """ set global enable_match_without_inverted_index = false """ boolean disableAutoCompaction = false def set_be_config = { key, value -> @@ -119,6 +120,7 @@ suite("test_index_compaction_null", "nonConcurrent") { } def run_sql = { -> + sql """ set enable_common_expr_pushdown=true """ // select all data qt_select_0 "SELECT * FROM ${tableName} ORDER BY id" @@ -317,5 +319,7 @@ suite("test_index_compaction_null", "nonConcurrent") { if (has_update_be_config) { set_be_config.call("inverted_index_compaction_enable", invertedIndexCompactionEnable.toString()) } + + sql """ set global enable_match_without_inverted_index = true """ } } diff --git a/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_unique_keys.groovy b/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_unique_keys.groovy index 5afd6bada192ace..87996687b93950f 100644 --- a/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_unique_keys.groovy +++ b/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_unique_keys.groovy @@ -24,6 +24,7 @@ suite("test_index_compaction_unique_keys", "nonConcurrent") { def backendId_to_backendHttpPort = [:] getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); + sql """ set global enable_match_without_inverted_index = false """ boolean disableAutoCompaction = false def set_be_config = { key, value -> @@ -246,5 +247,6 @@ suite("test_index_compaction_unique_keys", "nonConcurrent") { if (has_update_be_config) { set_be_config.call("inverted_index_compaction_enable", invertedIndexCompactionEnable.toString()) } + sql """ set global enable_match_without_inverted_index = true """ } } diff --git a/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_with_multi_index_segments.groovy b/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_with_multi_index_segments.groovy index e9cc33002120504..edcf41db13c8774 100644 --- a/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_with_multi_index_segments.groovy +++ b/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_with_multi_index_segments.groovy @@ -24,6 +24,7 @@ suite("test_index_compaction_with_multi_index_segments", "nonConcurrent") { def backendId_to_backendHttpPort = [:] getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); + sql """ set global enable_match_without_inverted_index = false """ boolean disableAutoCompaction = false def set_be_config = { key, value -> @@ -401,5 +402,6 @@ suite("test_index_compaction_with_multi_index_segments", "nonConcurrent") { set_be_config.call("inverted_index_compaction_enable", invertedIndexCompactionEnable.toString()) set_be_config.call("inverted_index_max_buffered_docs", invertedIndexMaxBufferedDocs.toString()) } + sql """ set global enable_match_without_inverted_index = true """ } } diff --git a/regression-test/suites/inverted_index_p0/test_ignore_above_in_index.groovy b/regression-test/suites/inverted_index_p0/test_ignore_above_in_index.groovy index d6b426614414af3..bde764fa332e1fd 100644 --- a/regression-test/suites/inverted_index_p0/test_ignore_above_in_index.groovy +++ b/regression-test/suites/inverted_index_p0/test_ignore_above_in_index.groovy @@ -84,6 +84,7 @@ suite("test_ignore_above_in_index", "p0") { assertTrue(json.NumberLoadedRows > 0 && json.LoadBytes > 0) } } - + sql """ set enable_common_expr_pushdown = true; """ qt_sql "select count() from ${tableName2} where clientip > '17.0';" + qt_sql "select count() from ${tableName2} where clientip > '17.0' or status = 200;" } diff --git a/regression-test/suites/load_p0/stream_load/test_group_commit_stream_load_with_nonexist_db_and_table.groovy b/regression-test/suites/load_p0/stream_load/test_group_commit_stream_load_with_nonexist_db_and_table.groovy index ba806967bf79b1e..57054b2a42088ac 100644 --- a/regression-test/suites/load_p0/stream_load/test_group_commit_stream_load_with_nonexist_db_and_table.groovy +++ b/regression-test/suites/load_p0/stream_load/test_group_commit_stream_load_with_nonexist_db_and_table.groovy @@ -31,7 +31,7 @@ suite("test_group_commit_stream_load_with_nonexist_db_and_table") { code = process.waitFor() out = process.text log.info("stream lad result: ${out}".toString()) - assertTrue(out.toString().contains("table not found")) + assertTrue(out.toString().contains("OlapTable not found")) } catch (Exception e) { logger.info("failed: " + e.getMessage()) assertTrue(false) diff --git a/regression-test/suites/nereids_function_p0/scalar_function/S.groovy b/regression-test/suites/nereids_function_p0/scalar_function/S.groovy index ea6cc5c817e2a2a..98a8685a88840b9 100644 --- a/regression-test/suites/nereids_function_p0/scalar_function/S.groovy +++ b/regression-test/suites/nereids_function_p0/scalar_function/S.groovy @@ -97,26 +97,11 @@ suite("nereids_scalar_fn_S") { qt_sql_sm3sum_Varchar_notnull "select sm3sum(kvchrs1) from fn_test_not_nullable order by kvchrs1" qt_sql_sm3sum_String "select sm3sum(kstr) from fn_test order by kstr" qt_sql_sm3sum_String_notnull "select sm3sum(kstr) from fn_test_not_nullable order by kstr" + sql "select sm4_decrypt(kvchrs1, kvchrs1) from fn_test order by kvchrs1, kvchrs1" - test { - sql "select sm4_decrypt_v2(kvchrs1, kvchrs1) from fn_test order by kvchrs1, kvchrs1" - exception "Incorrect parameter count in the call to native function 'sm4_decrypt'" - } sql "select sm4_decrypt(kvchrs1, kvchrs1) from fn_test_not_nullable order by kvchrs1, kvchrs1" - test { - sql "select sm4_decrypt_v2(kvchrs1, kvchrs1) from fn_test_not_nullable order by kvchrs1, kvchrs1" - exception "Incorrect parameter count in the call to native function 'sm4_decrypt'" - } sql "select sm4_decrypt(kstr, kstr) from fn_test order by kstr, kstr" - test { - sql "select sm4_decrypt_v2(kstr, kstr) from fn_test order by kstr, kstr" - exception "Incorrect parameter count in the call to native function 'sm4_decrypt'" - } sql "select sm4_decrypt(kstr, kstr) from fn_test_not_nullable order by kstr, kstr" - test { - sql "select sm4_decrypt_v2(kstr, kstr) from fn_test_not_nullable order by kstr, kstr" - exception "Incorrect parameter count in the call to native function 'sm4_decrypt'" - } sql "select sm4_decrypt(kvchrs1, kvchrs1, kvchrs1) from fn_test order by kvchrs1, kvchrs1, kvchrs1" sql "select sm4_decrypt(kvchrs1, kvchrs1, kvchrs1) from fn_test_not_nullable order by kvchrs1, kvchrs1, kvchrs1" sql "select sm4_decrypt(kstr, kstr, kstr) from fn_test order by kstr, kstr, kstr" @@ -125,34 +110,11 @@ suite("nereids_scalar_fn_S") { sql "select sm4_decrypt(kvchrs1, kvchrs1, kvchrs1, 'SM4_128_ECB') from fn_test_not_nullable order by kvchrs1, kvchrs1, kvchrs1" sql "select sm4_decrypt(kstr, kstr, kstr, 'SM4_128_ECB') from fn_test order by kstr, kstr, kstr" sql "select sm4_decrypt(kstr, kstr, kstr, 'SM4_128_ECB') from fn_test_not_nullable order by kstr, kstr, kstr" - sql "select sm4_decrypt_v2(kvchrs1, kvchrs1, kvchrs1) from fn_test order by kvchrs1, kvchrs1, kvchrs1" - sql "select sm4_decrypt_v2(kvchrs1, kvchrs1, kvchrs1) from fn_test_not_nullable order by kvchrs1, kvchrs1, kvchrs1" - sql "select sm4_decrypt_v2(kstr, kstr, kstr) from fn_test order by kstr, kstr, kstr" - sql "select sm4_decrypt_v2(kstr, kstr, kstr) from fn_test_not_nullable order by kstr, kstr, kstr" - sql "select sm4_decrypt_v2(kvchrs1, kvchrs1, kvchrs1, 'SM4_128_ECB') from fn_test order by kvchrs1, kvchrs1, kvchrs1" - sql "select sm4_decrypt_v2(kvchrs1, kvchrs1, kvchrs1, 'SM4_128_ECB') from fn_test_not_nullable order by kvchrs1, kvchrs1, kvchrs1" - sql "select sm4_decrypt_v2(kstr, kstr, kstr, 'SM4_128_ECB') from fn_test order by kstr, kstr, kstr" - sql "select sm4_decrypt_v2(kstr, kstr, kstr, 'SM4_128_ECB') from fn_test_not_nullable order by kstr, kstr, kstr" + sql "select sm4_encrypt(kvchrs1, kvchrs1) from fn_test order by kvchrs1, kvchrs1" - test { - sql "select sm4_encrypt_v2(kvchrs1, kvchrs1) from fn_test order by kvchrs1, kvchrs1" - exception "Incorrect parameter count in the call to native function 'sm4_encrypt'" - } sql "select sm4_encrypt(kvchrs1, kvchrs1) from fn_test_not_nullable order by kvchrs1, kvchrs1" - test { - sql "select sm4_encrypt_v2(kvchrs1, kvchrs1) from fn_test_not_nullable order by kvchrs1, kvchrs1" - exception "Incorrect parameter count in the call to native function 'sm4_encrypt'" - } sql "select sm4_encrypt(kstr, kstr) from fn_test order by kstr, kstr" - test { - sql "select sm4_encrypt_v2(kstr, kstr) from fn_test order by kstr, kstr" - exception "Incorrect parameter count in the call to native function 'sm4_encrypt'" - } sql "select sm4_encrypt(kstr, kstr) from fn_test_not_nullable order by kstr, kstr" - test { - sql "select sm4_encrypt_v2(kstr, kstr) from fn_test_not_nullable order by kstr, kstr" - exception "Incorrect parameter count in the call to native function 'sm4_encrypt'" - } sql "select sm4_encrypt(kvchrs1, kvchrs1, kvchrs1) from fn_test order by kvchrs1, kvchrs1, kvchrs1" sql "select sm4_encrypt(kvchrs1, kvchrs1, kvchrs1) from fn_test_not_nullable order by kvchrs1, kvchrs1, kvchrs1" sql "select sm4_encrypt(kstr, kstr, kstr) from fn_test order by kstr, kstr, kstr" @@ -161,14 +123,7 @@ suite("nereids_scalar_fn_S") { sql "select sm4_encrypt(kvchrs1, kvchrs1, kvchrs1, 'SM4_128_ECB') from fn_test_not_nullable order by kvchrs1, kvchrs1, kvchrs1" sql "select sm4_encrypt(kstr, kstr, kstr, 'SM4_128_ECB') from fn_test order by kstr, kstr, kstr" sql "select sm4_encrypt(kstr, kstr, kstr, 'SM4_128_ECB') from fn_test_not_nullable order by kstr, kstr, kstr" - sql "select sm4_encrypt_v2(kvchrs1, kvchrs1, kvchrs1) from fn_test order by kvchrs1, kvchrs1, kvchrs1" - sql "select sm4_encrypt_v2(kvchrs1, kvchrs1, kvchrs1) from fn_test_not_nullable order by kvchrs1, kvchrs1, kvchrs1" - sql "select sm4_encrypt_v2(kstr, kstr, kstr) from fn_test order by kstr, kstr, kstr" - sql "select sm4_encrypt_v2(kstr, kstr, kstr) from fn_test_not_nullable order by kstr, kstr, kstr" - sql "select sm4_encrypt_v2(kvchrs1, kvchrs1, kvchrs1, 'SM4_128_ECB') from fn_test order by kvchrs1, kvchrs1, kvchrs1" - sql "select sm4_encrypt_v2(kvchrs1, kvchrs1, kvchrs1, 'SM4_128_ECB') from fn_test_not_nullable order by kvchrs1, kvchrs1, kvchrs1" - sql "select sm4_encrypt_v2(kstr, kstr, kstr, 'SM4_128_ECB') from fn_test order by kstr, kstr, kstr" - sql "select sm4_encrypt_v2(kstr, kstr, kstr, 'SM4_128_ECB') from fn_test_not_nullable order by kstr, kstr, kstr" + sql "select space(10) from fn_test order by kint" sql "select space(10) from fn_test_not_nullable order by kint" qt_sql_split_part_Varchar_Varchar_Integer "select split_part(kvchrs1, ' ', 1) from fn_test order by kvchrs1" diff --git a/regression-test/suites/nereids_p0/hint/test_use_mv.groovy b/regression-test/suites/nereids_p0/hint/test_use_mv.groovy new file mode 100644 index 000000000000000..e511ccc11ae071c --- /dev/null +++ b/regression-test/suites/nereids_p0/hint/test_use_mv.groovy @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +suite("test_use_mv") { + // create database and tables + sql 'DROP DATABASE IF EXISTS test_use_mv' + sql 'CREATE DATABASE IF NOT EXISTS test_use_mv' + sql 'use test_use_mv' + + // setting planner to nereids + sql 'set exec_mem_limit=21G' + sql 'set be_number_for_test=1' + sql 'set parallel_pipeline_task_num=1' + sql "set disable_nereids_rules=PRUNE_EMPTY_PARTITION" + sql 'set enable_nereids_planner=true' + sql 'set enable_nereids_distribute_planner=false' + sql "set ignore_shape_nodes='PhysicalProject'" + sql 'set enable_fallback_to_original_planner=false' + sql 'set runtime_filter_mode=OFF' + + sql """drop table if exists t1;""" + // create tables + sql """ + CREATE TABLE `t1` ( + `k1` int(11) NULL, + `k2` int(11) NULL, + `v1` int(11) SUM NULL + ) ENGINE=OLAP + AGGREGATE KEY(`k1`, `k2`) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(`k1`) BUCKETS 3 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "in_memory" = "false", + "storage_format" = "V2", + "disable_auto_compaction" = "false" + ); + """ + sql """ alter table t1 add rollup r1(k2, k1); """ + waitForRollUpJob("t1", 5000, 1) + sql """ alter table t1 add rollup r2(k2); """ + waitForRollUpJob("t1", 5000, 1) + createMV("create materialized view k1_k2_sumk3 as select k1, k2, sum(v1) from t1 group by k1, k2;") + sql """set enable_sync_mv_cost_based_rewrite = false""" + explain { + sql """select k1 from t1;""" + contains("t1(r1)") + } + sql """set enable_sync_mv_cost_based_rewrite = true""" + explain { + sql """select /*+ no_use_mv */ k1 from t1;""" + notContains("t1(r1)") + } + explain { + sql """select /*+ no_use_mv(t1) */ k1 from t1;""" + contains("parameter of no_use_mv hint must be in pairs") + } + explain { + sql """select /*+ no_use_mv(t1.`*`) */ k1 from t1;""" + contains("t1(t1)") + } + explain { + sql """select /*+ use_mv(t1.`*`) */ k1 from t1;""" + contains("use_mv hint should only have one mv in one table") + } + explain { + sql """select /*+ use_mv(t1.r1,t1.r2) */ k1 from t1;""" + contains("use_mv hint should only have one mv in one table") + } + explain { + sql """select /*+ use_mv(t1.r1) use_mv(t1.r2) */ k1 from t1;""" + contains("one use_mv hint is allowed") + } + explain { + sql """select /*+ no_use_mv(t1.r1) no_use_mv(t1.r2) */ k1 from t1;""" + contains("only one no_use_mv hint is allowed") + } + explain { + sql """select /*+ no_use_mv(t1.r3) */ k1 from t1;""" + contains("do not have mv: r3 in table: t1") + } + explain { + sql """select /*+ use_mv(t1.r1) no_use_mv(t1.r1) */ k1 from t1;""" + contains("conflict mv exist in use_mv and no_use_mv in the same time") + } + explain { + sql """select /*+ use_mv(t1.k1_k2_sumk3) */ k1, k2, sum(v1) from t1 group by k1, k2;""" + contains("t1(k1_k2_sumk3)") + } + explain { + sql """select /*+ use_mv(t1.k1_k2_sumk3) */ k1, k2, min(v1) from t1 group by k1, k2;""" + notContains("t1(k1_k2_sumk3)") + } + +} diff --git a/regression-test/suites/nereids_p0/insert_into_table/update_on_current_timestamp.groovy b/regression-test/suites/nereids_p0/insert_into_table/update_on_current_timestamp.groovy index 597facb3ea345ba..ef5496a7a76fcbe 100644 --- a/regression-test/suites/nereids_p0/insert_into_table/update_on_current_timestamp.groovy +++ b/regression-test/suites/nereids_p0/insert_into_table/update_on_current_timestamp.groovy @@ -190,6 +190,6 @@ suite("nereids_update_on_current_timestamp") { k int, `update_time` datetime(6) default current_timestamp(4) on update current_timestamp(3)) replace, ) AGGREGATE KEY(k) DISTRIBUTED BY HASH(k) BUCKETS 1 properties("replication_num" = "1");""" - exception "Syntax error in line 3" + exception "mismatched input 'replace'" } } diff --git a/regression-test/suites/nereids_p0/sql_functions/encryption_digest/test_encryption_function.groovy b/regression-test/suites/nereids_p0/sql_functions/encryption_digest/test_encryption_function.groovy index 84b67091b1e02dd..0a2d42610012dc5 100644 --- a/regression-test/suites/nereids_p0/sql_functions/encryption_digest/test_encryption_function.groovy +++ b/regression-test/suites/nereids_p0/sql_functions/encryption_digest/test_encryption_function.groovy @@ -16,10 +16,9 @@ // under the License. suite("test_encryption_function") { - def tableName = "dwd_candidates" - sql "DROP TABLE IF EXISTS ${tableName}" + sql "DROP TABLE IF EXISTS dwd_candidates" sql """ - CREATE TABLE IF NOT EXISTS ${tableName} ( + CREATE TABLE IF NOT EXISTS dwd_candidates ( c_int INT, `name` varchar(65530) NULL COMMENT "" ) @@ -29,26 +28,12 @@ suite("test_encryption_function") { ) """ - sql "SET enable_nereids_planner=true" - - sql "set batch_size = 4096;" - sql "set block_encryption_mode=\"AES_128_ECB\";" qt_sql "SELECT TO_BASE64(AES_ENCRYPT('text','F3229A0B371ED2D9441B830D21A390C3'));" // 'wr2JEDVXzL9+2XtRhgIloA==' qt_sql "SELECT TO_BASE64(AES_ENCRYPT('text','F3229A0B371ED2D9441B830D21A390C3', '0123456789'));" // wr2JEDVXzL9+2XtRhgIloA== qt_sql "SELECT AES_DECRYPT(FROM_BASE64('wr2JEDVXzL9+2XtRhgIloA=='),'F3229A0B371ED2D9441B830D21A390C3');" // text qt_sql "SELECT AES_DECRYPT(FROM_BASE64('mvZT1KJw7N0RJf27aipUpg=='),'F3229A0B371ED2D9441B830D21A390C3', '0123456789');" // NULL - sql "set block_encryption_mode=\"AES_128_ECB\";" - qt_sql "SELECT TO_BASE64(AES_ENCRYPT_v2('text','F3229A0B371ED2D9441B830D21A390C3'));" // 'wr2JEDVXzL9+2XtRhgIloA==' - qt_sql "SELECT TO_BASE64(AES_ENCRYPT_v2('text','F3229A0B371ED2D9441B830D21A390C3', '0123456789'));" // wr2JEDVXzL9+2XtRhgIloA== - qt_sql "SELECT AES_DECRYPT_v2(FROM_BASE64('wr2JEDVXzL9+2XtRhgIloA=='),'F3229A0B371ED2D9441B830D21A390C3');" // text - qt_sql "SELECT AES_DECRYPT_v2(FROM_BASE64('mvZT1KJw7N0RJf27aipUpg=='),'F3229A0B371ED2D9441B830D21A390C3', '0123456789');" // NULL - test { - sql "SELECT TO_BASE64(SM4_ENCRYPT_v2('text','F3229A0B371ED2D9441B830D21A390C3'));" - exception "session variable block_encryption_mode is invalid with sm4" - } - sql "set block_encryption_mode=\"AES_256_ECB\";" qt_sql "SELECT TO_BASE64(AES_ENCRYPT('text','F3229A0B371ED2D9441B830D21A390C3'));" // 'wr2JEDVXzL9+2XtRhgIloA==' qt_sql "SELECT TO_BASE64(AES_ENCRYPT('text','F3229A0B371ED2D9441B830D21A390C3', '0123456789'));" // 'BO2vxHeUcw5BQQalSBbo1w==' @@ -56,13 +41,6 @@ suite("test_encryption_function") { qt_sql "SELECT AES_DECRYPT(FROM_BASE64('BO2vxHeUcw5BQQalSBbo1w=='),'F3229A0B371ED2D9441B830D21A390C3');" // NULL qt_sql "SELECT AES_DECRYPT(FROM_BASE64('BO2vxHeUcw5BQQalSBbo1w=='),'F3229A0B371ED2D9441B830D21A390C3', '0123456789');" // text - sql "set block_encryption_mode=\"AES_256_ECB\";" - qt_sql "SELECT TO_BASE64(AES_ENCRYPT_v2('text','F3229A0B371ED2D9441B830D21A390C3'));" // 'BO2vxHeUcw5BQQalSBbo1w==' - qt_sql "SELECT TO_BASE64(AES_ENCRYPT_v2('text','F3229A0B371ED2D9441B830D21A390C3', '0123456789'));" // 'BO2vxHeUcw5BQQalSBbo1w==' - qt_sql "SELECT AES_DECRYPT_v2(FROM_BASE64('wr2JEDVXzL9+2XtRhgIloA=='),'F3229A0B371ED2D9441B830D21A390C3');" // NULL - qt_sql "SELECT AES_DECRYPT_v2(FROM_BASE64('BO2vxHeUcw5BQQalSBbo1w=='),'F3229A0B371ED2D9441B830D21A390C3');" // text - qt_sql "SELECT AES_DECRYPT_v2(FROM_BASE64('BO2vxHeUcw5BQQalSBbo1w=='),'F3229A0B371ED2D9441B830D21A390C3', '0123456789');" // text - sql "set block_encryption_mode=\"AES_256_CBC\";" qt_sql "SELECT TO_BASE64(AES_ENCRYPT('text','F3229A0B371ED2D9441B830D21A390C3'));" // 'wr2JEDVXzL9+2XtRhgIloA==' qt_sql "SELECT TO_BASE64(AES_ENCRYPT('text','F3229A0B371ED2D9441B830D21A390C3', '0123456789'));" // tsmK1HzbpnEdR2//WhO+MA== @@ -76,53 +54,6 @@ suite("test_encryption_function") { "GROUP BY x0;" } - sql "set block_encryption_mode=\"AES_256_CBC\";" - test { - sql "SELECT TO_BASE64(AES_ENCRYPT_v2('text','F3229A0B371ED2D9441B830D21A390C3'));" - exception "Incorrect parameter count in the call to native function 'aes_encrypt'" - } - qt_sql "SELECT TO_BASE64(AES_ENCRYPT_v2('text','F3229A0B371ED2D9441B830D21A390C3', '0123456789'));" // tsmK1HzbpnEdR2//WhO+MA== - qt_sql "SELECT TO_BASE64(AES_ENCRYPT('text','F3229A0B371ED2D9441B830D21A390C3', '0123456789ff'));" // ciacXDLHMNG7CD9Kws8png== - test { - sql "SELECT AES_DECRYPT_v2(FROM_BASE64('wr2JEDVXzL9+2XtRhgIloA=='),'F3229A0B371ED2D9441B830D21A390C3');" - exception "Incorrect parameter count in the call to native function 'aes_decrypt'" - } - qt_sql "SELECT AES_DECRYPT_v2(FROM_BASE64('mvZT1KJw7N0RJf27aipUpg=='),'F3229A0B371ED2D9441B830D21A390C3', '0123456789');" // NULL - qt_sql "SELECT AES_DECRYPT_v2(FROM_BASE64('tsmK1HzbpnEdR2//WhO+MA=='),'F3229A0B371ED2D9441B830D21A390C3', '0123456789');" // text - qt_sql "SELECT AES_DECRYPT_v2(FROM_BASE64('ciacXDLHMNG7CD9Kws8png=='),'F3229A0B371ED2D9441B830D21A390C3', '0123456789ff');" // text - explain { - sql "SELECT AES_DECRYPT_v2(UNHEX(r_2_3.`name`), 'namePnhe3E0MWyfZivUnVzDy12caymnrKp', '0123456789') AS x0 FROM dwd_candidates AS r_2_3\n" + - "GROUP BY x0;" - } - - sql "set block_encryption_mode=\"SM4_128_CBC\";" - qt_sql "SELECT TO_BASE64(SM4_ENCRYPT('text','F3229A0B371ED2D9441B830D21A390C3'));" // aDjwRflBrDjhBZIOFNw3Tg== - qt_sql "SELECT TO_BASE64(SM4_ENCRYPT('text','F3229A0B371ED2D9441B830D21A390C3', '0123456789'));" // 1Y4NGIukSbv9OrkZnRD1bQ== - qt_sql "SELECT TO_BASE64(SM4_ENCRYPT('text','F3229A0B371ED2D9441B830D21A390C3', '0123456789ff'));" // G5POcFAJwiZHeTtN6DjInQ== - qt_sql "SELECT SM4_DECRYPT(FROM_BASE64('aDjwRflBrDjhBZIOFNw3Tg=='),'F3229A0B371ED2D9441B830D21A390C3');" // text - qt_sql "SELECT SM4_DECRYPT(FROM_BASE64('1Y4NGIukSbv9OrkZnRD1bQ=='),'F3229A0B371ED2D9441B830D21A390C3', '0123456789');" // text - qt_sql "SELECT SM4_DECRYPT(FROM_BASE64('G5POcFAJwiZHeTtN6DjInQ=='),'F3229A0B371ED2D9441B830D21A390C3', '0123456789');" // NULL - qt_sql "SELECT SM4_DECRYPT(FROM_BASE64('G5POcFAJwiZHeTtN6DjInQ=='),'F3229A0B371ED2D9441B830D21A390C3', '0123456789ff');" // text - - sql "set block_encryption_mode=\"SM4_128_CBC\";" - test { - sql "SELECT TO_BASE64(SM4_ENCRYPT_v2('text','F3229A0B371ED2D9441B830D21A390C3'));" - exception "Incorrect parameter count in the call to native function 'sm4_encrypt'" - } - qt_sql "SELECT TO_BASE64(SM4_ENCRYPT_v2('text','F3229A0B371ED2D9441B830D21A390C3', '0123456789'));" // 1Y4NGIukSbv9OrkZnRD1bQ== - qt_sql "SELECT TO_BASE64(SM4_ENCRYPT_v2('text','F3229A0B371ED2D9441B830D21A390C3', '0123456789ff'));" // G5POcFAJwiZHeTtN6DjInQ== - test { - sql "SELECT SM4_DECRYPT_v2(FROM_BASE64('aDjwRflBrDjhBZIOFNw3Tg=='),'F3229A0B371ED2D9441B830D21A390C3');" - exception "Incorrect parameter count in the call to native function 'sm4_decrypt'" - } - qt_sql "SELECT SM4_DECRYPT_v2(FROM_BASE64('1Y4NGIukSbv9OrkZnRD1bQ=='),'F3229A0B371ED2D9441B830D21A390C3', '0123456789');" // text - qt_sql "SELECT SM4_DECRYPT_v2(FROM_BASE64('G5POcFAJwiZHeTtN6DjInQ=='),'F3229A0B371ED2D9441B830D21A390C3', '0123456789');" // NULL - qt_sql "SELECT SM4_DECRYPT_v2(FROM_BASE64('G5POcFAJwiZHeTtN6DjInQ=='),'F3229A0B371ED2D9441B830D21A390C3', '0123456789ff');" // text - test { - sql "SELECT TO_BASE64(AES_ENCRYPT_v2('text','F3229A0B371ED2D9441B830D21A390C3'));" - exception "session variable block_encryption_mode is invalid with aes" - } - qt_sql "SELECT SM3(\"abc\");" qt_sql "select sm3(\"abcd\");" qt_sql "select sm3sum(\"ab\",\"cd\");" diff --git a/regression-test/suites/nereids_rules_p0/grouping_sets/valid_grouping.groovy b/regression-test/suites/nereids_rules_p0/grouping_sets/valid_grouping.groovy new file mode 100644 index 000000000000000..dd69f23f027f214 --- /dev/null +++ b/regression-test/suites/nereids_rules_p0/grouping_sets/valid_grouping.groovy @@ -0,0 +1,51 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("valid_grouping"){ + sql "SET enable_fallback_to_original_planner=true" + sql "drop table if exists valid_grouping" + sql """ + CREATE TABLE `valid_grouping` ( + `a` INT NULL, + `b` VARCHAR(10) NULL, + `c` INT NULL, + `d` INT NULL + ) ENGINE=OLAP + DUPLICATE KEY(`a`, `b`) + DISTRIBUTED BY RANDOM BUCKETS AUTO + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + ); + """ + sql "insert into valid_grouping values(1,'d2',3,5);" + test { + sql """select + b, 'day' as DT_TYPE + from valid_grouping + group by grouping sets ( (grouping_id(b)),(b));""" + exception("GROUP BY expression must not contain grouping scalar functions: grouping_id(`b`)") + } + + test { + sql """select + b, 'day' as DT_TYPE + from valid_grouping + group by grouping sets ( (grouping(b)),(b));""" + exception("GROUP BY expression must not contain grouping scalar functions: grouping(`b`)") + } + +} \ No newline at end of file diff --git a/regression-test/suites/nereids_rules_p0/max_min_filter_push_down/max_min_filter_push_down.groovy b/regression-test/suites/nereids_rules_p0/max_min_filter_push_down/max_min_filter_push_down.groovy new file mode 100644 index 000000000000000..47610f2e1257019 --- /dev/null +++ b/regression-test/suites/nereids_rules_p0/max_min_filter_push_down/max_min_filter_push_down.groovy @@ -0,0 +1,214 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +suite("max_min_filter_push_down") { + sql "SET enable_nereids_planner=true" + sql "SET enable_fallback_to_original_planner=false" + sql "SET ignore_shape_nodes='PhysicalDistribute,PhysicalProject'" + + sql "drop table if exists max_min_filter_push_down1" + sql""" + CREATE TABLE max_min_filter_push_down1 ( + id INT, + value1 INT, + value2 VARCHAR(50) + ) properties("replication_num"="1"); + """ + + sql """ + INSERT INTO max_min_filter_push_down1 (id, value1, value2) VALUES + (1, 10, 'A'),(1, 11, 'A'),(2, 20, 'B'),(2, 73, 'B'),(2, 19, 'B'),(3, 30, 'C'),(3, 61, 'C'),(4, 40, 'D'),(4, 43, 'D'),(4, 45, 'D'); + """ + sql "drop table if exists max_min_filter_push_down_empty" + sql "create table max_min_filter_push_down_empty like max_min_filter_push_down1" + + qt_scalar_agg_empty_table """ + explain shape plan + select min(value1) from max_min_filter_push_down_empty having min(value1) <40 and min(value1) <20; + """ + qt_min """ + explain shape plan + select id,min(value1) from max_min_filter_push_down1 group by id having min(value1) <40 and min(value1) <20; + """ + qt_max """ + explain shape plan + select id,max(value1) from max_min_filter_push_down1 group by id having max(value1) >40; + """ + + qt_min_expr """ + explain shape plan + select id,min(value1+1) from max_min_filter_push_down1 group by id having min(value1+1) <40 and min(value1+1) <20; + """ + qt_max_expr """ + explain shape plan + select id,max(abs(value1)+1) from max_min_filter_push_down1 group by id having max(abs(value1)+1) >40; + """ + + qt_min_commute """ + explain shape plan + select id,min(value1) from max_min_filter_push_down1 group by id having 40>min(value1); + """ + qt_max """ + explain shape plan + select id,max(value1) from max_min_filter_push_down1 group by id having 40=40; + """ + + qt_min_commute_equal """ + explain shape plan + select id,min(value1) from max_min_filter_push_down1 group by id having 40>=min(value1); + """ + qt_max_commute_equal """ + explain shape plan + select id,max(value1) from max_min_filter_push_down1 group by id having 40<=max(value1); + """ + + qt_has_other_agg_func """ + explain shape plan + select id,max(value1),min(value1) from max_min_filter_push_down1 group by id having 40<=max(value1); + """ + + qt_min_scalar_agg """ + explain shape plan + select min(value1) from max_min_filter_push_down1 having min(value1) <40; + """ + qt_max_scalar_agg """ + explain shape plan + select max(value1) from max_min_filter_push_down1 having max(value1) >40; + """ + qt_max_scalar_agg """ + explain shape plan + select max(value1) from max_min_filter_push_down1 having 40=40; + """ + + qt_depend_prune_column """ + explain shape plan + select c1 from (select min(value1) c1,max(value2) from max_min_filter_push_down1 group by id having min(value1)<10) t + """ + + qt_scalar_agg_empty_table_res """ + select min(value1) from max_min_filter_push_down_empty having min(value1) <40 and min(value1) <20; + """ + qt_min_res """ + select id,min(value1) from max_min_filter_push_down1 group by id having min(value1) <40 and min(value1) <20 order by 1,2; + """ + qt_max_res """ + select id,max(value1) from max_min_filter_push_down1 group by id having max(value1) >40 order by 1,2; + """ + qt_min_expr_res """ + select id,min(value1+1) from max_min_filter_push_down1 group by id having min(value1+1) <40 and min(value1+1) <20 order by 1,2; + """ + qt_max_expr_res """ + select id,max(abs(value1)+1) from max_min_filter_push_down1 group by id having max(abs(value1)+1) >40 order by 1,2; + """ + qt_min_commute_res """ + select id,min(value1) from max_min_filter_push_down1 group by id having 40>min(value1) order by 1,2; + """ + qt_max_res """ + select id,max(value1) from max_min_filter_push_down1 group by id having 40=40 order by 1,2; + """ + + qt_min_commute_equal_res """ + select id,min(value1) from max_min_filter_push_down1 group by id having 40>=min(value1) order by 1,2; + """ + qt_max_commute_equal_res """ + select id,max(value1) from max_min_filter_push_down1 group by id having 40<=max(value1) order by 1,2; + """ + + qt_has_other_agg_func_res """ + select id,max(value1),min(value1) from max_min_filter_push_down1 group by id having 40<=max(value1) order by 1,2; + """ + + qt_min_scalar_agg_res """ + select min(value1) from max_min_filter_push_down1 having min(value1) <40; + """ + qt_max_scalar_agg_res """ + select max(value1) from max_min_filter_push_down1 having max(value1) >40; + """ + qt_max_scalar_agg_res """ + select max(value1) from max_min_filter_push_down1 having 40=40; + """ + qt_depend_prune_column_res """ + select c1 from (select min(value1) c1,max(value2) from max_min_filter_push_down1 group by id having min(value1)<20) t order by c1 + """ + + sql "drop table if exists max_min_filter_push_down2" + sql """create table max_min_filter_push_down2(d_int int, d_char100 char(100), d_smallint smallint, d_tinyint tinyint, d_char10 char(10),d_datetimev2 datetimev2, d_datev2 datev2) + properties("replication_num"="1");""" + sql """insert into max_min_filter_push_down2 values(1,'01234567890123456789', 3,3,'0123456789','2020-01-09 10:00:00.99','2020-01-09') + ,(14,'01234567890123456789', 29,23,'0123456789','2020-01-7 10:00:00.99','2020-01-11'),(1,'01234567890123456789', 7,23,'0123456789','2020-01-7 10:00:00.99','2020-01-11') + ,(14,'01234567890123456789', 32,23,'0123456789','2020-01-11 10:00:00.99','2020-01-11'),(1,'01234567890123456789', 8,23,'0123456789','2020-01-11 10:00:00.99','2020-01-11');""" + + qt_smallint """explain shape plan + select d_int,max(d_smallint) from max_min_filter_push_down2 group by d_int having max(d_smallint)>10;""" + qt_tinyint """explain shape plan + select d_int,min(d_tinyint) from max_min_filter_push_down2 group by d_int having min(d_tinyint)<10;""" + qt_char100 """explain shape plan + select d_int,max(d_char100) from max_min_filter_push_down2 group by d_int having max(d_char100)>'ab';""" + qt_char100_cmp_num_cannot_rewrite """explain shape plan + select d_int,min(d_char100) from max_min_filter_push_down2 group by d_int having min(d_char100)<10;""" + qt_datetimev2 """explain shape plan + select d_int,min(d_datetimev2) from max_min_filter_push_down2 group by d_int having min(d_datetimev2)<'2020-01-09';""" + qt_datev2 """explain shape plan + select d_int,max(d_datev2) from max_min_filter_push_down2 group by d_int having max(d_datev2)>'2020-01-09 10:00:00';""" + qt_smallint_group_by_key """explain shape plan + select max(d_smallint) from max_min_filter_push_down2 group by d_smallint having max(d_smallint)>10;""" + qt_tinyint_group_by_key """explain shape plan + select min(d_tinyint) from max_min_filter_push_down2 group by d_tinyint having min(d_tinyint)<10;""" + qt_char100_group_by_key """explain shape plan + select max(d_char100) from max_min_filter_push_down2 group by d_char100 having max(d_char100)>'ab';""" + + qt_smallint_res """select d_int,max(d_smallint) from max_min_filter_push_down2 group by d_int having max(d_smallint)>10 order by 1,2;""" + qt_tinyint_res """select d_int,min(d_tinyint) from max_min_filter_push_down2 group by d_int having min(d_tinyint)<10 order by 1,2;""" + qt_char100_res """select d_int,max(d_char100) from max_min_filter_push_down2 group by d_int having max(d_char100)>'ab' order by 1,2;""" + qt_char100_cmp_num_cannot_rewrite_res """select d_int,min(d_char100) from max_min_filter_push_down2 group by d_int having min(d_char100)<10 order by 1,2;""" + qt_datetimev2_res """select d_int,min(d_datetimev2) from max_min_filter_push_down2 group by d_int having min(d_datetimev2)<'2020-01-09' order by 1,2;""" + qt_datev2_res """select d_int,max(d_datev2) from max_min_filter_push_down2 group by d_int having max(d_datev2)>'2020-01-09 10:00:00' order by 1,2;""" + qt_smallint_group_by_key_res """select max(d_smallint) from max_min_filter_push_down2 group by d_smallint having max(d_smallint)>10 order by 1;""" + qt_tinyint_group_by_key_res """select min(d_tinyint) from max_min_filter_push_down2 group by d_tinyint having min(d_tinyint)<10 order by 1;""" + qt_char100_group_by_key_res """select max(d_char100) from max_min_filter_push_down2 group by d_char100 having max(d_char100)>'ab' order by 1;""" +} \ No newline at end of file diff --git a/regression-test/suites/nereids_rules_p0/mv/ssb/mv_ssb_test.groovy b/regression-test/suites/nereids_rules_p0/mv/ssb/mv_ssb_test.groovy index 17849eea2cbf885..a80690a7bbef5ab 100644 --- a/regression-test/suites/nereids_rules_p0/mv/ssb/mv_ssb_test.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/ssb/mv_ssb_test.groovy @@ -61,7 +61,7 @@ suite("mv_ssb_test") { // relate to ${DORIS_HOME}/regression-test/data/demo/streamload_input.csv. // also, you can stream load a http stream, e.g. http://xxx/some.csv - file """${getS3Url()}/regression/ssb/sf1/${tableName}.tbl.gz""" + file """${getS3Url()}/regression/ssb/sf0.1/${tableName}.tbl.gz""" time 10000 // limit inflight 10s @@ -358,7 +358,7 @@ suite("mv_ssb_test") { OR c_city='UNITED KI5') AND (s_city='UNITED KI1' OR s_city='UNITED KI5') - AND d_yearmonth = 'Dec1997' + AND d_yearmonth = 'Jul1992' GROUP BY c_city, s_city, d_year ORDER BY d_year ASC, REVENUE DESC; """ @@ -373,7 +373,7 @@ suite("mv_ssb_test") { OR c_city='UNITED KI5') AND (s_city='UNITED KI1' OR s_city='UNITED KI5') - AND d_yearmonth = 'Dec1997' + AND d_yearmonth = 'Jul1992' GROUP BY c_city, s_city, d_year ORDER BY d_year ASC, REVENUE DESC; """ @@ -428,7 +428,7 @@ suite("mv_ssb_test") { AND lo_orderdate = d_datekey AND c_region = 'AMERICA' AND s_region = 'AMERICA' - AND (d_year = 1997 OR d_year = 1998) + AND (d_year = 1992 OR d_year = 1993) AND (p_mfgr = 'MFGR#1' OR p_mfgr = 'MFGR#2') GROUP BY d_year, s_nation, p_category @@ -444,7 +444,7 @@ suite("mv_ssb_test") { AND lo_orderdate = d_datekey AND c_region = 'AMERICA' AND s_region = 'AMERICA' - AND (d_year = 1997 OR d_year = 1998) + AND (d_year = 1992 OR d_year = 1993) AND (p_mfgr = 'MFGR#1' OR p_mfgr = 'MFGR#2') GROUP BY d_year, s_nation, p_category @@ -465,7 +465,7 @@ suite("mv_ssb_test") { AND lo_partkey = p_partkey AND lo_orderdate = d_datekey AND s_nation = 'UNITED STATES' - AND (d_year = 1997 OR d_year = 1998) + AND (d_year = 1992 OR d_year = 1993) AND p_category = 'MFGR#14' GROUP BY d_year, s_city, p_brand ORDER BY d_year, s_city, p_brand; @@ -479,7 +479,7 @@ suite("mv_ssb_test") { AND lo_partkey = p_partkey AND lo_orderdate = d_datekey AND s_nation = 'UNITED STATES' - AND (d_year = 1997 OR d_year = 1998) + AND (d_year = 1992 OR d_year = 1993) AND p_category = 'MFGR#14' GROUP BY d_year, s_city, p_brand ORDER BY d_year, s_city, p_brand; diff --git a/regression-test/suites/partition_p0/auto_partition/test_auto_dynamic.groovy b/regression-test/suites/partition_p0/auto_partition/test_auto_dynamic.groovy new file mode 100644 index 000000000000000..c5fa8a736dbf740 --- /dev/null +++ b/regression-test/suites/partition_p0/auto_partition/test_auto_dynamic.groovy @@ -0,0 +1,130 @@ + + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_auto_dynamic", "nonConcurrent") { + // PROHIBIT different timeunit of interval when use both auto & dynamic partition + test{ + sql """ + CREATE TABLE tbl3 + ( + k1 DATETIME NOT NULL, + col1 int + ) + auto partition by range (date_trunc(`k1`, 'year')) () + DISTRIBUTED BY HASH(k1) + PROPERTIES + ( + "replication_num" = "1", + "dynamic_partition.create_history_partition"="true", + "dynamic_partition.enable" = "true", + "dynamic_partition.time_unit" = "HOUR", + "dynamic_partition.start" = "-2", + "dynamic_partition.end" = "2", + "dynamic_partition.prefix" = "p", + "dynamic_partition.buckets" = "8" + ); + """ + exception "If support auto partition and dynamic partition at same time, they must have the same interval unit." + } + + sql " drop table if exists test_dynamic " + sql """ + create table test_dynamic( + k0 DATE not null + ) + auto partition by range (date_trunc(k0, 'year')) () + DISTRIBUTED BY HASH(`k0`) BUCKETS auto + properties("replication_num" = "1"); + """ + test { + sql """ + ALTER TABLE test_dynamic set ( + "dynamic_partition.enable" = "true", + "dynamic_partition.time_unit" = "DAY", + "dynamic_partition.end" = "3", + "dynamic_partition.prefix" = "p", + "dynamic_partition.buckets" = "32" + ); + """ + exception "If support auto partition and dynamic partition at same time, they must have the same interval unit." + } + sql """ + ALTER TABLE test_dynamic set ( + "dynamic_partition.enable" = "true", + "dynamic_partition.time_unit" = "YeAr", + "dynamic_partition.end" = "3", + "dynamic_partition.prefix" = "p", + "dynamic_partition.buckets" = "32" + ); + """ + + sql " drop table if exists auto_dynamic " + sql """ + create table auto_dynamic( + k0 datetime(6) NOT NULL + ) + auto partition by range (date_trunc(k0, 'hour')) + ( + ) + DISTRIBUTED BY HASH(`k0`) BUCKETS 2 + properties( + "dynamic_partition.enable" = "true", + "dynamic_partition.prefix" = "p", + "dynamic_partition.create_history_partition" = "true", + "dynamic_partition.start" = "-5", + "dynamic_partition.end" = "0", + "dynamic_partition.time_unit" = "hour", + "replication_num" = "1" + ); + """ + def part_result = sql " show partitions from auto_dynamic " + assertEquals(part_result.size, 6) + + sql " drop table if exists auto_dynamic " + sql """ + create table auto_dynamic( + k0 datetime(6) NOT NULL + ) + auto partition by range (date_trunc(k0, 'year')) + ( + ) + DISTRIBUTED BY HASH(`k0`) BUCKETS 2 + properties( + "dynamic_partition.enable" = "true", + "dynamic_partition.prefix" = "p", + "dynamic_partition.start" = "-50", + "dynamic_partition.end" = "0", + "dynamic_partition.time_unit" = "year", + "replication_num" = "1" + ); + """ + part_result = sql " show partitions from auto_dynamic " + assertEquals(part_result.size, 1) + + sql " insert into auto_dynamic values ('2024-01-01'), ('2900-01-01'), ('1900-01-01'), ('3000-01-01'); " + sql """ admin set frontend config ('dynamic_partition_check_interval_seconds' = '1') """ + sleep(2000) + part_result = sql " show partitions from auto_dynamic " + log.info("${part_result}".toString()) + assertEquals(part_result.size, 3) + + qt_sql_dynamic_auto "select * from auto_dynamic order by k0;" + + sql """ admin set frontend config ('dynamic_partition_check_interval_seconds' = '600') """ +} \ No newline at end of file diff --git a/regression-test/suites/partition_p0/auto_partition/test_auto_partition_behavior.groovy b/regression-test/suites/partition_p0/auto_partition/test_auto_partition_behavior.groovy index e5ce52af31e395c..01fdfd4d6342610 100644 --- a/regression-test/suites/partition_p0/auto_partition/test_auto_partition_behavior.groovy +++ b/regression-test/suites/partition_p0/auto_partition/test_auto_partition_behavior.groovy @@ -217,117 +217,6 @@ suite("test_auto_partition_behavior") { - // dynamic + auto partition - sql """ admin set frontend config ('dynamic_partition_check_interval_seconds' = '1') """ - // PROHIBIT different timeunit of interval when use both auto & dynamic partition - test{ - sql """ - CREATE TABLE tbl3 - ( - k1 DATETIME NOT NULL, - col1 int - ) - auto partition by range (date_trunc(`k1`, 'year')) () - DISTRIBUTED BY HASH(k1) - PROPERTIES - ( - "replication_num" = "1", - "dynamic_partition.create_history_partition"="true", - "dynamic_partition.enable" = "true", - "dynamic_partition.time_unit" = "HOUR", - "dynamic_partition.start" = "-2", - "dynamic_partition.end" = "2", - "dynamic_partition.prefix" = "p", - "dynamic_partition.buckets" = "8" - ); - """ - exception "If support auto partition and dynamic partition at same time, they must have the same interval unit." - } - - sql " drop table if exists test_dynamic " - sql """ - create table test_dynamic( - k0 DATE not null - ) - auto partition by range (date_trunc(k0, 'year')) () - DISTRIBUTED BY HASH(`k0`) BUCKETS auto - properties("replication_num" = "1"); - """ - test { - sql """ - ALTER TABLE test_dynamic set ( - "dynamic_partition.enable" = "true", - "dynamic_partition.time_unit" = "DAY", - "dynamic_partition.end" = "3", - "dynamic_partition.prefix" = "p", - "dynamic_partition.buckets" = "32" - ); - """ - exception "If support auto partition and dynamic partition at same time, they must have the same interval unit." - } - sql """ - ALTER TABLE test_dynamic set ( - "dynamic_partition.enable" = "true", - "dynamic_partition.time_unit" = "YeAr", - "dynamic_partition.end" = "3", - "dynamic_partition.prefix" = "p", - "dynamic_partition.buckets" = "32" - ); - """ - - sql " drop table if exists auto_dynamic " - sql """ - create table auto_dynamic( - k0 datetime(6) NOT NULL - ) - auto partition by range (date_trunc(k0, 'hour')) - ( - ) - DISTRIBUTED BY HASH(`k0`) BUCKETS 2 - properties( - "dynamic_partition.enable" = "true", - "dynamic_partition.prefix" = "p", - "dynamic_partition.create_history_partition" = "true", - "dynamic_partition.start" = "-5", - "dynamic_partition.end" = "0", - "dynamic_partition.time_unit" = "hour", - "replication_num" = "1" - ); - """ - def part_result = sql " show partitions from auto_dynamic " - assertEquals(part_result.size, 6) - - sql " drop table if exists auto_dynamic " - sql """ - create table auto_dynamic( - k0 datetime(6) NOT NULL - ) - auto partition by range (date_trunc(k0, 'year')) - ( - ) - DISTRIBUTED BY HASH(`k0`) BUCKETS 2 - properties( - "dynamic_partition.enable" = "true", - "dynamic_partition.prefix" = "p", - "dynamic_partition.start" = "-50", - "dynamic_partition.end" = "0", - "dynamic_partition.time_unit" = "year", - "replication_num" = "1" - ); - """ - part_result = sql " show partitions from auto_dynamic " - assertEquals(part_result.size, 1) - sql " insert into auto_dynamic values ('2024-01-01'), ('2900-01-01'), ('1900-01-01'), ('3000-01-01'); " - sleep(3000) - part_result = sql " show partitions from auto_dynamic " - log.info("${part_result}".toString()) - assertEquals(part_result.size, 3) - qt_sql_dynamic_auto "select * from auto_dynamic order by k0;" - sql """ admin set frontend config ('dynamic_partition_check_interval_seconds' = '600') """ - - - - // prohibit too long value for partition column sql "drop table if exists `long_value`" sql """ diff --git a/regression-test/suites/point_query_p0/test_point_query.groovy b/regression-test/suites/point_query_p0/test_point_query.groovy index f27c366efbbd0b3..f05d0af1305a68e 100644 --- a/regression-test/suites/point_query_p0/test_point_query.groovy +++ b/regression-test/suites/point_query_p0/test_point_query.groovy @@ -304,4 +304,31 @@ suite("test_point_query", "nonConcurrent") { sql """set global enable_nereids_planner=true""" sql "set global enable_fallback_to_original_planner = true" } + + // test partial update/delete + sql "DROP TABLE IF EXISTS table_3821461" + sql """ + CREATE TABLE `table_3821461` ( + `col1` smallint NOT NULL, + `col2` int NOT NULL, + `loc3` char(10) NOT NULL, + `value` char(10) NOT NULL, + INDEX col3 (`loc3`) USING INVERTED, + INDEX col2 (`col2`) USING INVERTED ) + ENGINE=OLAP UNIQUE KEY(`col1`, `col2`, `loc3`) + DISTRIBUTED BY HASH(`col1`, `col2`, `loc3`) BUCKETS 1 + PROPERTIES ( "replication_allocation" = "tag.location.default: 1", "bloom_filter_columns" = "col1", "store_row_column" = "true" ); + """ + sql "insert into table_3821461 values (-10, 20, 'aabc', 'value')" + sql "insert into table_3821461 values (10, 20, 'aabc', 'value');" + sql "insert into table_3821461 values (20, 30, 'aabc', 'value');" + explain { + sql("select * from table_3821461 where col1 = -10 and col2 = 20 and loc3 = 'aabc'") + contains "SHORT-CIRCUIT" + } + qt_sql "select * from table_3821461 where col1 = 10 and col2 = 20 and loc3 = 'aabc';" + sql "delete from table_3821461 where col1 = 10 and col2 = 20 and loc3 = 'aabc';" + qt_sql "select * from table_3821461 where col1 = 10 and col2 = 20 and loc3 = 'aabc';" + sql "update table_3821461 set value = 'update value' where col1 = -10 or col1 = 20;" + qt_sql """select * from table_3821461 where col1 = -10 and col2 = 20 and loc3 = 'aabc'""" } \ No newline at end of file diff --git a/regression-test/suites/query_p0/grouping_sets/test_grouping_sets.groovy b/regression-test/suites/query_p0/grouping_sets/test_grouping_sets.groovy index 76a7476080090d0..85bfae6a13c301c 100644 --- a/regression-test/suites/query_p0/grouping_sets/test_grouping_sets.groovy +++ b/regression-test/suites/query_p0/grouping_sets/test_grouping_sets.groovy @@ -98,7 +98,7 @@ suite("test_grouping_sets", "p0") { SELECT k1, k3, MAX( k8 ) FROM test_query_db.test GROUP BY k1, GROUPING SETS ( (k1, k3), (k1), ( ) ), ROLLUP(k1, k3) """ - exception "Syntax error" + exception "mismatched input 'SETS'" } qt_select13""" diff --git a/regression-test/suites/query_p0/join/test_join2.groovy b/regression-test/suites/query_p0/join/test_join2.groovy index fb8cdb95f00a72d..9158133948f7549 100644 --- a/regression-test/suites/query_p0/join/test_join2.groovy +++ b/regression-test/suites/query_p0/join/test_join2.groovy @@ -84,7 +84,7 @@ suite("test_join2", "query,p0,arrow_flight_sql") { FROM ${TBname1} NATURAL JOIN ${TBname2} ORDER BY 1,2,3,4,5,6; """ - exception "natural join is not supported" + exception "mismatched input 'NATURAL'" } qt_join4 """ diff --git a/regression-test/suites/query_p0/sql_functions/encryption_digest/test_encryption_function.groovy b/regression-test/suites/query_p0/sql_functions/encryption_digest/test_encryption_function.groovy index 4b0fd3d497d420b..f4a67b052c37501 100644 --- a/regression-test/suites/query_p0/sql_functions/encryption_digest/test_encryption_function.groovy +++ b/regression-test/suites/query_p0/sql_functions/encryption_digest/test_encryption_function.groovy @@ -16,10 +16,9 @@ // under the License. suite("test_encryption_function") { - def tableName = "dwd_candidates" - sql "DROP TABLE IF EXISTS ${tableName}" + sql "DROP TABLE IF EXISTS dwd_candidates" sql """ - CREATE TABLE IF NOT EXISTS ${tableName} ( + CREATE TABLE IF NOT EXISTS dwd_candidates ( c_int INT, `name` varchar(65530) NULL COMMENT "" ) @@ -28,7 +27,6 @@ suite("test_encryption_function") { "replication_num" = "1" ) """ - sql "set batch_size = 4096;" sql "set block_encryption_mode=\"AES_128_ECB\";" qt_sql "SELECT TO_BASE64(AES_ENCRYPT('text','F3229A0B371ED2D9441B830D21A390C3'));" // 'wr2JEDVXzL9+2XtRhgIloA==' @@ -40,16 +38,6 @@ suite("test_encryption_function") { exception "session variable block_encryption_mode is invalid with sm4" } - sql "set block_encryption_mode=\"AES_128_ECB\";" - qt_sql "SELECT TO_BASE64(AES_ENCRYPT_v2('text','F3229A0B371ED2D9441B830D21A390C3'));" // 'wr2JEDVXzL9+2XtRhgIloA==' - qt_sql "SELECT TO_BASE64(AES_ENCRYPT_v2('text','F3229A0B371ED2D9441B830D21A390C3', '0123456789'));" // wr2JEDVXzL9+2XtRhgIloA== - qt_sql "SELECT AES_DECRYPT_v2(FROM_BASE64('wr2JEDVXzL9+2XtRhgIloA=='),'F3229A0B371ED2D9441B830D21A390C3');" // text - qt_sql "SELECT AES_DECRYPT_v2(FROM_BASE64('mvZT1KJw7N0RJf27aipUpg=='),'F3229A0B371ED2D9441B830D21A390C3', '0123456789');" // NULL - test { - sql "SELECT TO_BASE64(SM4_ENCRYPT_v2('text','F3229A0B371ED2D9441B830D21A390C3'));" - exception "session variable block_encryption_mode is invalid with sm4" - } - sql "set block_encryption_mode=\"AES_256_ECB\";" qt_sql "SELECT TO_BASE64(AES_ENCRYPT('text','F3229A0B371ED2D9441B830D21A390C3'));" // 'wr2JEDVXzL9+2XtRhgIloA==' qt_sql "SELECT TO_BASE64(AES_ENCRYPT('text','F3229A0B371ED2D9441B830D21A390C3', '0123456789'));" // 'BO2vxHeUcw5BQQalSBbo1w==' @@ -57,13 +45,6 @@ suite("test_encryption_function") { qt_sql "SELECT AES_DECRYPT(FROM_BASE64('BO2vxHeUcw5BQQalSBbo1w=='),'F3229A0B371ED2D9441B830D21A390C3');" // NULL qt_sql "SELECT AES_DECRYPT(FROM_BASE64('BO2vxHeUcw5BQQalSBbo1w=='),'F3229A0B371ED2D9441B830D21A390C3', '0123456789');" // text - sql "set block_encryption_mode=\"AES_256_ECB\";" - qt_sql "SELECT TO_BASE64(AES_ENCRYPT_v2('text','F3229A0B371ED2D9441B830D21A390C3'));" // 'BO2vxHeUcw5BQQalSBbo1w==' - qt_sql "SELECT TO_BASE64(AES_ENCRYPT_v2('text','F3229A0B371ED2D9441B830D21A390C3', '0123456789'));" // 'BO2vxHeUcw5BQQalSBbo1w==' - qt_sql "SELECT AES_DECRYPT_v2(FROM_BASE64('wr2JEDVXzL9+2XtRhgIloA=='),'F3229A0B371ED2D9441B830D21A390C3');" // NULL - qt_sql "SELECT AES_DECRYPT_v2(FROM_BASE64('BO2vxHeUcw5BQQalSBbo1w=='),'F3229A0B371ED2D9441B830D21A390C3');" // text - qt_sql "SELECT AES_DECRYPT_v2(FROM_BASE64('BO2vxHeUcw5BQQalSBbo1w=='),'F3229A0B371ED2D9441B830D21A390C3', '0123456789');" // text - sql "set block_encryption_mode=\"AES_256_CBC\";" qt_sql "SELECT TO_BASE64(AES_ENCRYPT('text','F3229A0B371ED2D9441B830D21A390C3'));" // 'wr2JEDVXzL9+2XtRhgIloA==' qt_sql "SELECT TO_BASE64(AES_ENCRYPT('text','F3229A0B371ED2D9441B830D21A390C3', '0123456789'));" // tsmK1HzbpnEdR2//WhO+MA== @@ -77,26 +58,6 @@ suite("test_encryption_function") { "GROUP BY x0;" } - sql "set block_encryption_mode=\"AES_256_CBC\";" - test { - sql "SELECT TO_BASE64(AES_ENCRYPT_v2('text','F3229A0B371ED2D9441B830D21A390C3'));" - exception "Incorrect parameter count in the call to native function 'aes_encrypt'" - } - qt_sql "SELECT TO_BASE64(AES_ENCRYPT_v2('text','F3229A0B371ED2D9441B830D21A390C3', '0123456789'));" // tsmK1HzbpnEdR2//WhO+MA== - qt_sql "SELECT TO_BASE64(AES_ENCRYPT('text','F3229A0B371ED2D9441B830D21A390C3', '0123456789ff'));" // ciacXDLHMNG7CD9Kws8png== - test { - sql "SELECT AES_DECRYPT_v2(FROM_BASE64('wr2JEDVXzL9+2XtRhgIloA=='),'F3229A0B371ED2D9441B830D21A390C3');" - exception "Incorrect parameter count in the call to native function 'aes_decrypt'" - } - qt_sql "SELECT AES_DECRYPT(FROM_BASE64('mvZT1KJw7N0RJf27aipUpg=='),'F3229A0B371ED2D9441B830D21A390C3', '0123456789');" // NULL - qt_sql "SELECT AES_DECRYPT_v2(FROM_BASE64('mvZT1KJw7N0RJf27aipUpg=='),'F3229A0B371ED2D9441B830D21A390C3', '0123456789');" // NULL - qt_sql "SELECT AES_DECRYPT_v2(FROM_BASE64('tsmK1HzbpnEdR2//WhO+MA=='),'F3229A0B371ED2D9441B830D21A390C3', '0123456789');" // text - qt_sql "SELECT AES_DECRYPT_v2(FROM_BASE64('ciacXDLHMNG7CD9Kws8png=='),'F3229A0B371ED2D9441B830D21A390C3', '0123456789ff');" // text - explain { - sql "SELECT AES_DECRYPT_v2(UNHEX(r_2_3.`name`), 'namePnhe3E0MWyfZivUnVzDy12caymnrKp', '0123456789') AS x0 FROM dwd_candidates AS r_2_3\n" + - "GROUP BY x0;" - } - sql "set block_encryption_mode=\"SM4_128_CBC\";" qt_sql "SELECT TO_BASE64(SM4_ENCRYPT('text','F3229A0B371ED2D9441B830D21A390C3'));" // aDjwRflBrDjhBZIOFNw3Tg== qt_sql "SELECT TO_BASE64(SM4_ENCRYPT('text','F3229A0B371ED2D9441B830D21A390C3', '0123456789'));" // 1Y4NGIukSbv9OrkZnRD1bQ== @@ -110,25 +71,6 @@ suite("test_encryption_function") { exception "session variable block_encryption_mode is invalid with aes" } - sql "set block_encryption_mode=\"SM4_128_CBC\";" - test { - sql "SELECT TO_BASE64(SM4_ENCRYPT_v2('text','F3229A0B371ED2D9441B830D21A390C3'));" - exception "Incorrect parameter count in the call to native function 'sm4_encrypt'" - } - qt_sql "SELECT TO_BASE64(SM4_ENCRYPT_v2('text','F3229A0B371ED2D9441B830D21A390C3', '0123456789'));" // 1Y4NGIukSbv9OrkZnRD1bQ== - qt_sql "SELECT TO_BASE64(SM4_ENCRYPT_v2('text','F3229A0B371ED2D9441B830D21A390C3', '0123456789ff'));" // G5POcFAJwiZHeTtN6DjInQ== - test { - sql "SELECT SM4_DECRYPT_v2(FROM_BASE64('aDjwRflBrDjhBZIOFNw3Tg=='),'F3229A0B371ED2D9441B830D21A390C3');" - exception "Incorrect parameter count in the call to native function 'sm4_decrypt'" - } - qt_sql "SELECT SM4_DECRYPT_v2(FROM_BASE64('1Y4NGIukSbv9OrkZnRD1bQ=='),'F3229A0B371ED2D9441B830D21A390C3', '0123456789');" // text - qt_sql "SELECT SM4_DECRYPT_v2(FROM_BASE64('G5POcFAJwiZHeTtN6DjInQ=='),'F3229A0B371ED2D9441B830D21A390C3', '0123456789');" // NULL - qt_sql "SELECT SM4_DECRYPT_v2(FROM_BASE64('G5POcFAJwiZHeTtN6DjInQ=='),'F3229A0B371ED2D9441B830D21A390C3', '0123456789ff');" // text - test { - sql "SELECT TO_BASE64(AES_ENCRYPT_v2('text','F3229A0B371ED2D9441B830D21A390C3'));" - exception "session variable block_encryption_mode is invalid with aes" - } - qt_sql "SELECT SM3(\"abc\");" qt_sql "select sm3(\"abcd\");" qt_sql "select sm3sum(\"ab\",\"cd\");" diff --git a/regression-test/suites/query_p0/sql_functions/math_functions/test_math_unary_always_nullable.groovy b/regression-test/suites/query_p0/sql_functions/math_functions/test_math_unary_always_nullable.groovy new file mode 100644 index 000000000000000..282d4e3c5754e4c --- /dev/null +++ b/regression-test/suites/query_p0/sql_functions/math_functions/test_math_unary_always_nullable.groovy @@ -0,0 +1,85 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_math_unary_alway_nullable") { + sql """ + set debug_skip_fold_constant=true; + """ + + qt_acos_1 """ + select acos(1.1), acos(1.1) is null; + """ + qt_acos_2 """ + select acos(-1.1), acos(-1.1) is null; + """ + qt_acos_3 """ + select acos(-1.1), acos(-1.1) is NULL, number from numbers("number"="10") + """ + + qt_asin_1 """ + select asin(1.1), asin(1.1) is null; + """ + qt_asin_2 """ + select asin(-1.1), asin(-1.1) is null; + """ + qt_asin_3 """ + select asin(-1.1), asin(-1.1) is NULL, number from numbers("number"="10") + """ + + qt_sqrt_1 """ + select sqrt(-1), sqrt(-1) is null; + """ + qt_sqrt_2 """ + select sqrt(-1.1), sqrt(-1.1) is null; + """ + qt_sqrt_3 """ + select sqrt(-1.1), sqrt(-1.1) is NULL, number from numbers("number"="10") + """ + + sql "drop table if exists test_math_unary_alway_nullable" + + sql """ + create table if not exists test_math_unary_alway_nullable (rowid int, val double NULL) + distributed by hash(rowid) properties ("replication_num"="1"); + """ + + sql """ + insert into test_math_unary_alway_nullable values + (1, 1.1), (2, -1.1), (3, 0), (4, NULL) + """ + sql """ + insert into test_math_unary_alway_nullable values + (5, NULL), (6, NULL), (7, NULL), (8, NULL) + """ + + qt_acos_tbl_1 """ + select rowid, acos(val), acos(val) is null from test_math_unary_alway_nullable order by rowid; + """ + + qt_asin_tbl_1 """ + select rowid, asin(val), asin(val) is null from test_math_unary_alway_nullable order by rowid; + """ + + qt_sqrt_tbl_1 """ + select rowid, sqrt(val), sqrt(val) is null from test_math_unary_alway_nullable order by rowid; + """ + + qt_dsqrt_tbl_1 """ + select rowid, dsqrt(val), dsqrt(val) is null from test_math_unary_alway_nullable order by rowid; + """ + +} \ No newline at end of file diff --git a/regression-test/suites/query_profile/adaptive_pipeline_task_serial_read_on_limit.groovy b/regression-test/suites/query_profile/adaptive_pipeline_task_serial_read_on_limit.groovy new file mode 100644 index 000000000000000..15481fe7c9a8dc0 --- /dev/null +++ b/regression-test/suites/query_profile/adaptive_pipeline_task_serial_read_on_limit.groovy @@ -0,0 +1,175 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import groovy.json.JsonOutput +import groovy.json.JsonSlurper +import groovy.json.StringEscapeUtils + + +def getProfileList = { + def dst = 'http://' + context.config.feHttpAddress + def conn = new URL(dst + "/rest/v1/query_profile").openConnection() + conn.setRequestMethod("GET") + def encoding = Base64.getEncoder().encodeToString((context.config.feHttpUser + ":" + + (context.config.feHttpPassword == null ? "" : context.config.feHttpPassword)).getBytes("UTF-8")) + conn.setRequestProperty("Authorization", "Basic ${encoding}") + return conn.getInputStream().getText() +} + + +def getProfile = { id -> + def dst = 'http://' + context.config.feHttpAddress + def conn = new URL(dst + "/api/profile/text/?query_id=$id").openConnection() + conn.setRequestMethod("GET") + def encoding = Base64.getEncoder().encodeToString((context.config.feHttpUser + ":" + + (context.config.feHttpPassword == null ? "" : context.config.feHttpPassword)).getBytes("UTF-8")) + conn.setRequestProperty("Authorization", "Basic ${encoding}") + return conn.getInputStream().getText() +} + +suite('adaptive_pipeline_task_serial_read_on_limit') { + sql """ + DROP TABLE IF EXISTS adaptive_pipeline_task_serial_read_on_limit; + """ + sql """ + CREATE TABLE if not exists `adaptive_pipeline_task_serial_read_on_limit` ( + `id` INT, + `name` varchar(32) + ) ENGINE=OLAP + DISTRIBUTED BY HASH(`id`) BUCKETS 10 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + ); + """ + + // Insert data to table + sql """ + insert into adaptive_pipeline_task_serial_read_on_limit values + (1, "A"),(2, "B"),(3, "C"),(4, "D"),(5,"E"),(6,"F"),(7,"G"),(8,"H"),(9,"K"); + """ + sql """ + insert into adaptive_pipeline_task_serial_read_on_limit values + (10, "A"),(20, "B"),(30, "C"),(40, "D"),(50,"E"),(60,"F"),(70,"G"),(80,"H"),(90,"K"); + """ + sql """ + insert into adaptive_pipeline_task_serial_read_on_limit values + (101, "A"),(201, "B"),(301, "C"),(401, "D"),(501,"E"),(601,"F"),(701,"G"),(801,"H"),(901,"K"); + """ + sql """ + insert into adaptive_pipeline_task_serial_read_on_limit values + (1010, "A"),(2010, "B"),(3010, "C"),(4010, "D"),(5010,"E"),(6010,"F"),(7010,"G"),(8010,"H"),(9010,"K"); + """ + + def uuidString = UUID.randomUUID().toString() + sql "set enable_profile=true" + // set parallel_pipeline_task_num to 1 so that only one scan node, + // and we can check MaxScannerThreadNum in profile. + sql "set parallel_pipeline_task_num=1;" + // no limit, MaxScannerThreadNum = TabletNum + sql """ + select "no_limit_1_${uuidString}", * from adaptive_pipeline_task_serial_read_on_limit; + """ + sql "set parallel_pipeline_task_num=0;" + // With Limit, MaxScannerThreadNum = 1 + sql """ + select "with_limit_1_${uuidString}", * from adaptive_pipeline_task_serial_read_on_limit limit 10000; + """ + // With Limit, but bigger then adaptive_pipeline_task_serial_read_on_limit, MaxScannerThreadNum = TabletNum + sql """ + select "with_limit_2_${uuidString}", * from adaptive_pipeline_task_serial_read_on_limit limit 10001; + """ + sql """ + set enable_adaptive_pipeline_task_serial_read_on_limit=false; + """ + sql "set parallel_pipeline_task_num=1;" + // Forbid the strategy, with limit, MaxScannerThreadNum = TabletNum + sql """ + select "not_enable_limit_${uuidString}", * from adaptive_pipeline_task_serial_read_on_limit limit 100; + """ + + sql "set parallel_pipeline_task_num=0;" + + // Enable the strategy, with limit 20, MaxScannerThreadNum = 1 + sql """ + set enable_adaptive_pipeline_task_serial_read_on_limit=true; + """ + sql """ + set adaptive_pipeline_task_serial_read_on_limit=10; + """ + sql """ + select "modify_to_20_${uuidString}", * from adaptive_pipeline_task_serial_read_on_limit limit 15; + """ + + sql "set enable_profile=false" + + def wholeString = getProfileList() + List profileData = new JsonSlurper().parseText(wholeString).data.rows + String queryIdNoLimit1 = ""; + String queryIdWithLimit1 = ""; + String queryIdWithLimit2 = ""; + String queryIDNotEnableLimit = ""; + String queryIdModifyTo20 = ""; + + logger.info("{}", uuidString) + + for (def profileItem in profileData) { + if (profileItem["Sql Statement"].toString().contains("no_limit_1_${uuidString}")) { + queryIdNoLimit1 = profileItem["Profile ID"].toString() + logger.info("profileItem: {}", profileItem) + } + if (profileItem["Sql Statement"].toString().contains("with_limit_1_${uuidString}")) { + queryIdWithLimit1 = profileItem["Profile ID"].toString() + logger.info("profileItem: {}", profileItem) + } + if (profileItem["Sql Statement"].toString().contains("with_limit_2_${uuidString}")) { + queryIdWithLimit2 = profileItem["Profile ID"].toString() + logger.info("profileItem: {}", profileItem) + } + if (profileItem["Sql Statement"].toString().contains("not_enable_limit_${uuidString}")) { + queryIDNotEnableLimit = profileItem["Profile ID"].toString() + logger.info("profileItem: {}", profileItem) + } + if (profileItem["Sql Statement"].toString().contains("modify_to_20_${uuidString}")) { + queryIdModifyTo20 = profileItem["Profile ID"].toString() + logger.info("profileItem: {}", profileItem) + } + } + + logger.info("queryIdNoLimit1_${uuidString}: {}", queryIdNoLimit1) + logger.info("queryIdWithLimit1_${uuidString}: {}", queryIdWithLimit1) + logger.info("queryIdWithLimit2_${uuidString}: {}", queryIdWithLimit2) + logger.info("queryIDNotEnableLimit_${uuidString}: {}", queryIDNotEnableLimit) + logger.info("queryIdModifyTo20_${uuidString}: {}", queryIdModifyTo20) + + assertTrue(queryIdNoLimit1 != "") + assertTrue(queryIdWithLimit1 != "") + assertTrue(queryIdWithLimit2 != "") + assertTrue(queryIDNotEnableLimit != "") + assertTrue(queryIdModifyTo20 != "") + + def String profileNoLimit1 = getProfile(queryIdNoLimit1).toString() + def String profileWithLimit1 = getProfile(queryIdWithLimit1).toString() + def String profileWithLimit2 = getProfile(queryIdWithLimit2).toString() + def String profileNotEnableLimit = getProfile(queryIDNotEnableLimit).toString() + def String profileModifyTo20 = getProfile(queryIdModifyTo20).toString() + + assertTrue(profileNoLimit1.contains("- MaxScannerThreadNum: 10")) + assertTrue(profileWithLimit1.contains("- MaxScannerThreadNum: 1")) + assertTrue(profileWithLimit2.contains("- MaxScannerThreadNum: 10")) + assertTrue(profileNotEnableLimit.contains("- MaxScannerThreadNum: 10")) + assertTrue(profileModifyTo20.contains("- MaxScannerThreadNum: 1")) +} \ No newline at end of file diff --git a/regression-test/suites/statistics/test_auto_analyze_black_white_list.groovy b/regression-test/suites/statistics/test_auto_analyze_black_white_list.groovy new file mode 100644 index 000000000000000..8ba453b81f505eb --- /dev/null +++ b/regression-test/suites/statistics/test_auto_analyze_black_white_list.groovy @@ -0,0 +1,75 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_auto_analyze_black_white_list") { + + sql """drop database if exists test_auto_analyze_black_white_list""" + sql """create database test_auto_analyze_black_white_list""" + sql """use test_auto_analyze_black_white_list""" + sql """set global force_sample_analyze=false""" + sql """set global enable_auto_analyze=false""" + + sql """CREATE TABLE test_bw ( + key1 bigint NOT NULL, + key2 bigint NOT NULL, + value1 int NOT NULL, + value2 int NOT NULL, + value3 int NOT NULL + )ENGINE=OLAP + DUPLICATE KEY(`key1`, `key2`) + COMMENT "OLAP" + DISTRIBUTED BY HASH(`key1`) BUCKETS 2 + PROPERTIES ( + "replication_num" = "1" + ) + """ + + // Test show index row count + def result = sql """show table stats test_bw""" + assertEquals(1, result.size()) + assertEquals("true", result[0][8]) + + sql """ALTER TABLE test_bw SET ("auto_analyze_policy" = "disable");""" + result = sql """show table stats test_bw""" + assertEquals(1, result.size()) + assertEquals("false", result[0][8]) + + sql """analyze table test_bw PROPERTIES("use.auto.analyzer"="true")""" + result = sql """show auto analyze test_bw""" + assertEquals(0, result.size()) + + sql """ALTER TABLE test_bw SET ("auto_analyze_policy" = "enable");""" + result = sql """show table stats test_bw""" + assertEquals(1, result.size()) + assertEquals("true", result[0][8]) + + sql """analyze table test_bw PROPERTIES("use.auto.analyzer"="true")""" + result = sql """show auto analyze test_bw""" + assertEquals(1, result.size()) + + sql """ALTER TABLE test_bw SET ("auto_analyze_policy" = "disable");""" + result = sql """show table stats test_bw""" + assertEquals(1, result.size()) + assertEquals("false", result[0][8]) + sql """ALTER TABLE test_bw SET ("auto_analyze_policy" = "base_on_catalog");""" + result = sql """show table stats test_bw""" + assertEquals(1, result.size()) + assertEquals("true", result[0][8]) + + sql """drop database if exists test_auto_analyze_black_white_list""" +} + diff --git a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_delete.groovy b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_delete.groovy index 0d83d94f91c9187..fc59038122a1d29 100644 --- a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_delete.groovy +++ b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_delete.groovy @@ -20,96 +20,210 @@ suite('test_partial_update_delete') { String db = context.config.getDbNameByFile(context.file) sql "select 1;" // to create database - for (def use_row_store : [false, true]) { - logger.info("current params: use_row_store: ${use_row_store}") - - connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl) { - sql "use ${db};" - - def tableName1 = "test_partial_update_delete1" - sql "DROP TABLE IF EXISTS ${tableName1};" - sql """ CREATE TABLE IF NOT EXISTS ${tableName1} ( - `k1` int NOT NULL, - `c1` int, - `c2` int, - `c3` int NOT NULL, - `c4` int - )UNIQUE KEY(k1) - DISTRIBUTED BY HASH(k1) BUCKETS 1 - PROPERTIES ( - "enable_unique_key_merge_on_write" = "true", - "disable_auto_compaction" = "true", - "replication_num" = "1", - "store_row_column" = "${use_row_store}"); """ - - def tableName2 = "test_partial_update_delete2" - sql "DROP TABLE IF EXISTS ${tableName2};" - sql """ CREATE TABLE IF NOT EXISTS ${tableName2} ( - `k` BIGINT NULL - ) UNIQUE KEY(k) - DISTRIBUTED BY HASH(k) BUCKETS 1 - PROPERTIES ( - "enable_unique_key_merge_on_write" = "true", - "disable_auto_compaction" = "true", - "replication_num" = "1", - "store_row_column" = "${use_row_store}"); """ - - sql "insert into ${tableName1} values(1,1,1,1,1),(2,2,2,2,2),(3,3,3,3,3),(4,4,4,4,4),(5,5,5,5,5);" - qt_sql "select * from ${tableName1} order by k1;" - sql "insert into ${tableName2} values(1),(3);" - sql "delete from ${tableName1} A using ${tableName2} B where A.k1=B.k;" - qt_sql "select * from ${tableName1} order by k1;" - - sql "delete from ${tableName1} where c2=2;" - qt_sql "select * from ${tableName1} order by k1;" - - sql "set skip_delete_sign=true;" - sql "set skip_storage_engine_merge=true;" - sql "set skip_delete_bitmap=true;" - qt_with_delete_sign "select k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__ from ${tableName1} order by k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__;" - sql "drop table if exists ${tableName1};" - sql "drop table if exists ${tableName2};" - - sql "set skip_delete_sign=false;" - sql "set skip_storage_engine_merge=false;" - sql "set skip_delete_bitmap=false;" - def tableName3 = "test_partial_update_delete3" - sql "DROP TABLE IF EXISTS ${tableName3};" - sql """ CREATE TABLE IF NOT EXISTS ${tableName3} ( - `k1` int NOT NULL, - `c1` int, - `c2` int, - `c3` int, - `c4` int - )UNIQUE KEY(k1) - DISTRIBUTED BY HASH(k1) BUCKETS 1 - PROPERTIES ( - "enable_unique_key_merge_on_write" = "true", - "disable_auto_compaction" = "true", - "replication_num" = "1", - "store_row_column" = "${use_row_store}"); """ - - sql "insert into ${tableName3} values(1,1,1,1,1),(2,2,2,2,2),(3,3,3,3,3),(4,4,4,4,4),(5,5,5,5,5);" - qt_sql "select k1,c1,c2,c3,c4 from ${tableName3} order by k1,c1,c2,c3,c4;" - streamLoad { - table "${tableName3}" - - set 'column_separator', ',' - set 'format', 'csv' - set 'columns', 'k1' - set 'partial_columns', 'true' - set 'merge_type', 'DELETE' - - file 'partial_update_delete.csv' - time 10000 + for (def use_nereids : [true, false]) { + for (def use_row_store : [false, true]) { + logger.info("current params: use_nereids: ${use_nereids}, use_row_store: ${use_row_store}") + connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl) { + sql "use ${db};" + if (use_nereids) { + sql "set enable_nereids_planner=true" + sql "set enable_fallback_to_original_planner=false" + } else { + sql "set enable_nereids_planner=false" + } + sql "sync;" + + def tableName1 = "test_partial_update_delete1" + sql "DROP TABLE IF EXISTS ${tableName1};" + sql """ CREATE TABLE IF NOT EXISTS ${tableName1} ( + `k1` int NOT NULL, + `c1` int, + `c2` int, + `c3` int NOT NULL, + `c4` int + )UNIQUE KEY(k1) + DISTRIBUTED BY HASH(k1) BUCKETS 1 + PROPERTIES ( + "enable_unique_key_merge_on_write" = "true", + "disable_auto_compaction" = "true", + "replication_num" = "1", + "store_row_column" = "${use_row_store}"); """ + + + def tableName2 = "test_partial_update_delete2" + sql "DROP TABLE IF EXISTS ${tableName2};" + sql """ CREATE TABLE IF NOT EXISTS ${tableName2} ( + `k` BIGINT NULL + ) UNIQUE KEY(k) + DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES ( + "enable_unique_key_merge_on_write" = "true", + "disable_auto_compaction" = "true", + "replication_num" = "1", + "store_row_column" = "${use_row_store}"); """ + + sql "insert into ${tableName1} values(1,1,1,1,1),(2,2,2,2,2),(3,3,3,3,3),(4,4,4,4,4),(5,5,5,5,5);" + qt_sql1 "select * from ${tableName1} order by k1;" + sql "insert into ${tableName2} values(1),(3);" + sql "delete from ${tableName1} A using ${tableName2} B where A.k1=B.k;" + qt_sql1 "select * from ${tableName1} order by k1;" + + sql "delete from ${tableName1} where c2=2;" + qt_sql1 "select * from ${tableName1} order by k1;" + + sql "set skip_delete_sign=true;" + sql "set skip_storage_engine_merge=true;" + sql "set skip_delete_bitmap=true;" + qt_with_delete_sign1 "select k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__ from ${tableName1} order by k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__;" + sql "drop table if exists ${tableName1};" + sql "drop table if exists ${tableName2};" + + + sql "set skip_delete_sign=false;" + sql "set skip_storage_engine_merge=false;" + sql "set skip_delete_bitmap=false;" + def tableName3 = "test_partial_update_delete3" + sql "DROP TABLE IF EXISTS ${tableName3};" + sql """ CREATE TABLE IF NOT EXISTS ${tableName3} ( + `k1` int NOT NULL, + `c1` int, + `c2` int, + `c3` int NOT NULL, + `c4` int + )UNIQUE KEY(k1) + DISTRIBUTED BY HASH(k1) BUCKETS 1 + PROPERTIES ( + "enable_unique_key_merge_on_write" = "true", + "disable_auto_compaction" = "true", + "replication_num" = "1", + "store_row_column" = "${use_row_store}"); """ + + sql "insert into ${tableName3} values(1,1,1,1,1),(2,2,2,2,2),(3,3,3,3,3),(4,4,4,4,4),(5,5,5,5,5),(6,6,6,6,6);" + qt_sql2 "select k1,c1,c2,c3,c4 from ${tableName3} order by k1,c1,c2,c3,c4;" + streamLoad { + table "${tableName3}" + set 'column_separator', ',' + set 'format', 'csv' + set 'columns', 'k1' + set 'partial_columns', 'true' + set 'merge_type', 'DELETE' + file 'partial_update_delete.csv' + time 10000 + } + sql "sync" + qt_sql2 "select k1,c1,c2,c3,c4 from ${tableName3} order by k1,c1,c2,c3,c4;" + + sql "set enable_insert_strict=false;" + sql "set enable_unique_key_partial_update=true;" + sql "sync;" + sql "insert into ${tableName3}(k1, __DORIS_DELETE_SIGN__) values(8,1),(4,1),(9,1);" + qt_sql2 "select k1,c1,c2,c3,c4 from ${tableName3} order by k1,c1,c2,c3,c4;" + sql "set enable_insert_strict=true;" + sql "set enable_unique_key_partial_update=false;" + sql "sync;" + + sql "set skip_delete_sign=true;" + sql "set skip_storage_engine_merge=true;" + sql "set skip_delete_bitmap=true;" + qt_sql2 "select k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__ from ${tableName3} order by k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__;" + sql "drop table if exists ${tableName3};" + + + sql "set skip_delete_sign=false;" + sql "set skip_storage_engine_merge=false;" + sql "set skip_delete_bitmap=false;" + def tableName4 = "test_partial_update_delete4" + sql "DROP TABLE IF EXISTS ${tableName4};" + sql """ CREATE TABLE IF NOT EXISTS ${tableName4} ( + `k1` int NOT NULL, + `c1` int, + `c2` int, + `c3` int NOT NULL, + `c4` int + )UNIQUE KEY(k1) + DISTRIBUTED BY HASH(k1) BUCKETS 1 + PROPERTIES ( + "enable_unique_key_merge_on_write" = "true", + "disable_auto_compaction" = "true", + "replication_num" = "1", + "store_row_column" = "${use_row_store}", + "function_column.sequence_col" = "c3"); """ + + sql "insert into ${tableName4} values(1,1,1,1,1),(2,2,2,2,2),(3,3,3,3,3),(4,4,4,4,4),(5,5,5,5,5),(6,6,6,6,6);" + qt_sql3 "select k1,c1,c2,c3,c4 from ${tableName4} order by k1,c1,c2,c3,c4;" + // if the table has sequence map col, can not set sequence map col when merge_type=delete + streamLoad { + table "${tableName4}" + set 'column_separator', ',' + set 'format', 'csv' + set 'columns', 'k1' + set 'partial_columns', 'true' + set 'merge_type', 'DELETE' + file 'partial_update_delete.csv' + time 10000 + } + sql "sync" + qt_sql3 "select k1,c1,c2,c3,c4 from ${tableName4} order by k1,c1,c2,c3,c4;" + + sql "set enable_insert_strict=false;" + sql "set enable_unique_key_partial_update=true;" + sql "sync;" + sql "insert into ${tableName4}(k1, __DORIS_DELETE_SIGN__) values(8,1),(4,1),(9,1);" + qt_sql3 "select k1,c1,c2,c3,c4 from ${tableName4} order by k1,c1,c2,c3,c4;" + sql "set enable_insert_strict=true;" + sql "set enable_unique_key_partial_update=false;" + sql "sync;" + + sql "set skip_delete_sign=true;" + sql "set skip_storage_engine_merge=true;" + sql "set skip_delete_bitmap=true;" + qt_sql3 "select k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__,__DORIS_SEQUENCE_COL__ from ${tableName4} order by k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__;" + sql "drop table if exists ${tableName4};" + + + sql "set skip_delete_sign=false;" + sql "set skip_storage_engine_merge=false;" + sql "set skip_delete_bitmap=false;" + def tableName5 = "test_partial_update_delete5" + sql "DROP TABLE IF EXISTS ${tableName5};" + sql """ CREATE TABLE IF NOT EXISTS ${tableName5} ( + `k1` int NOT NULL, + `c1` int, + `c2` int, + `c3` int NOT NULL, + `c4` int + )UNIQUE KEY(k1) + DISTRIBUTED BY HASH(k1) BUCKETS 1 + PROPERTIES ( + "enable_unique_key_merge_on_write" = "true", + "disable_auto_compaction" = "true", + "replication_num" = "1", + "store_row_column" = "${use_row_store}", + "function_column.sequence_type" = "int"); """ + sql "insert into ${tableName5}(k1,c1,c2,c3,c4,__DORIS_SEQUENCE_COL__) values(1,1,1,1,1,1),(2,2,2,2,2,2),(3,3,3,3,3,3),(4,4,4,4,4,4),(5,5,5,5,5,5),(6,6,6,6,6,6);" + qt_sql4 "select k1,c1,c2,c3,c4 from ${tableName5} order by k1,c1,c2,c3,c4;" + // if the table has sequence type col, users must set sequence col even if merge_type=delete + streamLoad { + table "${tableName5}" + set 'column_separator', ',' + set 'format', 'csv' + set 'columns', 'k1' + set 'partial_columns', 'true' + set 'merge_type', 'DELETE' + file 'partial_update_delete.csv' + time 10000 + check { result, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + log.info("Stream load result: ${result}".toString()) + def json = parseJson(result) + assertEquals("fail", json.Status.toLowerCase()) + assertTrue(json.Message.contains('need to specify the sequence column')) + } + } + sql "drop table if exists ${tableName5};" } - sql "sync" - qt_sql "select k1,c1,c2,c3,c4 from ${tableName3} order by k1,c1,c2,c3,c4;" - sql "set skip_delete_sign=true;" - sql "set skip_storage_engine_merge=true;" - sql "set skip_delete_bitmap=true;" - qt_sql "select k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__ from ${tableName3} order by k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__;" - sql "drop table if exists ${tableName3};" } } } diff --git a/regression-test/suites/variant_p0/load.groovy b/regression-test/suites/variant_p0/load.groovy index d74155b182ddfab..967ff63f0baa945 100644 --- a/regression-test/suites/variant_p0/load.groovy +++ b/regression-test/suites/variant_p0/load.groovy @@ -290,7 +290,7 @@ suite("regression_test_variant", "p0"){ sql """insert into ${table_name} values (5, '{"i" : 1}'), (1, '{"a" : 1}')""" sql """insert into ${table_name} values (6, '{"j" : 1}'), (1, '{"a" : 1}')""" sql """insert into ${table_name} values (6, '{"k" : 1}'), (1, '{"a" : 1}')""" - sql "select * from ${table_name}" + sql "select /*+SET_VAR(batch_size=4064,broker_load_batch_size=16352,disable_streaming_preaggregations=false,enable_distinct_streaming_aggregation=true,parallel_fragment_exec_instance_num=1,parallel_pipeline_task_num=4,profile_level=1,enable_pipeline_engine=true,enable_parallel_scan=true,parallel_scan_max_scanners_count=16,parallel_scan_min_rows_per_scanner=128,enable_fold_constant_by_be=true,enable_rewrite_element_at_to_slot=true,runtime_filter_type=2,enable_parallel_result_sink=false,sort_phase_num=0,enable_nereids_planner=true,rewrite_or_to_in_predicate_threshold=2,enable_function_pushdown=true,enable_common_expr_pushdown=true,enable_local_exchange=true,partitioned_hash_join_rows_threshold=1048576,partitioned_hash_agg_rows_threshold=8,partition_pruning_expand_threshold=10,enable_share_hash_table_for_broadcast_join=true,enable_two_phase_read_opt=true,enable_common_expr_pushdown_for_inverted_index=false,enable_delete_sub_predicate_v2=true,min_revocable_mem=33554432,fetch_remote_schema_timeout_seconds=120,max_fetch_remote_schema_tablet_count=512,enable_join_spill=false,enable_sort_spill=false,enable_agg_spill=false,enable_force_spill=false,data_queue_max_blocks=1,spill_streaming_agg_mem_limit=268435456,external_agg_partition_bits=5) */ * from ${table_name}" qt_sql_36_1 "select cast(v['a'] as int), cast(v['b'] as int), cast(v['c'] as int) from ${table_name} order by k limit 10" sql "DELETE FROM ${table_name} WHERE k=1" sql "select * from ${table_name}" diff --git a/regression-test/suites/vaults/privilege_restart.groovy b/regression-test/suites/vaults/privilege_restart.groovy new file mode 100644 index 000000000000000..4e8c8fcc04dade5 --- /dev/null +++ b/regression-test/suites/vaults/privilege_restart.groovy @@ -0,0 +1,178 @@ + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import java.util.stream.Collectors; + +// This test suite is intent to test the granted privilege for specific user will +// not disappear +suite("test_privilege_vault_restart", "nonConcurrent") { + if (!enableStoragevault()) { + logger.info("skip test_privilege_vault_restart case") + return + } + + // user1 will be kept before and after running this test in order to check + // the granted vault privilege is persisted well eventhough FE restarts many times + def user1 = "test_privilege_vault_restart_user1" + def passwd = "Cloud12345" + + def vault1 = "test_privilege_vault_restart_vault1" + // this vaule is derived from current file location: regression-test/vaults + def db = "regression_test_vaults" + def table1 = "test_privilege_vault_restart_t1" + def table2 = "test_privilege_vault_restart_t2" + def hdfsLinkWeDontReallyCare = "127.0.0.1:10086" // a dummy link, it doesn't need to work + + //========================================================================== + // prepare the basic vault and tables for further check + //========================================================================== + sql """ + CREATE STORAGE VAULT IF NOT EXISTS ${vault1} + PROPERTIES ( + "type"="hdfs", + "fs.defaultFS"="${hdfsLinkWeDontReallyCare}", + "path_prefix" = "test_vault_privilege_restart" + ); + """ + + def storageVaults = (sql " SHOW STORAGE VAULT; ").stream().map(row -> row[0]).collect(Collectors.toSet()) + logger.info("all vaults: ${storageVaults}") + org.junit.Assert.assertTrue("${vault1} is not present after creating, all vaults: ${storageVaults}", storageVaults.contains(vault1)) + + def allTables = (sql " SHOW tables").stream().map(row -> row[0]).collect(Collectors.toSet()) + logger.info("all tables ${allTables}") + + // table1 is the sign to check if the user1 has been created and granted well + def targetTableExist = allTables.contains(table1) + + if (targetTableExist) { + // the grant procedure at least run once before, user1 has been granted vault1 + logger.info("${user1} has been granted with usage_priv to ${vault1} before") + } else { + logger.info("this is the frist run, or there was a crash during the very first run, ${user1} has not been granted with usage_priv to ${vault1} before") + // create user and grant storage vault and create a table with that vault + sql """drop user if exists ${user1}""" + sql """create user ${user1} identified by '${passwd}'""" + sql """ + GRANT usage_priv ON storage vault ${vault1} TO '${user1}'; + """ + sql """ + GRANT create_priv ON *.*.* TO '${user1}'; + """ + + // ATTN: create table1, if successful, the sign has been set + // there wont be any execuse that user1 misses the privilege to vault1 from now on + sql """ + CREATE TABLE IF NOT EXISTS ${table1} ( + C_CUSTKEY INTEGER NOT NULL, + C_NAME INTEGER NOT NULL + ) + DUPLICATE KEY(C_CUSTKEY, C_NAME) + DISTRIBUTED BY HASH(C_CUSTKEY) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "storage_vault_name" = ${vault1} + ) + """ + } + + //========================================================================== + // check the prepared users and tables + //========================================================================== + def allUsers = (sql " SHOW all grants ").stream().map(row -> row[0]).collect(Collectors.toSet()) + logger.info("all users: ${allUsers}") + def userPresent = !(allUsers.stream().filter(i -> i.contains(user1)).collect(Collectors.toSet()).isEmpty()) + org.junit.Assert.assertTrue("${user1} is not in the priv table ${allUsers}", userPresent) + + allTables = (sql " SHOW tables").stream().map(row -> row[0]).collect(Collectors.toSet()) + logger.info("all tables: ${allTables}") + org.junit.Assert.assertTrue("${table1} is not present, all tables: ${allUsers}", allTables.contains(table1)) + + // Test user privilege, the newly created user cannot create or set default vault + // Only users with admin role can create storage vault + connect(user = user1, password = passwd, url = context.config.jdbcUrl) { + sql """use ${db}""" + expectExceptionLike({ + sql """ + CREATE STORAGE VAULT IF NOT EXISTS ${vault1} + PROPERTIES ( + "type"="hdfs", + "fs.defaultFS"="${hdfsLinkWeDontReallyCare}", + "path_prefix" = "test_vault_privilege" + ); + """ + }, "denied") + } + // Only users with admin role can set/unset default storage vault + connect(user = user1, password = passwd, url = context.config.jdbcUrl) { + sql """use ${db}""" + expectExceptionLike({ + sql """ + SET ${vault1} AS DEFAULT STORAGE VAULT + """ + }, "denied") + } + connect(user = user1, password = passwd, url = context.config.jdbcUrl) { + sql """use ${db}""" + expectExceptionLike({ + sql """ + UNSET DEFAULT STORAGE VAULT + """ + }, "denied") + } + + // user1 should see vault1 + def result = connect(user = user1, password = passwd, url = context.config.jdbcUrl) { + sql """use ${db}""" + sql " SHOW STORAGE VAULT; " + } + storageVaults = result.stream().map(row -> row[0]).collect(Collectors.toSet()) + org.junit.Assert.assertTrue("${user1} cannot see granted vault ${vault1} in result ${result}", storageVaults.contains(vault1)) + + + //========================================================================== + // to test that user1 has the privilege of vault1 to create new tables + // this is the main test for granted vault privilege after restarting FE + //========================================================================== + sql """ + DROP TABLE IF EXISTS ${table2} force; + """ + connect(user = user1, password = passwd, url = context.config.jdbcUrl) { + sql """use ${db}""" + sql """ + CREATE TABLE ${table2} ( + C_CUSTKEY INTEGER NOT NULL, + C_NAME INTEGER NOT NULL + ) + DUPLICATE KEY(C_CUSTKEY, C_NAME) + DISTRIBUTED BY HASH(C_CUSTKEY) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "storage_vault_name" = ${vault1} + ) + """ + } + + result = connect(user = user1, password = passwd, url = context.config.jdbcUrl) { + sql """use ${db}""" + sql " SHOW create table ${table2}; " + } + logger.info("show create table ${table2}, result ${result}") + org.junit.Assert.assertTrue("missing storage vault properties ${vault1} in table ${table2}", result.toString().contains(vault1)) + +} diff --git a/samples/datalake/deltalake_and_kudu/README.md b/samples/datalake/deltalake_and_kudu/README.md new file mode 100644 index 000000000000000..382a20ee0e4a2d3 --- /dev/null +++ b/samples/datalake/deltalake_and_kudu/README.md @@ -0,0 +1,147 @@ + + +# Doris + DeltaLake + Kudu + MINIO Environments +Launch spark / doris / hive / deltalake / kudu /minio test environments, and give examples to query deltalake and kudu tables in Doris. + +## Launch Docker Compose +**Create Network** +```shell +sudo docker network create -d bridge trinoconnector-net +``` +**Launch all components in docker** +```shell +sudo sh start-trinoconnector-compose.sh +``` +**Login into Spark** +```shell +sudo sh login-spark.sh +``` +**Login into Doris** +```shell +sudo sh login-doris.sh +``` + +## Prepare DeltaLake Data +There's already a deltalake table named `customer` in default database. + +## Create Catalog +The Doris Cluster has created two catalogs called `delta_lake` and `kudu_catalog`. You can view both of them by using the `SHOW CATALOGS` command or the `SHOW CREATE CATALOG ${catalog_name}` command after you log in to the Doris. Here are the creation statements for the two catalogs: + +```sql +-- The catalog has been created, and no further action is required. +create catalog delta_lake properties ( + "type"="trino-connector", + "trino.connector.name"="delta_lake", + "trino.hive.metastore.uri"="thrift://hive-metastore:9083", + "trino.hive.s3.endpoint"="http://minio:9000", + "trino.hive.s3.region"="us-east-1", + "trino.hive.s3.aws-access-key"="minio", + "trino.hive.s3.aws-secret-key"="minio123", + "trino.hive.s3.path-style-access"="true" +); + +-- The catalog has been created, and no further action is required. +CREATE CATALOG `kudu_catalog` PROPERTIES ( + "type" = "trino-connector", + "trino.connector.name" = "kudu", + "trino.kudu.authentication.type" = "NONE", + "trino.kudu.client.master-addresses" = "kudu-master-1:7051,kudu-master-2:7151,kudu-master-3:7251" +); +``` + +## Query Catalog Data +The data of `Delta Lake` and `Kudu` have been prepared in Doris Cluster. You can select these data directly in Doris. + +- select Delta Lake data + +```sql +mysql> switch delta_lake; +Query OK, 0 rows affected (0.00 sec) + +mysql> use default; +Reading table information for completion of table and column names +You can turn off this feature to get a quicker startup with -A + +Database changed +mysql> select * from customer limit 10; ++-----------+--------------------+------------------------------------+-------------+-----------------+-----------+--------------+---------------------------------------------------------------------------------------------------------------+ +| c_custkey | c_name | c_address | c_nationkey | c_phone | c_acctbal | c_mktsegment | c_comment | ++-----------+--------------------+------------------------------------+-------------+-----------------+-----------+--------------+---------------------------------------------------------------------------------------------------------------+ +| 2 | Customer#000000002 | XSTf4,NCwDVaWNe6tEgvwfmRchLXak | 13 | 23-768-687-3665 | 121.65 | AUTOMOBILE | l accounts. blithely ironic theodolites integrate boldly: caref | +| 34 | Customer#000000034 | Q6G9wZ6dnczmtOx509xgE,M2KV | 15 | 25-344-968-5422 | 8589.70 | HOUSEHOLD | nder against the even, pending accounts. even | +| 66 | Customer#000000066 | XbsEqXH1ETbJYYtA1A | 22 | 32-213-373-5094 | 242.77 | HOUSEHOLD | le slyly accounts. carefully silent packages benea | +| 98 | Customer#000000098 | 7yiheXNSpuEAwbswDW | 12 | 22-885-845-6889 | -551.37 | BUILDING | ages. furiously pending accounts are quickly carefully final foxes: busily pe | +| 130 | Customer#000000130 | RKPx2OfZy0Vn 8wGWZ7F2EAvmMORl1k8iH | 9 | 19-190-993-9281 | 5073.58 | HOUSEHOLD | ix slowly. express packages along the furiously ironic requests integrate daringly deposits. fur | +| 162 | Customer#000000162 | JE398sXZt2QuKXfJd7poNpyQFLFtth | 8 | 18-131-101-2267 | 6268.99 | MACHINERY | accounts along the doggedly special asymptotes boost blithely during the quickly regular theodolites. slyly | +| 194 | Customer#000000194 | mksKhdWuQ1pjbc4yffHp8rRmLOMcJ | 16 | 26-597-636-3003 | 6696.49 | HOUSEHOLD | quickly across the fluffily dogged requests. regular platelets around the ironic, even requests cajole quickl | +| 226 | Customer#000000226 | ToEmqB90fM TkLqyEgX8MJ8T8NkK | 3 | 13-452-318-7709 | 9008.61 | AUTOMOBILE | ic packages. ideas cajole furiously slyly special theodolites: carefully express pinto beans acco | +| 258 | Customer#000000258 | 7VbADek8qYezQYotxNUmnNI | 12 | 22-278-425-9944 | 6022.27 | MACHINERY | about the regular, bold accounts; pending packages use furiously stealthy warhorses. bold accounts sleep fur | +| 290 | Customer#000000290 | 8OlPT9G 8UqVXmVZNbmxVTPO8 | 4 | 14-458-625-5633 | 1811.35 | MACHINERY | sts. blithely pending requests sleep fluffily on the regular excuses. carefully expre | ++-----------+--------------------+------------------------------------+-------------+-----------------+-----------+--------------+---------------------------------------------------------------------------------------------------------------+ +10 rows in set (0.12 sec) +``` + +- select Kudu data + +```sql +mysql> switch kudu_catalog; +Query OK, 0 rows affected (0.00 sec) + +mysql> use default; +Reading table information for completion of table and column names +You can turn off this feature to get a quicker startup with -A + +Database changed + +mysql> select * from test_table limit 10; ++------+----------+--------+ +| key | value | added | ++------+----------+--------+ +| 0 | NULL | 12.345 | +| 4 | NULL | 12.345 | +| 20 | NULL | 12.345 | +| 26 | NULL | 12.345 | +| 29 | value 29 | 12.345 | +| 42 | NULL | 12.345 | +| 50 | NULL | 12.345 | +| 56 | NULL | 12.345 | +| 66 | NULL | 12.345 | +| 74 | NULL | 12.345 | ++------+----------+--------+ +10 rows in set (1.49 sec) +``` + +- federation query + +```sql +mysql> select * from delta_lake.`default`.customer c join kudu_catalog.`default`.test_table t on c.c_custkey = t.`key` where c.c_custkey < 50; ++-----------+--------------------+---------------------------------------+-------------+-----------------+-----------+--------------+--------------------------------------------------------------------------------------------------------+------+----------+--------+ +| c_custkey | c_name | c_address | c_nationkey | c_phone | c_acctbal | c_mktsegment | c_comment | key | value | added | ++-----------+--------------------+---------------------------------------+-------------+-----------------+-----------+--------------+--------------------------------------------------------------------------------------------------------+------+----------+--------+ +| 1 | Customer#000000001 | IVhzIApeRb ot,c,E | 15 | 25-989-741-2988 | 711.56 | BUILDING | to the even, regular platelets. regular, ironic epitaphs nag e | 1 | value 1 | 12.345 | +| 33 | Customer#000000033 | qFSlMuLucBmx9xnn5ib2csWUweg D | 17 | 27-375-391-1280 | -78.56 | AUTOMOBILE | s. slyly regular accounts are furiously. carefully pending requests | 33 | value 33 | 12.345 | +| 3 | Customer#000000003 | MG9kdTD2WBHm | 1 | 11-719-748-3364 | 7498.12 | AUTOMOBILE | deposits eat slyly ironic, even instructions. express foxes detect slyly. blithely even accounts abov | 3 | value 3 | 12.345 | +| 35 | Customer#000000035 | TEjWGE4nBzJL2 | 17 | 27-566-888-7431 | 1228.24 | HOUSEHOLD | requests. special, express requests nag slyly furiousl | 35 | value 35 | 12.345 | +| 2 | Customer#000000002 | XSTf4,NCwDVaWNe6tEgvwfmRchLXak | 13 | 23-768-687-3665 | 121.65 | AUTOMOBILE | l accounts. blithely ironic theodolites integrate boldly: caref | 2 | NULL | 12.345 | +| 34 | Customer#000000034 | Q6G9wZ6dnczmtOx509xgE,M2KV | 15 | 25-344-968-5422 | 8589.70 | HOUSEHOLD | nder against the even, pending accounts. even | 34 | NULL | 12.345 | +| 32 | Customer#000000032 | jD2xZzi UmId,DCtNBLXKj9q0Tlp2iQ6ZcO3J | 15 | 25-430-914-2194 | 3471.53 | BUILDING | cial ideas. final, furious requests across the e | 32 | NULL | 12.345 | ++-----------+--------------------+---------------------------------------+-------------+-----------------+-----------+--------------+--------------------------------------------------------------------------------------------------------+------+----------+--------+ +7 rows in set (0.13 sec) +``` diff --git a/samples/datalake/deltalake_and_kudu/data/customer/_delta_log/00000000000000000000.json b/samples/datalake/deltalake_and_kudu/data/customer/_delta_log/00000000000000000000.json new file mode 100644 index 000000000000000..a5d9fc6052bd212 --- /dev/null +++ b/samples/datalake/deltalake_and_kudu/data/customer/_delta_log/00000000000000000000.json @@ -0,0 +1,7 @@ +{"commitInfo":{"timestamp":1724747485883,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","partitionBy":"[]"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"4","numOutputRows":"18751","numOutputBytes":"1564827"},"engineInfo":"Apache-Spark/3.4.2 Delta-Lake/2.4.0","txnId":"1646d68c-f6f2-4da5-a9bf-56318b2b7216"}} +{"protocol":{"minReaderVersion":1,"minWriterVersion":2}} +{"metaData":{"id":"421eb35b-e9ec-44ed-92fd-25e0fda91036","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"c_custkey\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"c_name\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"c_address\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"c_nationkey\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"c_phone\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"c_acctbal\",\"type\":\"decimal(12,2)\",\"nullable\":true,\"metadata\":{}},{\"name\":\"c_mktsegment\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"c_comment\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1724747483772}} +{"add":{"path":"part-00000-44ff362c-110d-44ca-aed8-93ed65c19492-c000.snappy.parquet","partitionValues":{},"size":392744,"modificationTime":1724747485000,"dataChange":true,"stats":"{\"numRecords\":4688,\"minValues\":{\"c_custkey\":2,\"c_name\":\"Customer#000000002\",\"c_address\":\" UfkcgKnrSL0VRSDuuXjXW,\",\"c_nationkey\":0,\"c_phone\":\"10-103-318-6809\",\"c_acctbal\":-998.90,\"c_mktsegment\":\"AUTOMOBILE\",\"c_comment\":\" Tiresias detect always about \"},\"maxValues\":{\"c_custkey\":149986,\"c_name\":\"Customer#000149986\",\"c_address\":\"zyTHzirSOvDeqwIs4R7qn76825FPYr8Y�\",\"c_nationkey\":24,\"c_phone\":\"34-999-195-7029\",\"c_acctbal\":9994.63,\"c_mktsegment\":\"MACHINERY\",\"c_comment\":\"ze! special, even deposits nag q�\"},\"nullCount\":{\"c_custkey\":0,\"c_name\":0,\"c_address\":0,\"c_nationkey\":0,\"c_phone\":0,\"c_acctbal\":0,\"c_mktsegment\":0,\"c_comment\":0}}"}} +{"add":{"path":"part-00001-749ded2d-a84b-4e2c-9e6f-5ac6a59ee91d-c000.snappy.parquet","partitionValues":{},"size":392284,"modificationTime":1724747485000,"dataChange":true,"stats":"{\"numRecords\":4687,\"minValues\":{\"c_custkey\":32,\"c_name\":\"Customer#000000032\",\"c_address\":\" FjVZqAg2Pd9jhTN8pVD4DkvmxlCxMm\",\"c_nationkey\":0,\"c_phone\":\"10-105-777-9167\",\"c_acctbal\":-994.43,\"c_mktsegment\":\"AUTOMOBILE\",\"c_comment\":\" about the fluffily bold ideas. \"},\"maxValues\":{\"c_custkey\":149984,\"c_name\":\"Customer#000149984\",\"c_address\":\"zz5LSqGU2QoyQTcMzkOxVqWrHedmhqQ6�\",\"c_nationkey\":24,\"c_phone\":\"34-997-204-5897\",\"c_acctbal\":9998.01,\"c_mktsegment\":\"MACHINERY\",\"c_comment\":\"zzle quickly bold packages. sile�\"},\"nullCount\":{\"c_custkey\":0,\"c_name\":0,\"c_address\":0,\"c_nationkey\":0,\"c_phone\":0,\"c_acctbal\":0,\"c_mktsegment\":0,\"c_comment\":0}}"}} +{"add":{"path":"part-00002-137a1b68-bafd-46a7-b231-400a174b520c-c000.snappy.parquet","partitionValues":{},"size":390594,"modificationTime":1724747485000,"dataChange":true,"stats":"{\"numRecords\":4688,\"minValues\":{\"c_custkey\":1,\"c_name\":\"Customer#000000001\",\"c_address\":\" NUi8asf651zG096JTGeXdh\",\"c_nationkey\":0,\"c_phone\":\"10-100-220-4520\",\"c_acctbal\":-999.55,\"c_mktsegment\":\"AUTOMOBILE\",\"c_comment\":\" Tiresias detect slyly according\"},\"maxValues\":{\"c_custkey\":149985,\"c_name\":\"Customer#000149985\",\"c_address\":\"zzbUlYAy9rhCprBVHlzA\",\"c_nationkey\":24,\"c_phone\":\"34-999-363-7145\",\"c_acctbal\":9997.80,\"c_mktsegment\":\"MACHINERY\",\"c_comment\":\"zzle blithely against the carefu�\"},\"nullCount\":{\"c_custkey\":0,\"c_name\":0,\"c_address\":0,\"c_nationkey\":0,\"c_phone\":0,\"c_acctbal\":0,\"c_mktsegment\":0,\"c_comment\":0}}"}} +{"add":{"path":"part-00003-75203d54-ef95-4fbe-95c9-2012fd9dbaed-c000.snappy.parquet","partitionValues":{},"size":389205,"modificationTime":1724747485000,"dataChange":true,"stats":"{\"numRecords\":4688,\"minValues\":{\"c_custkey\":3,\"c_name\":\"Customer#000000003\",\"c_address\":\" 821GWWou3sOyp,\",\"c_nationkey\":0,\"c_phone\":\"10-105-204-5643\",\"c_acctbal\":-999.95,\"c_mktsegment\":\"AUTOMOBILE\",\"c_comment\":\" about the fluffily regular asym\"},\"maxValues\":{\"c_custkey\":149987,\"c_name\":\"Customer#000149987\",\"c_address\":\"zwrbKxxY yL Go\",\"c_nationkey\":24,\"c_phone\":\"34-999-283-6448\",\"c_acctbal\":9997.73,\"c_mktsegment\":\"MACHINERY\",\"c_comment\":\"ymptotes. unusual theodolites ab�\"},\"nullCount\":{\"c_custkey\":0,\"c_name\":0,\"c_address\":0,\"c_nationkey\":0,\"c_phone\":0,\"c_acctbal\":0,\"c_mktsegment\":0,\"c_comment\":0}}"}} diff --git a/samples/datalake/deltalake_and_kudu/data/customer/part-00000-44ff362c-110d-44ca-aed8-93ed65c19492-c000.snappy.parquet b/samples/datalake/deltalake_and_kudu/data/customer/part-00000-44ff362c-110d-44ca-aed8-93ed65c19492-c000.snappy.parquet new file mode 100644 index 000000000000000..db5f6570c5cfec9 Binary files /dev/null and b/samples/datalake/deltalake_and_kudu/data/customer/part-00000-44ff362c-110d-44ca-aed8-93ed65c19492-c000.snappy.parquet differ diff --git a/samples/datalake/deltalake_and_kudu/data/customer/part-00001-749ded2d-a84b-4e2c-9e6f-5ac6a59ee91d-c000.snappy.parquet b/samples/datalake/deltalake_and_kudu/data/customer/part-00001-749ded2d-a84b-4e2c-9e6f-5ac6a59ee91d-c000.snappy.parquet new file mode 100644 index 000000000000000..7abc7ccd604c715 Binary files /dev/null and b/samples/datalake/deltalake_and_kudu/data/customer/part-00001-749ded2d-a84b-4e2c-9e6f-5ac6a59ee91d-c000.snappy.parquet differ diff --git a/samples/datalake/deltalake_and_kudu/data/customer/part-00002-137a1b68-bafd-46a7-b231-400a174b520c-c000.snappy.parquet b/samples/datalake/deltalake_and_kudu/data/customer/part-00002-137a1b68-bafd-46a7-b231-400a174b520c-c000.snappy.parquet new file mode 100644 index 000000000000000..020abd85fc8bfb5 Binary files /dev/null and b/samples/datalake/deltalake_and_kudu/data/customer/part-00002-137a1b68-bafd-46a7-b231-400a174b520c-c000.snappy.parquet differ diff --git a/samples/datalake/deltalake_and_kudu/data/customer/part-00003-75203d54-ef95-4fbe-95c9-2012fd9dbaed-c000.snappy.parquet b/samples/datalake/deltalake_and_kudu/data/customer/part-00003-75203d54-ef95-4fbe-95c9-2012fd9dbaed-c000.snappy.parquet new file mode 100644 index 000000000000000..458f5c5f4d2608f Binary files /dev/null and b/samples/datalake/deltalake_and_kudu/data/customer/part-00003-75203d54-ef95-4fbe-95c9-2012fd9dbaed-c000.snappy.parquet differ diff --git a/samples/datalake/deltalake_and_kudu/login-doris.sh b/samples/datalake/deltalake_and_kudu/login-doris.sh new file mode 100755 index 000000000000000..90aaaa100bc5d93 --- /dev/null +++ b/samples/datalake/deltalake_and_kudu/login-doris.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +docker exec -it spark-hive mysql -u root -h doris-env -P 9030 diff --git a/samples/datalake/deltalake_and_kudu/login-spark.sh b/samples/datalake/deltalake_and_kudu/login-spark.sh new file mode 100755 index 000000000000000..6a55539dcefc1d5 --- /dev/null +++ b/samples/datalake/deltalake_and_kudu/login-spark.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +docker exec -it spark-hive /opt/scripts/spark-delta.sh diff --git a/samples/datalake/deltalake_and_kudu/scripts/create-delta-table.sh b/samples/datalake/deltalake_and_kudu/scripts/create-delta-table.sh new file mode 100644 index 000000000000000..9cf5a14ca3a45a6 --- /dev/null +++ b/samples/datalake/deltalake_and_kudu/scripts/create-delta-table.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +export SPARK_HOME=/opt/spark +export HIVE_HOME=/opt/apache-hive-3.1.2-bin +export HADOOP_HOME=/opt/hadoop-3.3.1 + +if [[ ! -d "${SPARK_HOME}" ]]; then + cp -r /opt/spark-3.4.2-bin-hadoop3 "${SPARK_HOME}" +fi + +cp "${HIVE_HOME}"/conf/hive-site.xml "${SPARK_HOME}"/conf/ +cp "${HIVE_HOME}"/lib/postgresql-jdbc.jar "${SPARK_HOME}"/jars/ +cp "${HADOOP_HOME}"/etc/hadoop/core-site.xml "${SPARK_HOME}"/conf/ + +"${SPARK_HOME}"/bin/spark-sql \ + --master local[*] \ + --name "spark-delta-sql" \ + --conf "spark.sql.extensions=io.delta.sql.DeltaSparkSessionExtension" \ + --conf "spark.sql.catalog.spark_catalog=org.apache.spark.sql.delta.catalog.DeltaCatalog" \ + -f /opt/scripts/spark-delta.sql diff --git a/samples/datalake/deltalake_and_kudu/scripts/doris-sql.sql b/samples/datalake/deltalake_and_kudu/scripts/doris-sql.sql new file mode 100644 index 000000000000000..8d003a1f31706ca --- /dev/null +++ b/samples/datalake/deltalake_and_kudu/scripts/doris-sql.sql @@ -0,0 +1,20 @@ +create catalog delta_lake properties ( + "type"="trino-connector", + "trino.connector.name"="delta_lake", + "trino.hive.metastore.uri"="thrift://hive-metastore:9083", + "trino.hive.s3.endpoint"="http://minio:9000", + "trino.hive.s3.region"="us-east-1", + "trino.hive.s3.aws-access-key"="minio", + "trino.hive.s3.aws-secret-key"="minio123", + "trino.hive.s3.path-style-access"="true" +); + + +CREATE CATALOG `kudu_catalog` PROPERTIES ( + "type" = "trino-connector", + "trino.connector.name" = "kudu", + "trino.kudu.authentication.type" = "NONE", + "trino.kudu.client.master-addresses" = "kudu-master-1:7051,kudu-master-2:7151,kudu-master-3:7251" +); + +ALTER SYSTEM ADD BACKEND 'doris-env:9050'; diff --git a/samples/datalake/deltalake_and_kudu/scripts/spark-delta.sh b/samples/datalake/deltalake_and_kudu/scripts/spark-delta.sh new file mode 100755 index 000000000000000..f4a640726588261 --- /dev/null +++ b/samples/datalake/deltalake_and_kudu/scripts/spark-delta.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +export SPARK_HOME=/opt/spark +export HIVE_HOME=/opt/apache-hive-3.1.2-bin +export HADOOP_HOME=/opt/hadoop-3.3.1 + +if [[ ! -d "${SPARK_HOME}" ]]; then + cp -r /opt/spark-3.4.2-bin-hadoop3 "${SPARK_HOME}" +fi + +cp "${HIVE_HOME}"/conf/hive-site.xml "${SPARK_HOME}"/conf/ +cp "${HIVE_HOME}"/lib/postgresql-jdbc.jar "${SPARK_HOME}"/jars/ +cp "${HADOOP_HOME}"/etc/hadoop/core-site.xml "${SPARK_HOME}"/conf/ + +"${SPARK_HOME}"/bin/spark-sql \ + --master local[*] \ + --name "spark-delta-sql" \ + --conf "spark.sql.extensions=io.delta.sql.DeltaSparkSessionExtension" \ + --conf "spark.sql.catalog.spark_catalog=org.apache.spark.sql.delta.catalog.DeltaCatalog" diff --git a/samples/datalake/deltalake_and_kudu/scripts/spark-delta.sql b/samples/datalake/deltalake_and_kudu/scripts/spark-delta.sql new file mode 100644 index 000000000000000..25f8757889f66d4 --- /dev/null +++ b/samples/datalake/deltalake_and_kudu/scripts/spark-delta.sql @@ -0,0 +1 @@ +CREATE TABLE default.customer USING DELTA LOCATION 's3://data/customer/'; \ No newline at end of file diff --git a/samples/datalake/deltalake_and_kudu/scripts/start_doris.sh b/samples/datalake/deltalake_and_kudu/scripts/start_doris.sh new file mode 100755 index 000000000000000..91965d4e4ed6831 --- /dev/null +++ b/samples/datalake/deltalake_and_kudu/scripts/start_doris.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +export JAVA_HOME=/opt/jdk-17.0.2 + +cp -r /opt/doris-bin /opt/doris +echo "trino_connector_plugin_dir=/opt/connectors/" >> /opt/doris/fe/conf/fe.conf +echo "trino_connector_plugin_dir=/opt/connectors/" >> /opt/doris/be/conf/be.conf + +/opt/doris/fe/bin/start_fe.sh --daemon +/opt/doris/be/bin/start_be.sh --daemon +tail -F /dev/null diff --git a/samples/datalake/deltalake_and_kudu/start-trinoconnector-compose.sh b/samples/datalake/deltalake_and_kudu/start-trinoconnector-compose.sh new file mode 100755 index 000000000000000..96fc6c739b016cf --- /dev/null +++ b/samples/datalake/deltalake_and_kudu/start-trinoconnector-compose.sh @@ -0,0 +1,181 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +DORIS_PACKAGE=apache-doris-3.0.1-bin-x64 +DORIS_DOWNLOAD_URL=https://apache-doris-releases.oss-accelerate.aliyuncs.com + +md5_aws_java_sdk="452d1e00efb11bff0ee17c42a6a44a0a" +md5_hadoop_aws="a3e19d42cadd1a6862a41fd276f94382" +md5_jdk17="0930efa680ac61e833699ccc36bfc739" +md5_spark="b393d314ffbc03facdc85575197c5db9" +md5_doris="fecd81c2d043542363422de6f111dbdb" +delta_core="65b8dec752d4984b7958d644848e3978" +delta_storage="a83011a52c66e081d4f53a7dc5c9708a" +antlr4_runtime="718f199bafa6574ffa1111fa3e10276a" +kudu_plugin="2d58bfcac5b84218c5d1055af189e30c" +delta_plugin="6b33448ce42d3d05e7b500ccafbe9698" +hdfs_plugin="ff4a3e3b32dcce27f4df58f17938abde" +kudu_java_example="1afe0a890785e8d0011ea7342ae5e43d" + + +download_source_file() { + local FILE_PATH="$1" + local EXPECTED_MD5="$2" + local DOWNLOAD_URL="$3" + + echo "Download ${FILE_PATH}" + + if [[ -f "${FILE_PATH}" ]]; then + local FILE_MD5 + FILE_MD5=$(md5sum "${FILE_PATH}" | awk '{ print $1 }') + echo "${FILE_PATH} 's md5sum is = ${FILE_MD5} ; expected is = ${EXPECTED_MD5}" + if [[ "${FILE_MD5}" = "${EXPECTED_MD5}" ]]; then + echo "${FILE_PATH} is ready!" + else + echo "${FILE_PATH} is broken, Redownloading ..." + rm "${FILE_PATH}" + wget "${DOWNLOAD_URL}"/"${FILE_PATH}" + fi + else + echo "Downloading ${FILE_PATH} ..." + wget "${DOWNLOAD_URL}"/"${FILE_PATH}" + fi +} + +curdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" +cd "${curdir}" || exit + +if [[ ! -d "packages" ]]; then + mkdir packages +fi +cd packages || exit + +download_source_file "aws-java-sdk-bundle-1.12.48.jar" "${md5_aws_java_sdk}" "https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-bundle/1.12.48" +download_source_file "hadoop-aws-3.3.1.jar" "${md5_hadoop_aws}" "https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/3.3.1" +download_source_file "openjdk-17.0.2_linux-x64_bin.tar.gz" "${md5_jdk17}" "https://download.java.net/java/GA/jdk17.0.2/dfd4a8d0985749f896bed50d7138ee7f/8/GPL" +download_source_file "spark-3.4.2-bin-hadoop3.tgz" "${md5_spark}" "https://archive.apache.org/dist/spark/spark-3.4.2" +download_source_file "${DORIS_PACKAGE}.tar.gz" "${md5_doris}" "${DORIS_DOWNLOAD_URL}" +download_source_file "delta-core_2.12-2.4.0.jar" "${delta_core}" "https://repo1.maven.org/maven2/io/delta/delta-core_2.12/2.4.0" +download_source_file "delta-storage-2.4.0.jar" "${delta_storage}" "https://repo1.maven.org/maven2/io/delta/delta-storage/2.4.0" +download_source_file "antlr4-runtime-4.9.3.jar" "${antlr4_runtime}" "https://repo1.maven.org/maven2/org/antlr/antlr4-runtime/4.9.3" +download_source_file "trino-delta-lake-435-20240724.tar.gz" "${delta_plugin}" "https://github.com/apache/doris-thirdparty/releases/download/trino-435-20240724" +download_source_file "trino-kudu-435-20240724.tar.gz" "${kudu_plugin}" "https://github.com/apache/doris-thirdparty/releases/download/trino-435-20240724" +download_source_file "trino-hdfs-435-20240724.tar.gz" "${hdfs_plugin}" "https://github.com/apache/doris-thirdparty/releases/download/trino-435-20240724" +download_source_file "kudu-java-example-1.0-SNAPSHOT.jar" "${kudu_java_example}" "https://github.com/apache/doris-thirdparty/releases/download/trino-435-20240724" + + + + +if [[ ! -f "jdk-17.0.2/SUCCESS" ]]; then + echo "Prepare jdk17 environment" + if [[ -d "jdk-17.0.2" ]]; then + echo "Remove broken jdk-17.0.2" + rm -rf jdk-17.0.2 + fi + echo "Unpackage jdk-17.0.2" + tar xzf openjdk-17.0.2_linux-x64_bin.tar.gz + touch jdk-17.0.2/SUCCESS +fi +if [[ ! -f "spark-3.4.2-bin-hadoop3/SUCCESS" ]]; then + echo "Prepare spark3.4 environment" + if [[ -d "spark-3.4.2-bin-hadoop3" ]]; then + echo "Remove broken spark-3.4.2-bin-hadoop3" + rm -rf spark-3.4.2-bin-hadoop3 + fi + echo "Unpackage spark-3.4.2-bin-hadoop3" + tar -xf spark-3.4.2-bin-hadoop3.tgz + cp aws-java-sdk-bundle-1.12.48.jar spark-3.4.2-bin-hadoop3/jars/ + cp hadoop-aws-3.3.1.jar spark-3.4.2-bin-hadoop3/jars/ + cp delta-core_2.12-2.4.0.jar spark-3.4.2-bin-hadoop3/jars/ + cp delta-storage-2.4.0.jar spark-3.4.2-bin-hadoop3/jars/ + cp antlr4-runtime-4.9.3.jar spark-3.4.2-bin-hadoop3/jars/ + touch spark-3.4.2-bin-hadoop3/SUCCESS +fi +if [[ ! -f "doris-bin/SUCCESS" ]]; then + echo "Prepare ${DORIS_PACKAGE} environment" + if [[ -d "doris-bin" ]]; then + echo "Remove broken ${DORIS_PACKAGE}" + rm -rf doris-bin + fi + echo "Unpackage ${DORIS_PACKAGE}" + tar xzf "${DORIS_PACKAGE}".tar.gz + mv "${DORIS_PACKAGE}" doris-bin + touch doris-bin/SUCCESS +fi + +mkdir connectors +if [[ ! -f "connectors/trino-delta-lake-435/SUCCESS" ]]; then + echo "Prepare trino-delta-lake-435 plugin" + if [[ -d "connectors/trino-delta-lake-435" ]]; then + echo "Remove broken trino-delta-lake-435" + rm -rf connectors/trino-delta-lake-435 + fi + echo "Unpackage trino-delta-lake-435" + tar xzf trino-delta-lake-435-20240724.tar.gz + mv trino-delta-lake-435 connectors/trino-delta-lake-435 + touch connectors/trino-delta-lake-435/SUCCESS +fi + +if [[ ! -f "connectors/trino-kudu-435/SUCCESS" ]]; then + echo "Prepare trino-kudu-435 plugin" + if [[ -d "connectors/trino-kudu-435" ]]; then + echo "Remove broken trino-kudu-435" + rm -rf connectors/trino-kudu-435 + fi + echo "Unpackage trino-kudu-435" + tar xzf trino-kudu-435-20240724.tar.gz + mv trino-kudu-435 connectors/trino-kudu-435 + touch connectors/trino-kudu-435/SUCCESS +fi + +if [[ ! -f "connectors/trino-delta-lake-435/hdfs/SUCCESS" ]]; then + echo "Prepare hdfs plugin" + if [[ -d "connectors/trino-delta-lake-435/hdfs" ]]; then + echo "Remove broken connectors/trino-delta-lake-435/hdfs" + rm -rf connectors/trino-delta-lake-435/hdfs + fi + echo "Unpackage trino-delta-lake-435/hdfs" + tar xzf trino-hdfs-435-20240724.tar.gz + mv hdfs connectors/trino-delta-lake-435/hdfs + touch connectors/trino-delta-lake-435/hdfs/SUCCESS +fi + + +cd ../ + +export KUDU_QUICKSTART_IP=$(ifconfig | grep "inet " | grep -Fv 127.0.0.1 | awk '{print $2}' | tail -1) + +docker compose -f trinoconnector-compose.yml --env-file trinoconnector-compose.env up -d +echo "Create hive table ..." +sleep 5 +docker exec -it spark-hive sh -c "/opt/hadoop-3.3.1/bin/hadoop fs -chmod 777 /tmp/hive" +docker exec -it spark-hive sh -c "sh /opt/scripts/create-delta-table.sh" +sleep 5 +echo "Build hive catalog in Doris ..." +docker exec -it spark-hive sh -c "mysql -u root -h doris-env -P 9030 < /opt/scripts/doris-sql.sql" +sleep 10 +echo "Create kudu table ..." +docker exec -it kudu-master-1 sh -c "/opt/jdk-17.0.2/bin/java -DkuduMasters=kudu-master-1:7051,kudu-master-2:7151,kudu-master-3:7251 -jar /opt/kudu-java-example-1.0-SNAPSHOT.jar" +sleep 10 + +echo "======================================================" +echo "Success to launch spark+doris+deltalake+kudu+minio environments!" +echo "./login-spark.sh to login into spark" +echo "./login-doris.sh to login into doris" +echo "======================================================" diff --git a/samples/datalake/deltalake_and_kudu/stop-trinoconnector-compose.sh b/samples/datalake/deltalake_and_kudu/stop-trinoconnector-compose.sh new file mode 100755 index 000000000000000..838630d0249297c --- /dev/null +++ b/samples/datalake/deltalake_and_kudu/stop-trinoconnector-compose.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +export KUDU_QUICKSTART_IP=$(ifconfig | grep "inet " | grep -Fv 127.0.0.1 | awk '{print $2}' | tail -1) + +docker compose -f trinoconnector-compose.yml --env-file trinoconnector-compose.env down diff --git a/samples/datalake/deltalake_and_kudu/trinoconnector-compose.env b/samples/datalake/deltalake_and_kudu/trinoconnector-compose.env new file mode 100644 index 000000000000000..8abf3700ba1fe78 --- /dev/null +++ b/samples/datalake/deltalake_and_kudu/trinoconnector-compose.env @@ -0,0 +1,22 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +HIVE_THRIFT_PORT=9771 +MINIO_SERVICE_PORT=9772 +DORIS_QUERY_PORT=9773 +KUDU_QUICKSTART_VERSION="1.17.0" \ No newline at end of file diff --git a/samples/datalake/deltalake_and_kudu/trinoconnector-compose.yml b/samples/datalake/deltalake_and_kudu/trinoconnector-compose.yml new file mode 100644 index 000000000000000..e88348b3e6f4b13 --- /dev/null +++ b/samples/datalake/deltalake_and_kudu/trinoconnector-compose.yml @@ -0,0 +1,323 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +version: "3.9" +services: + metastore_db: + image: postgres:11 + hostname: metastore_db + environment: + POSTGRES_USER: hive + POSTGRES_PASSWORD: hive + POSTGRES_DB: metastore + + hive-metastore: + hostname: hive-metastore + container_name: spark-hive + image: 'starburstdata/hive:3.1.2-e.18' + volumes: + - './packages/spark-3.4.2-bin-hadoop3:/opt/spark-3.4.2-bin-hadoop3' + - './scripts:/opt/scripts' + ports: + - '${HIVE_THRIFT_PORT}:9083' # Metastore Thrift + environment: + HIVE_METASTORE_DRIVER: org.postgresql.Driver + HIVE_METASTORE_JDBC_URL: jdbc:postgresql://metastore_db:5432/metastore + HIVE_METASTORE_USER: hive + HIVE_METASTORE_PASSWORD: hive + HIVE_METASTORE_WAREHOUSE_DIR: s3://datalake/ + S3_ENDPOINT: http://minio:9000 + S3_ACCESS_KEY: minio + S3_SECRET_KEY: minio123 + S3_PATH_STYLE_ACCESS: "true" + REGION: "" + GOOGLE_CLOUD_KEY_FILE_PATH: "" + AZURE_ADL_CLIENT_ID: "" + AZURE_ADL_CREDENTIAL: "" + AZURE_ADL_REFRESH_URL: "" + AZURE_ABFS_STORAGE_ACCOUNT: "" + AZURE_ABFS_ACCESS_KEY: "" + AZURE_WASB_STORAGE_ACCOUNT: "" + AZURE_ABFS_OAUTH: "" + AZURE_ABFS_OAUTH_TOKEN_PROVIDER: "" + AZURE_ABFS_OAUTH_CLIENT_ID: "" + AZURE_ABFS_OAUTH_SECRET: "" + AZURE_ABFS_OAUTH_ENDPOINT: "" + AZURE_WASB_ACCESS_KEY: "" + HIVE_METASTORE_USERS_IN_ADMIN_ROLE: "hive" + depends_on: + - metastore_db + + doris-env: + hostname: doris-env + container_name: doris-env + image: 'apache/hadoop:3.3.6' + environment: + LD_LIBRARY_PATH: /opt/doris/be/lib + ports: + - '${DORIS_QUERY_PORT}:9030' + volumes: + - './packages/jdk-17.0.2:/opt/jdk-17.0.2' + - './packages/doris-bin:/opt/doris-bin' + - './scripts:/opt/scripts' + - './packages/connectors:/opt/connectors' + command: sh /opt/scripts/start_doris.sh + + minio: + hostname: minio + image: 'minio/minio:RELEASE.2022-05-26T05-48-41Z' + container_name: minio-storage + ports: + - '${MINIO_SERVICE_PORT}:9000' + environment: + MINIO_ACCESS_KEY: minio + MINIO_SECRET_KEY: minio123 + command: server /data --console-address ":9001" + + # This job will create the "datalake" bucket on Minio + mc-job: + image: 'minio/mc:RELEASE.2022-05-09T04-08-26Z' + volumes: + - './data:/data' + entrypoint: | + /bin/bash -c " + sleep 5; + /usr/bin/mc config --quiet host add myminio http://minio:9000 minio minio123; + /usr/bin/mc mb --quiet myminio/datalake; + /usr/bin/mc mb --quiet myminio/data; + /usr/bin/mc mirror /data myminio/data + " + depends_on: + - minio + + kudu-master-1: + image: apache/kudu:${KUDU_QUICKSTART_VERSION:-latest} + hostname: kudu-master-1 + container_name: kudu-master-1 + ports: + - "7051:7051" + - "8051:8051" + command: ["master"] + volumes: + - kudu-master-1:/var/lib/kudu + - './packages/jdk-17.0.2:/opt/jdk-17.0.2' + - './packages/kudu-java-example-1.0-SNAPSHOT.jar:/opt/kudu-java-example-1.0-SNAPSHOT.jar' + environment: + - KUDU_MASTERS=kudu-master-1:7051,kudu-master-2:7151,kudu-master-3:7251 + # TODO: Use `host.docker.internal` instead of KUDU_QUICKSTART_IP when it + # works on Linux (https://github.com/docker/for-linux/issues/264) + - > + MASTER_ARGS=--fs_wal_dir=/var/lib/kudu/master + --rpc_bind_addresses=0.0.0.0:7051 + --rpc_advertised_addresses=${KUDU_QUICKSTART_IP:?Please set KUDU_QUICKSTART_IP environment variable}:7051 + --webserver_port=8051 + --webserver_advertised_addresses=${KUDU_QUICKSTART_IP}:8051 + --webserver_doc_root=/opt/kudu/www + --stderrthreshold=0 + --use_hybrid_clock=false + --unlock_unsafe_flags=true + # command: /opt/jdk-17.0.2/bin/java -DkuduMasters=kudu-master-1:7051,kudu-master-2:7151,kudu-master-3:7251 -jar /opt/kudu-java-example-1.0-SNAPSHOT.jar + + kudu-master-2: + image: apache/kudu:${KUDU_QUICKSTART_VERSION:-latest} + hostname: kudu-master-2 + container_name: kudu-master-2 + ports: + - "7151:7151" + - "8151:8151" + command: ["master"] + volumes: + - kudu-master-2:/var/lib/kudu + environment: + - KUDU_MASTERS=kudu-master-1:7051,kudu-master-2:7151,kudu-master-3:7251 + - > + MASTER_ARGS=--fs_wal_dir=/var/lib/kudu/master + --rpc_bind_addresses=0.0.0.0:7151 + --rpc_advertised_addresses=${KUDU_QUICKSTART_IP}:7151 + --webserver_port=8151 + --webserver_advertised_addresses=${KUDU_QUICKSTART_IP}:8151 + --webserver_doc_root=/opt/kudu/www + --stderrthreshold=0 + --use_hybrid_clock=false + --unlock_unsafe_flags=true + kudu-master-3: + image: apache/kudu:${KUDU_QUICKSTART_VERSION:-latest} + hostname: kudu-master-3 + container_name: kudu-master-3 + ports: + - "7251:7251" + - "8251:8251" + command: ["master"] + volumes: + - kudu-master-3:/var/lib/kudu + environment: + - KUDU_MASTERS=kudu-master-1:7051,kudu-master-2:7151,kudu-master-3:7251 + - > + MASTER_ARGS=--fs_wal_dir=/var/lib/kudu/master + --rpc_bind_addresses=0.0.0.0:7251 + --rpc_advertised_addresses=${KUDU_QUICKSTART_IP}:7251 + --webserver_port=8251 + --webserver_advertised_addresses=${KUDU_QUICKSTART_IP}:8251 + --webserver_doc_root=/opt/kudu/www + --stderrthreshold=0 + --use_hybrid_clock=false + --unlock_unsafe_flags=true + kudu-tserver-1: + image: apache/kudu:${KUDU_QUICKSTART_VERSION:-latest} + hostname: kudu-tserver-1 + container_name: kudu-tserver-1 + depends_on: + - kudu-master-1 + - kudu-master-2 + - kudu-master-3 + ports: + - "7050:7050" + - "8050:8050" + command: ["tserver"] + volumes: + - kudu-tserver-1:/var/lib/kudu + environment: + - KUDU_MASTERS=kudu-master-1:7051,kudu-master-2:7151,kudu-master-3:7251 + - > + TSERVER_ARGS=--fs_wal_dir=/var/lib/kudu/tserver + --rpc_bind_addresses=0.0.0.0:7050 + --rpc_advertised_addresses=${KUDU_QUICKSTART_IP}:7050 + --webserver_port=8050 + --webserver_advertised_addresses=${KUDU_QUICKSTART_IP}:8050 + --webserver_doc_root=/opt/kudu/www + --stderrthreshold=0 + --use_hybrid_clock=false + --unlock_unsafe_flags=true + kudu-tserver-2: + image: apache/kudu:${KUDU_QUICKSTART_VERSION:-latest} + hostname: kudu-tserver-2 + container_name: kudu-tserver-2 + depends_on: + - kudu-master-1 + - kudu-master-2 + - kudu-master-3 + ports: + - "7150:7150" + - "8150:8150" + command: ["tserver"] + volumes: + - kudu-tserver-2:/var/lib/kudu + environment: + - KUDU_MASTERS=kudu-master-1:7051,kudu-master-2:7151,kudu-master-3:7251 + - > + TSERVER_ARGS=--fs_wal_dir=/var/lib/kudu/tserver + --rpc_bind_addresses=0.0.0.0:7150 + --rpc_advertised_addresses=${KUDU_QUICKSTART_IP}:7150 + --webserver_port=8150 + --webserver_advertised_addresses=${KUDU_QUICKSTART_IP}:8150 + --webserver_doc_root=/opt/kudu/www + --stderrthreshold=0 + --use_hybrid_clock=false + --unlock_unsafe_flags=true + kudu-tserver-3: + image: apache/kudu:${KUDU_QUICKSTART_VERSION:-latest} + hostname: kudu-tserver-3 + container_name: kudu-tserver-3 + depends_on: + - kudu-master-1 + - kudu-master-2 + - kudu-master-3 + ports: + - "7250:7250" + - "8250:8250" + command: ["tserver"] + volumes: + - kudu-tserver-3:/var/lib/kudu + environment: + - KUDU_MASTERS=kudu-master-1:7051,kudu-master-2:7151,kudu-master-3:7251 + - > + TSERVER_ARGS=--fs_wal_dir=/var/lib/kudu/tserver + --rpc_bind_addresses=0.0.0.0:7250 + --rpc_advertised_addresses=${KUDU_QUICKSTART_IP}:7250 + --webserver_port=8250 + --webserver_advertised_addresses=${KUDU_QUICKSTART_IP}:8250 + --webserver_doc_root=/opt/kudu/www + --stderrthreshold=0 + --use_hybrid_clock=false + --unlock_unsafe_flags=true + kudu-tserver-4: + image: apache/kudu:${KUDU_QUICKSTART_VERSION:-latest} + hostname: kudu-tserver-4 + container_name: kudu-tserver-4 + depends_on: + - kudu-master-1 + - kudu-master-2 + - kudu-master-3 + ports: + - "7350:7350" + - "8350:8350" + command: ["tserver"] + volumes: + - kudu-tserver-4:/var/lib/kudu + environment: + - KUDU_MASTERS=kudu-master-1:7051,kudu-master-2:7151,kudu-master-3:7251 + - > + TSERVER_ARGS=--fs_wal_dir=/var/lib/kudu/tserver + --rpc_bind_addresses=0.0.0.0:7350 + --rpc_advertised_addresses=${KUDU_QUICKSTART_IP}:7350 + --webserver_port=8350 + --webserver_advertised_addresses=${KUDU_QUICKSTART_IP}:8350 + --webserver_doc_root=/opt/kudu/www + --stderrthreshold=0 + --use_hybrid_clock=false + --unlock_unsafe_flags=true + kudu-tserver-5: + image: apache/kudu:${KUDU_QUICKSTART_VERSION:-latest} + hostname: kudu-tserver-5 + container_name: kudu-tserver-5 + depends_on: + - kudu-master-1 + - kudu-master-2 + - kudu-master-3 + ports: + - "7450:7450" + - "8450:8450" + command: ["tserver"] + volumes: + - kudu-tserver-5:/var/lib/kudu + environment: + - KUDU_MASTERS=kudu-master-1:7051,kudu-master-2:7151,kudu-master-3:7251 + - > + TSERVER_ARGS=--fs_wal_dir=/var/lib/kudu/tserver + --rpc_bind_addresses=0.0.0.0:7450 + --rpc_advertised_addresses=${KUDU_QUICKSTART_IP}:7450 + --webserver_port=8450 + --webserver_advertised_addresses=${KUDU_QUICKSTART_IP}:8450 + --webserver_doc_root=/opt/kudu/www + --stderrthreshold=0 + --use_hybrid_clock=false + --unlock_unsafe_flags=true + +volumes: + kudu-master-1: + kudu-master-2: + kudu-master-3: + kudu-tserver-1: + kudu-tserver-2: + kudu-tserver-3: + kudu-tserver-4: + kudu-tserver-5: + +networks: + default: + name: trinoconnector-net + external: true