[opt](test) shorten the external p0 running time (#37320) (#37473)

bp #37320

.gitignore

@@ -60,6 +60,8 @@ docker/thirdparties/docker-compose/hive/scripts/paimon1
 fe_plugins/output
 fe_plugins/**/.factorypath
 
+docker/thirdparties/docker-compose/hive/scripts/data/*/*/data
+
 fs_brokers/apache_hdfs_broker/src/main/resources/
 fs_brokers/apache_hdfs_broker/src/main/thrift/
 

.licenserc.yaml

@@ -77,12 +77,7 @@ header:
     - "docs/package-lock.json"
     - "regression-test/script/README"
     - "regression-test/suites/load_p0/stream_load/data"
-    - "docker/thirdparties/docker-compose/hive/scripts/README"
-    - "docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_table.hql"
-    - "docker/thirdparties/docker-compose/hive/scripts/create_tpch1_orc.hql"
-    - "docker/thirdparties/docker-compose/hive/scripts/create_tpch1_parquet.hql"
-    - "docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/"
-    - "docker/thirdparties/docker-compose/hive/scripts/data/**"
+    - "docker/thirdparties/docker-compose/hive/scripts/**"
     - "docker/thirdparties/docker-compose/iceberg/spark-defaults.conf.tpl"
     - "conf/mysql_ssl_default_certificate/*"
     - "conf/mysql_ssl_default_certificate/client_certificate/ca.pem"

be/src/util/thrift_rpc_helper.cpp

@@ -66,7 +66,8 @@ Status ThriftRpcHelper::rpc(const std::string& ip, const int32_t port,
     Status status;
     ClientConnection<T> client(_s_exec_env->get_client_cache<T>(), address, timeout_ms, &status);
     if (!status.ok()) {
-        LOG(WARNING) << "Connect frontend failed, address=" << address << ", status=" << status;
+        LOG(WARNING) << "Connect frontend failed, address=" << ip << ":" << port
+                     << ", status=" << status;
         return status;
     }
     try {
@@ -75,35 +76,35 @@ Status ThriftRpcHelper::rpc(const std::string& ip, const int32_t port,
         } catch (apache::thrift::transport::TTransportException& e) {
 #ifndef ADDRESS_SANITIZER
             LOG(WARNING) << "retrying call frontend service after "
-                         << config::thrift_client_retry_interval_ms << " ms, address=" << address
-                         << ", reason=" << e.what();
+                         << config::thrift_client_retry_interval_ms << " ms, address=" << ip << ":"
+                         << port << ", reason=" << e.what();
 #else
             std::cerr << "retrying call frontend service after "
-                      << config::thrift_client_retry_interval_ms << " ms, address=" << address
-                      << ", reason=" << e.what() << std::endl;
+                      << config::thrift_client_retry_interval_ms << " ms, address=" << ip << ":"
+                      << port << ", reason=" << e.what() << std::endl;
 #endif
             std::this_thread::sleep_for(
                     std::chrono::milliseconds(config::thrift_client_retry_interval_ms));
             status = client.reopen(timeout_ms);
             if (!status.ok()) {
 #ifndef ADDRESS_SANITIZER
-                LOG(WARNING) << "client reopen failed. address=" << address
+                LOG(WARNING) << "client reopen failed. address=" << ip << ":" << port
                              << ", status=" << status;
 #else
-                std::cerr << "client reopen failed. address=" << address << ", status=" << status
-                          << std::endl;
+                std::cerr << "client reopen failed. address=" << ip << ":" << port
+                          << ", status=" << status << std::endl;
 #endif
                 return status;
             }
             callback(client);
         }
     } catch (apache::thrift::TException& e) {
 #ifndef ADDRESS_SANITIZER
-        LOG(WARNING) << "call frontend service failed, address=" << address
+        LOG(WARNING) << "call frontend service failed, address=" << ip << ":" << port
                      << ", reason=" << e.what();
 #else
-        std::cerr << "call frontend service failed, address=" << address << ", reason=" << e.what()
-                  << std::endl;
+        std::cerr << "call frontend service failed, address=" << ip << ":" << port
+                  << ", reason=" << e.what() << std::endl;
 #endif
         std::this_thread::sleep_for(
                 std::chrono::milliseconds(config::thrift_client_retry_interval_ms * 2));

docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run01.hql

@@ -0,0 +1,2 @@
+use default;
+

docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run02.hql

@@ -0,0 +1,32 @@
+CREATE TABLE `partition_table`(
+  `l_orderkey` int,
+  `l_partkey` int,
+  `l_suppkey` int,
+  `l_linenumber` int,
+  `l_quantity` decimal(12,2),
+  `l_extendedprice` decimal(12,2),
+  `l_discount` decimal(12,2),
+  `l_tax` decimal(12,2),
+  `l_returnflag` string,
+  `l_linestatus` string,
+  `l_shipdate` date,
+  `l_commitdate` date,
+  `l_receiptdate` date,
+  `l_shipinstruct` string,
+  `l_shipmode` string,
+  `l_comment` string)
+partitioned by (nation string, city string)
+ROW FORMAT SERDE
+  'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe'
+STORED AS INPUTFORMAT
+  'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat'
+OUTPUTFORMAT
+  'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat'
+LOCATION
+  '/user/doris/preinstalled_data/parquet/partition_table'
+TBLPROPERTIES (
+  'transient_lastDdlTime'='1661955829');
+
+msck repair table partition_table;
+
+

docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run03.hql

@@ -0,0 +1,23 @@
+CREATE TABLE `delta_byte_array`(
+  `c_salutation` string,
+  `c_first_name` string,
+  `c_last_name` string,
+  `c_preferred_cust_flag` string,
+  `c_birth_country` string,
+  `c_login` string,
+  `c_email_address` string,
+  `c_last_review_date` string,
+  `c_customer_id` string
+  )
+ROW FORMAT SERDE
+  'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe'
+STORED AS INPUTFORMAT
+  'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat'
+OUTPUTFORMAT
+  'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat'
+LOCATION
+  '/user/doris/preinstalled_data/different_types_parquet/delta_byte_array'
+TBLPROPERTIES (
+  'transient_lastDdlTime'='1661955829');
+
+

docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run04.hql

@@ -0,0 +1,16 @@
+CREATE TABLE `delta_length_byte_array`(
+  `FRUIT` string
+  )
+ROW FORMAT SERDE
+  'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe'
+STORED AS INPUTFORMAT
+  'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat'
+OUTPUTFORMAT
+  'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat'
+LOCATION
+  '/user/doris/preinstalled_data/different_types_parquet/delta_length_byte_array'
+TBLPROPERTIES (
+  'transient_lastDdlTime'='1661955829');
+
+msck repair table delta_length_byte_array;
+

docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run05.hql

@@ -0,0 +1,77 @@
+CREATE EXTERNAL TABLE `delta_binary_packed`(
+  bitwidth0 bigint,
+  bitwidth1 bigint,
+  bitwidth2 bigint,
+  bitwidth3 bigint,
+  bitwidth4 bigint,
+  bitwidth5 bigint,
+  bitwidth6 bigint,
+  bitwidth7 bigint,
+  bitwidth8 bigint,
+  bitwidth9 bigint,
+  bitwidth10 bigint,
+  bitwidth11 bigint,
+  bitwidth12 bigint,
+  bitwidth13 bigint,
+  bitwidth14 bigint,
+  bitwidth15 bigint,
+  bitwidth16 bigint,
+  bitwidth17 bigint,
+  bitwidth18 bigint,
+  bitwidth19 bigint,
+  bitwidth20 bigint,
+  bitwidth21 bigint,
+  bitwidth22 bigint,
+  bitwidth23 bigint,
+  bitwidth24 bigint,
+  bitwidth25 bigint,
+  bitwidth26 bigint,
+  bitwidth27 bigint,
+  bitwidth28 bigint,
+  bitwidth29 bigint,
+  bitwidth30 bigint,
+  bitwidth31 bigint,
+  bitwidth32 bigint,
+  bitwidth33 bigint,
+  bitwidth34 bigint,
+  bitwidth35 bigint,
+  bitwidth36 bigint,
+  bitwidth37 bigint,
+  bitwidth38 bigint,
+  bitwidth39 bigint,
+  bitwidth40 bigint,
+  bitwidth41 bigint,
+  bitwidth42 bigint,
+  bitwidth43 bigint,
+  bitwidth44 bigint,
+  bitwidth45 bigint,
+  bitwidth46 bigint,
+  bitwidth47 bigint,
+  bitwidth48 bigint,
+  bitwidth49 bigint,
+  bitwidth50 bigint,
+  bitwidth51 bigint,
+  bitwidth52 bigint,
+  bitwidth53 bigint,
+  bitwidth54 bigint,
+  bitwidth55 bigint,
+  bitwidth56 bigint,
+  bitwidth57 bigint,
+  bitwidth58 bigint,
+  bitwidth59 bigint,
+  bitwidth60 bigint,
+  bitwidth61 bigint,
+  bitwidth62 bigint,
+  bitwidth63 bigint,
+  bitwidth64 bigint,
+  int_value  int
+  )
+STORED AS parquet
+LOCATION
+  '/user/doris/preinstalled_data/different_types_parquet/delta_binary_packed'
+TBLPROPERTIES (
+  'transient_lastDdlTime'='1661955829');
+
+msck repair table delta_binary_packed;
+
+

docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run06.hql

@@ -0,0 +1,33 @@
+CREATE TABLE `delta_encoding_required_column`(
+   c_customer_sk int,
+    c_current_cdemo_sk int,
+   c_current_hdemo_sk int,
+   c_current_addr_sk int,
+   c_first_shipto_date_sk int,
+   c_first_sales_date_sk  int,
+   c_birth_day  int,
+   c_birth_month  int,
+     c_birth_year int,
+     c_customer_id string,
+     c_salutation string,
+     c_first_name string,
+     c_last_name string,
+     c_preferred_cust_flag string,
+     c_birth_country string,
+     c_email_address string,
+     c_last_review_date  string
+  )
+ROW FORMAT SERDE
+  'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe'
+STORED AS INPUTFORMAT
+  'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat'
+OUTPUTFORMAT
+  'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat'
+LOCATION
+  '/user/doris/preinstalled_data/different_types_parquet/delta_encoding_required_column/'
+TBLPROPERTIES (
+  'transient_lastDdlTime'='1661955829');
+
+msck repair table delta_encoding_required_column;
+
+

docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run07.hql

@@ -0,0 +1,29 @@
+CREATE EXTERNAL TABLE `delta_encoding_optional_column`(
+    c_customer_sk int,
+    c_current_cdemo_sk int,
+    c_current_hdemo_sk int,
+    c_current_addr_sk int,
+    c_first_shipto_date_sk int,
+    c_first_sales_date_sk  int,
+     c_birth_year int,
+     c_customer_id string,
+     c_salutation string,
+     c_first_name string,
+     c_last_name string,
+     c_preferred_cust_flag string,
+     c_birth_country string
+  )
+ROW FORMAT SERDE
+  'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe'
+STORED AS INPUTFORMAT
+  'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat'
+OUTPUTFORMAT
+  'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat'
+LOCATION
+  '/user/doris/preinstalled_data/different_types_parquet/delta_encoding_optional_column'
+TBLPROPERTIES (
+  'transient_lastDdlTime'='1661955829');
+
+msck repair table delta_encoding_optional_column;
+
+

docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run08.hql

@@ -0,0 +1,18 @@
+CREATE TABLE `datapage_v1_snappy_compressed_checksum`(
+  `a` int,
+  `b` int
+  )
+ROW FORMAT SERDE
+  'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe'
+STORED AS INPUTFORMAT
+  'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat'
+OUTPUTFORMAT
+  'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat'
+LOCATION
+  '/user/doris/preinstalled_data/different_types_parquet/datapage_v1-snappy-compressed-checksum'
+TBLPROPERTIES (
+  'transient_lastDdlTime'='1661955829');
+
+msck repair table datapage_v1_snappy_compressed_checksum;
+
+

docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run09.hql

@@ -0,0 +1,17 @@
+CREATE TABLE `overflow_i16_page_cnt`(
+  `inc` boolean
+  )
+ROW FORMAT SERDE
+  'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe'
+STORED AS INPUTFORMAT
+  'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat'
+OUTPUTFORMAT
+  'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat'
+LOCATION
+  '/user/doris/preinstalled_data/different_types_parquet/overflow_i16_page_cnt'
+TBLPROPERTIES (
+  'transient_lastDdlTime'='1661955829');
+
+msck repair table overflow_i16_page_cnt;
+
+

docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run10.hql

@@ -0,0 +1,29 @@
+CREATE TABLE `alltypes_tiny_pages`(
+  bool_col boolean,
+  tinyint_col int,
+  smallint_col  int,
+  int_col int,
+  bigint_col bigint,
+  float_col float,
+  double_col double,
+  id int,
+  date_string_col string,
+  string_col string,
+  timestamp_col timestamp,
+  year int,
+  month int
+  )
+ROW FORMAT SERDE
+  'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe'
+STORED AS INPUTFORMAT
+  'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat'
+OUTPUTFORMAT
+  'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat'
+LOCATION
+  '/user/doris/preinstalled_data/different_types_parquet/alltypes_tiny_pages'
+TBLPROPERTIES (
+  'transient_lastDdlTime'='1661955829');
+
+msck repair table alltypes_tiny_pages;
+
+

docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run11.hql

@@ -0,0 +1,28 @@
+CREATE TABLE `alltypes_tiny_pages_plain`(
+  bool_col boolean,
+  tinyint_col int,
+  smallint_col  int,
+  int_col int,
+  bigint_col bigint,
+  float_col float,
+  double_col double,
+  id int,
+  date_string_col string,
+  string_col string,
+  timestamp_col timestamp,
+  year int,
+  month int
+  )
+ROW FORMAT SERDE
+  'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe'
+STORED AS INPUTFORMAT
+  'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat'
+OUTPUTFORMAT
+  'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat'
+LOCATION
+  '/user/doris/preinstalled_data/different_types_parquet/alltypes_tiny_pages_plain'
+TBLPROPERTIES (
+  'transient_lastDdlTime'='1661955829');
+
+msck repair table alltypes_tiny_pages_plain;
+