From 81360cf897ded495e7b31d5e8b97dac847906809 Mon Sep 17 00:00:00 2001 From: Mingyu Chen Date: Tue, 9 Jul 2024 15:35:15 +0800 Subject: [PATCH] [opt](test) shorten the external p0 running time (#37320) (#37473) bp #37320 --- .gitignore | 2 + .licenserc.yaml | 7 +- be/src/util/thrift_rpc_helper.cpp | 23 +- .../create_tpch1_orc.hql | 0 .../create_tpch1_parquet.hql | 0 .../create_preinstalled_scripts/run01.hql | 2 + .../create_preinstalled_scripts/run02.hql | 32 + .../create_preinstalled_scripts/run03.hql | 23 + .../create_preinstalled_scripts/run04.hql | 16 + .../create_preinstalled_scripts/run05.hql | 77 + .../create_preinstalled_scripts/run06.hql | 33 + .../create_preinstalled_scripts/run07.hql | 29 + .../create_preinstalled_scripts/run08.hql | 18 + .../create_preinstalled_scripts/run09.hql | 17 + .../create_preinstalled_scripts/run10.hql | 29 + .../create_preinstalled_scripts/run11.hql | 28 + .../create_preinstalled_scripts/run12.hql | 20 + .../create_preinstalled_scripts/run13.hql | 31 + .../create_preinstalled_scripts/run14.hql | 23 + .../create_preinstalled_scripts/run15.hql | 16 + .../create_preinstalled_scripts/run16.hql | 34 + .../create_preinstalled_scripts/run17.hql | 24 + .../create_preinstalled_scripts/run18.hql | 31 + .../create_preinstalled_scripts/run19.hql | 53 + .../create_preinstalled_scripts/run25.hql | 50 + .../create_preinstalled_scripts/run29.hql | 16 + .../create_preinstalled_scripts/run30.hql | 10 + .../create_preinstalled_scripts/run31.hql | 10 + .../create_preinstalled_scripts/run32.hql | 32 + .../create_preinstalled_scripts/run33.hql | 76 + .../create_preinstalled_scripts/run34.hql | 29 + .../create_preinstalled_scripts/run35.hql | 31 + .../create_preinstalled_scripts/run36.hql | 34 + .../create_preinstalled_scripts/run37.hql | 81 + .../create_preinstalled_scripts/run38.hql | 72 + .../create_preinstalled_scripts/run39.hql | 94 + .../create_preinstalled_scripts/run40.hql | 41 + .../create_preinstalled_scripts/run41.hql | 11 + .../create_preinstalled_scripts/run42.hql | 42 + .../create_preinstalled_scripts/run43.hql | 89 + .../create_preinstalled_scripts/run44.hql | 82 + .../create_preinstalled_scripts/run45.hql | 82 + .../create_preinstalled_scripts/run46.hql | 89 + .../create_preinstalled_scripts/run47.hql | 68 + .../create_preinstalled_scripts/run48.hql | 99 + .../create_preinstalled_scripts/run49.hql | 17 + .../create_preinstalled_scripts/run50.hql | 15 + .../create_preinstalled_scripts/run51.hql | 14 + .../create_preinstalled_scripts/run52.hql | 14 + .../create_preinstalled_scripts/run53.hql | 13 + .../create_preinstalled_scripts/run54.hql | 13 + .../create_preinstalled_scripts/run55.hql | 19 + .../create_preinstalled_scripts/run56.hql | 15 + .../create_preinstalled_scripts/run57.hql | 13 + .../create_preinstalled_scripts/run58.hql | 19 + .../create_preinstalled_scripts/run59.hql | 20 + .../create_preinstalled_scripts/run60.hql | 21 + .../create_preinstalled_scripts/run61.hql | 22 + .../create_preinstalled_scripts/run62.hql | 17 + .../create_preinstalled_scripts/run63.hql | 497 ++++ .../scripts/create_preinstalled_table.hql | 2414 ----------------- .../hive_text_complex_type/run.sh | 0 .../hive_text_complex_type2/run.sh | 0 .../hive_text_complex_type3/run.sh | 0 .../hive_text_complex_type_delimiter/run.sh | 0 .../hive_text_complex_type_delimiter2/run.sh | 0 .../hive_text_complex_type_delimiter3/run.sh | 0 .../data/multi_catalog/logs1_parquet/run.sh | 0 .../orc_partitioned_columns/run.sh | 0 .../orc_partitioned_one_column/run.sh | 0 .../parquet_alter_column_to_bigint/run.sh | 0 .../parquet_alter_column_to_boolean/run.sh | 0 .../parquet_alter_column_to_char/run.sh | 0 .../parquet_alter_column_to_date/run.sh | 0 .../parquet_alter_column_to_decimal/run.sh | 0 .../parquet_alter_column_to_double/run.sh | 0 .../parquet_alter_column_to_float/run.sh | 0 .../parquet_alter_column_to_int/run.sh | 0 .../parquet_alter_column_to_smallint/run.sh | 0 .../parquet_alter_column_to_string/run.sh | 0 .../parquet_alter_column_to_timestamp/run.sh | 0 .../parquet_alter_column_to_tinyint/run.sh | 0 .../parquet_alter_column_to_varchar/run.sh | 0 .../parquet_partitioned_columns/run.sh | 0 .../parquet_partitioned_one_column/run.sh | 0 .../parquet_predicate_table/run.sh | 0 .../multi_catalog/test_chinese_orc/run.sh | 0 .../multi_catalog/test_chinese_parquet/run.sh | 0 .../multi_catalog/test_chinese_text/run.sh | 0 .../test_hive_same_db_table_name/run.sh | 0 .../test_hive_special_char_partition/run.sh | 0 .../test_mixed_par_locations_orc/run.sh | 0 .../test_mixed_par_locations_parquet/run.sh | 0 .../multi_catalog/test_multi_langs_orc/run.sh | 0 .../test_multi_langs_parquet/run.sh | 0 .../test_multi_langs_text/run.sh | 0 .../run.sh | 0 .../run.sh | 0 .../run.sh | 0 .../text_partitioned_columns/run.sh | 0 .../text_partitioned_one_column/run.sh | 0 .../data/multi_catalog/type_change_orc/run.sh | 0 .../multi_catalog/type_change_origin/run.sh | 0 .../multi_catalog/type_change_parquet/run.sh | 0 .../partition_type/bigint_partition/run.sh | 0 .../data/partition_type/char_partition/run.sh | 0 .../data/partition_type/date_partition/run.sh | 0 .../partition_type/decimal_partition/run.sh | 0 .../partition_type/double_partition/run.sh | 0 .../partition_type/float_partition/run.sh | 0 .../data/partition_type/int_partition/run.sh | 0 .../partition_type/smallint_partition/run.sh | 0 .../partition_type/string_partition/run.sh | 0 .../partition_type/tinyint_partition/run.sh | 0 .../partition_type/varchar_partition/run.sh | 0 .../data/statistics/empty_table/run.sh | 0 .../hive/scripts/data/statistics/stats/run.sh | 0 .../data/tpch_1000_parquet/part/run.sh | 0 .../data/tvf/test_hdfs_tvf_compression/run.sh | 0 .../hive/scripts/data/tvf/test_tvf/run.sh | 0 .../hive/scripts/hive-metastore.sh | 25 +- 121 files changed, 2435 insertions(+), 2439 deletions(-) rename docker/thirdparties/docker-compose/hive/scripts/{ => create_preinstalled_scripts}/create_tpch1_orc.hql (100%) rename docker/thirdparties/docker-compose/hive/scripts/{ => create_preinstalled_scripts}/create_tpch1_parquet.hql (100%) create mode 100755 docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run01.hql create mode 100755 docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run02.hql create mode 100755 docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run03.hql create mode 100755 docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run04.hql create mode 100755 docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run05.hql create mode 100755 docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run06.hql create mode 100755 docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run07.hql create mode 100755 docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run08.hql create mode 100755 docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run09.hql create mode 100755 docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run10.hql create mode 100755 docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run11.hql create mode 100755 docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run12.hql create mode 100755 docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run13.hql create mode 100755 docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run14.hql create mode 100755 docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run15.hql create mode 100755 docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run16.hql create mode 100755 docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run17.hql create mode 100755 docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run18.hql create mode 100755 docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run19.hql create mode 100755 docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run25.hql create mode 100755 docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run29.hql create mode 100755 docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run30.hql create mode 100755 docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run31.hql create mode 100755 docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run32.hql create mode 100755 docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run33.hql create mode 100755 docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run34.hql create mode 100755 docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run35.hql create mode 100755 docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run36.hql create mode 100755 docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run37.hql create mode 100755 docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run38.hql create mode 100755 docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run39.hql create mode 100755 docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run40.hql create mode 100755 docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run41.hql create mode 100755 docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run42.hql create mode 100755 docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run43.hql create mode 100755 docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run44.hql create mode 100755 docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run45.hql create mode 100755 docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run46.hql create mode 100755 docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run47.hql create mode 100755 docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run48.hql create mode 100755 docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run49.hql create mode 100755 docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run50.hql create mode 100755 docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run51.hql create mode 100755 docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run52.hql create mode 100755 docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run53.hql create mode 100755 docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run54.hql create mode 100755 docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run55.hql create mode 100755 docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run56.hql create mode 100755 docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run57.hql create mode 100755 docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run58.hql create mode 100755 docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run59.hql create mode 100755 docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run60.hql create mode 100755 docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run61.hql create mode 100755 docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run62.hql create mode 100755 docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run63.hql delete mode 100644 docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_table.hql mode change 100644 => 100755 docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/hive_text_complex_type/run.sh mode change 100644 => 100755 docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/hive_text_complex_type2/run.sh mode change 100644 => 100755 docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/hive_text_complex_type3/run.sh mode change 100644 => 100755 docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/hive_text_complex_type_delimiter/run.sh mode change 100644 => 100755 docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/hive_text_complex_type_delimiter2/run.sh mode change 100644 => 100755 docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/hive_text_complex_type_delimiter3/run.sh mode change 100644 => 100755 docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/logs1_parquet/run.sh mode change 100644 => 100755 docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/orc_partitioned_columns/run.sh mode change 100644 => 100755 docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/orc_partitioned_one_column/run.sh mode change 100644 => 100755 docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/parquet_alter_column_to_bigint/run.sh mode change 100644 => 100755 docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/parquet_alter_column_to_boolean/run.sh mode change 100644 => 100755 docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/parquet_alter_column_to_char/run.sh mode change 100644 => 100755 docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/parquet_alter_column_to_date/run.sh mode change 100644 => 100755 docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/parquet_alter_column_to_decimal/run.sh mode change 100644 => 100755 docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/parquet_alter_column_to_double/run.sh mode change 100644 => 100755 docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/parquet_alter_column_to_float/run.sh mode change 100644 => 100755 docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/parquet_alter_column_to_int/run.sh mode change 100644 => 100755 docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/parquet_alter_column_to_smallint/run.sh mode change 100644 => 100755 docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/parquet_alter_column_to_string/run.sh mode change 100644 => 100755 docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/parquet_alter_column_to_timestamp/run.sh mode change 100644 => 100755 docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/parquet_alter_column_to_tinyint/run.sh mode change 100644 => 100755 docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/parquet_alter_column_to_varchar/run.sh mode change 100644 => 100755 docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/parquet_partitioned_columns/run.sh mode change 100644 => 100755 docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/parquet_partitioned_one_column/run.sh mode change 100644 => 100755 docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/parquet_predicate_table/run.sh mode change 100644 => 100755 docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/test_chinese_orc/run.sh mode change 100644 => 100755 docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/test_chinese_parquet/run.sh mode change 100644 => 100755 docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/test_chinese_text/run.sh mode change 100644 => 100755 docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/test_hive_same_db_table_name/run.sh mode change 100644 => 100755 docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/test_hive_special_char_partition/run.sh mode change 100644 => 100755 docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/test_mixed_par_locations_orc/run.sh mode change 100644 => 100755 docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/test_mixed_par_locations_parquet/run.sh mode change 100644 => 100755 docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/test_multi_langs_orc/run.sh mode change 100644 => 100755 docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/test_multi_langs_parquet/run.sh mode change 100644 => 100755 docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/test_multi_langs_text/run.sh mode change 100644 => 100755 docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/test_truncate_char_or_varchar_columns_orc/run.sh mode change 100644 => 100755 docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/test_truncate_char_or_varchar_columns_parquet/run.sh mode change 100644 => 100755 docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/test_truncate_char_or_varchar_columns_text/run.sh mode change 100644 => 100755 docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/text_partitioned_columns/run.sh mode change 100644 => 100755 docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/text_partitioned_one_column/run.sh mode change 100644 => 100755 docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/type_change_orc/run.sh mode change 100644 => 100755 docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/type_change_origin/run.sh mode change 100644 => 100755 docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/type_change_parquet/run.sh mode change 100644 => 100755 docker/thirdparties/docker-compose/hive/scripts/data/partition_type/bigint_partition/run.sh mode change 100644 => 100755 docker/thirdparties/docker-compose/hive/scripts/data/partition_type/char_partition/run.sh mode change 100644 => 100755 docker/thirdparties/docker-compose/hive/scripts/data/partition_type/date_partition/run.sh mode change 100644 => 100755 docker/thirdparties/docker-compose/hive/scripts/data/partition_type/decimal_partition/run.sh mode change 100644 => 100755 docker/thirdparties/docker-compose/hive/scripts/data/partition_type/double_partition/run.sh mode change 100644 => 100755 docker/thirdparties/docker-compose/hive/scripts/data/partition_type/float_partition/run.sh mode change 100644 => 100755 docker/thirdparties/docker-compose/hive/scripts/data/partition_type/int_partition/run.sh mode change 100644 => 100755 docker/thirdparties/docker-compose/hive/scripts/data/partition_type/smallint_partition/run.sh mode change 100644 => 100755 docker/thirdparties/docker-compose/hive/scripts/data/partition_type/string_partition/run.sh mode change 100644 => 100755 docker/thirdparties/docker-compose/hive/scripts/data/partition_type/tinyint_partition/run.sh mode change 100644 => 100755 docker/thirdparties/docker-compose/hive/scripts/data/partition_type/varchar_partition/run.sh mode change 100644 => 100755 docker/thirdparties/docker-compose/hive/scripts/data/statistics/empty_table/run.sh mode change 100644 => 100755 docker/thirdparties/docker-compose/hive/scripts/data/statistics/stats/run.sh mode change 100644 => 100755 docker/thirdparties/docker-compose/hive/scripts/data/tpch_1000_parquet/part/run.sh mode change 100644 => 100755 docker/thirdparties/docker-compose/hive/scripts/data/tvf/test_hdfs_tvf_compression/run.sh mode change 100644 => 100755 docker/thirdparties/docker-compose/hive/scripts/data/tvf/test_tvf/run.sh diff --git a/.gitignore b/.gitignore index 79d99ef6d148f7..9f4844d7e9326e 100644 --- a/.gitignore +++ b/.gitignore @@ -60,6 +60,8 @@ docker/thirdparties/docker-compose/hive/scripts/paimon1 fe_plugins/output fe_plugins/**/.factorypath +docker/thirdparties/docker-compose/hive/scripts/data/*/*/data + fs_brokers/apache_hdfs_broker/src/main/resources/ fs_brokers/apache_hdfs_broker/src/main/thrift/ diff --git a/.licenserc.yaml b/.licenserc.yaml index 7412436a9c15c7..066f8a90813742 100644 --- a/.licenserc.yaml +++ b/.licenserc.yaml @@ -77,12 +77,7 @@ header: - "docs/package-lock.json" - "regression-test/script/README" - "regression-test/suites/load_p0/stream_load/data" - - "docker/thirdparties/docker-compose/hive/scripts/README" - - "docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_table.hql" - - "docker/thirdparties/docker-compose/hive/scripts/create_tpch1_orc.hql" - - "docker/thirdparties/docker-compose/hive/scripts/create_tpch1_parquet.hql" - - "docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/" - - "docker/thirdparties/docker-compose/hive/scripts/data/**" + - "docker/thirdparties/docker-compose/hive/scripts/**" - "docker/thirdparties/docker-compose/iceberg/spark-defaults.conf.tpl" - "conf/mysql_ssl_default_certificate/*" - "conf/mysql_ssl_default_certificate/client_certificate/ca.pem" diff --git a/be/src/util/thrift_rpc_helper.cpp b/be/src/util/thrift_rpc_helper.cpp index 7904fc0b7b3452..bbad6e09a64d79 100644 --- a/be/src/util/thrift_rpc_helper.cpp +++ b/be/src/util/thrift_rpc_helper.cpp @@ -66,7 +66,8 @@ Status ThriftRpcHelper::rpc(const std::string& ip, const int32_t port, Status status; ClientConnection client(_s_exec_env->get_client_cache(), address, timeout_ms, &status); if (!status.ok()) { - LOG(WARNING) << "Connect frontend failed, address=" << address << ", status=" << status; + LOG(WARNING) << "Connect frontend failed, address=" << ip << ":" << port + << ", status=" << status; return status; } try { @@ -75,23 +76,23 @@ Status ThriftRpcHelper::rpc(const std::string& ip, const int32_t port, } catch (apache::thrift::transport::TTransportException& e) { #ifndef ADDRESS_SANITIZER LOG(WARNING) << "retrying call frontend service after " - << config::thrift_client_retry_interval_ms << " ms, address=" << address - << ", reason=" << e.what(); + << config::thrift_client_retry_interval_ms << " ms, address=" << ip << ":" + << port << ", reason=" << e.what(); #else std::cerr << "retrying call frontend service after " - << config::thrift_client_retry_interval_ms << " ms, address=" << address - << ", reason=" << e.what() << std::endl; + << config::thrift_client_retry_interval_ms << " ms, address=" << ip << ":" + << port << ", reason=" << e.what() << std::endl; #endif std::this_thread::sleep_for( std::chrono::milliseconds(config::thrift_client_retry_interval_ms)); status = client.reopen(timeout_ms); if (!status.ok()) { #ifndef ADDRESS_SANITIZER - LOG(WARNING) << "client reopen failed. address=" << address + LOG(WARNING) << "client reopen failed. address=" << ip << ":" << port << ", status=" << status; #else - std::cerr << "client reopen failed. address=" << address << ", status=" << status - << std::endl; + std::cerr << "client reopen failed. address=" << ip << ":" << port + << ", status=" << status << std::endl; #endif return status; } @@ -99,11 +100,11 @@ Status ThriftRpcHelper::rpc(const std::string& ip, const int32_t port, } } catch (apache::thrift::TException& e) { #ifndef ADDRESS_SANITIZER - LOG(WARNING) << "call frontend service failed, address=" << address + LOG(WARNING) << "call frontend service failed, address=" << ip << ":" << port << ", reason=" << e.what(); #else - std::cerr << "call frontend service failed, address=" << address << ", reason=" << e.what() - << std::endl; + std::cerr << "call frontend service failed, address=" << ip << ":" << port + << ", reason=" << e.what() << std::endl; #endif std::this_thread::sleep_for( std::chrono::milliseconds(config::thrift_client_retry_interval_ms * 2)); diff --git a/docker/thirdparties/docker-compose/hive/scripts/create_tpch1_orc.hql b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/create_tpch1_orc.hql similarity index 100% rename from docker/thirdparties/docker-compose/hive/scripts/create_tpch1_orc.hql rename to docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/create_tpch1_orc.hql diff --git a/docker/thirdparties/docker-compose/hive/scripts/create_tpch1_parquet.hql b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/create_tpch1_parquet.hql similarity index 100% rename from docker/thirdparties/docker-compose/hive/scripts/create_tpch1_parquet.hql rename to docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/create_tpch1_parquet.hql diff --git a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run01.hql b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run01.hql new file mode 100755 index 00000000000000..83718d11b1f81a --- /dev/null +++ b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run01.hql @@ -0,0 +1,2 @@ +use default; + diff --git a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run02.hql b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run02.hql new file mode 100755 index 00000000000000..1cb4b1d03a3de4 --- /dev/null +++ b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run02.hql @@ -0,0 +1,32 @@ +CREATE TABLE `partition_table`( + `l_orderkey` int, + `l_partkey` int, + `l_suppkey` int, + `l_linenumber` int, + `l_quantity` decimal(12,2), + `l_extendedprice` decimal(12,2), + `l_discount` decimal(12,2), + `l_tax` decimal(12,2), + `l_returnflag` string, + `l_linestatus` string, + `l_shipdate` date, + `l_commitdate` date, + `l_receiptdate` date, + `l_shipinstruct` string, + `l_shipmode` string, + `l_comment` string) +partitioned by (nation string, city string) +ROW FORMAT SERDE + 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' +STORED AS INPUTFORMAT + 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat' +OUTPUTFORMAT + 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat' +LOCATION + '/user/doris/preinstalled_data/parquet/partition_table' +TBLPROPERTIES ( + 'transient_lastDdlTime'='1661955829'); + +msck repair table partition_table; + + diff --git a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run03.hql b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run03.hql new file mode 100755 index 00000000000000..564755fdc70596 --- /dev/null +++ b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run03.hql @@ -0,0 +1,23 @@ +CREATE TABLE `delta_byte_array`( + `c_salutation` string, + `c_first_name` string, + `c_last_name` string, + `c_preferred_cust_flag` string, + `c_birth_country` string, + `c_login` string, + `c_email_address` string, + `c_last_review_date` string, + `c_customer_id` string + ) +ROW FORMAT SERDE + 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' +STORED AS INPUTFORMAT + 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat' +OUTPUTFORMAT + 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat' +LOCATION + '/user/doris/preinstalled_data/different_types_parquet/delta_byte_array' +TBLPROPERTIES ( + 'transient_lastDdlTime'='1661955829'); + + diff --git a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run04.hql b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run04.hql new file mode 100755 index 00000000000000..e6ecac80e8b9ba --- /dev/null +++ b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run04.hql @@ -0,0 +1,16 @@ +CREATE TABLE `delta_length_byte_array`( + `FRUIT` string + ) +ROW FORMAT SERDE + 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' +STORED AS INPUTFORMAT + 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat' +OUTPUTFORMAT + 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat' +LOCATION + '/user/doris/preinstalled_data/different_types_parquet/delta_length_byte_array' +TBLPROPERTIES ( + 'transient_lastDdlTime'='1661955829'); + +msck repair table delta_length_byte_array; + diff --git a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run05.hql b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run05.hql new file mode 100755 index 00000000000000..cb2e4a3772553c --- /dev/null +++ b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run05.hql @@ -0,0 +1,77 @@ +CREATE EXTERNAL TABLE `delta_binary_packed`( + bitwidth0 bigint, + bitwidth1 bigint, + bitwidth2 bigint, + bitwidth3 bigint, + bitwidth4 bigint, + bitwidth5 bigint, + bitwidth6 bigint, + bitwidth7 bigint, + bitwidth8 bigint, + bitwidth9 bigint, + bitwidth10 bigint, + bitwidth11 bigint, + bitwidth12 bigint, + bitwidth13 bigint, + bitwidth14 bigint, + bitwidth15 bigint, + bitwidth16 bigint, + bitwidth17 bigint, + bitwidth18 bigint, + bitwidth19 bigint, + bitwidth20 bigint, + bitwidth21 bigint, + bitwidth22 bigint, + bitwidth23 bigint, + bitwidth24 bigint, + bitwidth25 bigint, + bitwidth26 bigint, + bitwidth27 bigint, + bitwidth28 bigint, + bitwidth29 bigint, + bitwidth30 bigint, + bitwidth31 bigint, + bitwidth32 bigint, + bitwidth33 bigint, + bitwidth34 bigint, + bitwidth35 bigint, + bitwidth36 bigint, + bitwidth37 bigint, + bitwidth38 bigint, + bitwidth39 bigint, + bitwidth40 bigint, + bitwidth41 bigint, + bitwidth42 bigint, + bitwidth43 bigint, + bitwidth44 bigint, + bitwidth45 bigint, + bitwidth46 bigint, + bitwidth47 bigint, + bitwidth48 bigint, + bitwidth49 bigint, + bitwidth50 bigint, + bitwidth51 bigint, + bitwidth52 bigint, + bitwidth53 bigint, + bitwidth54 bigint, + bitwidth55 bigint, + bitwidth56 bigint, + bitwidth57 bigint, + bitwidth58 bigint, + bitwidth59 bigint, + bitwidth60 bigint, + bitwidth61 bigint, + bitwidth62 bigint, + bitwidth63 bigint, + bitwidth64 bigint, + int_value int + ) +STORED AS parquet +LOCATION + '/user/doris/preinstalled_data/different_types_parquet/delta_binary_packed' +TBLPROPERTIES ( + 'transient_lastDdlTime'='1661955829'); + +msck repair table delta_binary_packed; + + diff --git a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run06.hql b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run06.hql new file mode 100755 index 00000000000000..a99e8fa29d9d93 --- /dev/null +++ b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run06.hql @@ -0,0 +1,33 @@ +CREATE TABLE `delta_encoding_required_column`( + c_customer_sk int, + c_current_cdemo_sk int, + c_current_hdemo_sk int, + c_current_addr_sk int, + c_first_shipto_date_sk int, + c_first_sales_date_sk int, + c_birth_day int, + c_birth_month int, + c_birth_year int, + c_customer_id string, + c_salutation string, + c_first_name string, + c_last_name string, + c_preferred_cust_flag string, + c_birth_country string, + c_email_address string, + c_last_review_date string + ) +ROW FORMAT SERDE + 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' +STORED AS INPUTFORMAT + 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat' +OUTPUTFORMAT + 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat' +LOCATION + '/user/doris/preinstalled_data/different_types_parquet/delta_encoding_required_column/' +TBLPROPERTIES ( + 'transient_lastDdlTime'='1661955829'); + +msck repair table delta_encoding_required_column; + + diff --git a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run07.hql b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run07.hql new file mode 100755 index 00000000000000..ffceea62a78b73 --- /dev/null +++ b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run07.hql @@ -0,0 +1,29 @@ +CREATE EXTERNAL TABLE `delta_encoding_optional_column`( + c_customer_sk int, + c_current_cdemo_sk int, + c_current_hdemo_sk int, + c_current_addr_sk int, + c_first_shipto_date_sk int, + c_first_sales_date_sk int, + c_birth_year int, + c_customer_id string, + c_salutation string, + c_first_name string, + c_last_name string, + c_preferred_cust_flag string, + c_birth_country string + ) +ROW FORMAT SERDE + 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' +STORED AS INPUTFORMAT + 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat' +OUTPUTFORMAT + 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat' +LOCATION + '/user/doris/preinstalled_data/different_types_parquet/delta_encoding_optional_column' +TBLPROPERTIES ( + 'transient_lastDdlTime'='1661955829'); + +msck repair table delta_encoding_optional_column; + + diff --git a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run08.hql b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run08.hql new file mode 100755 index 00000000000000..d1a19a331903d6 --- /dev/null +++ b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run08.hql @@ -0,0 +1,18 @@ +CREATE TABLE `datapage_v1_snappy_compressed_checksum`( + `a` int, + `b` int + ) +ROW FORMAT SERDE + 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' +STORED AS INPUTFORMAT + 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat' +OUTPUTFORMAT + 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat' +LOCATION + '/user/doris/preinstalled_data/different_types_parquet/datapage_v1-snappy-compressed-checksum' +TBLPROPERTIES ( + 'transient_lastDdlTime'='1661955829'); + +msck repair table datapage_v1_snappy_compressed_checksum; + + diff --git a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run09.hql b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run09.hql new file mode 100755 index 00000000000000..20d3a308117253 --- /dev/null +++ b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run09.hql @@ -0,0 +1,17 @@ +CREATE TABLE `overflow_i16_page_cnt`( + `inc` boolean + ) +ROW FORMAT SERDE + 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' +STORED AS INPUTFORMAT + 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat' +OUTPUTFORMAT + 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat' +LOCATION + '/user/doris/preinstalled_data/different_types_parquet/overflow_i16_page_cnt' +TBLPROPERTIES ( + 'transient_lastDdlTime'='1661955829'); + +msck repair table overflow_i16_page_cnt; + + diff --git a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run10.hql b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run10.hql new file mode 100755 index 00000000000000..633a0883161012 --- /dev/null +++ b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run10.hql @@ -0,0 +1,29 @@ +CREATE TABLE `alltypes_tiny_pages`( + bool_col boolean, + tinyint_col int, + smallint_col int, + int_col int, + bigint_col bigint, + float_col float, + double_col double, + id int, + date_string_col string, + string_col string, + timestamp_col timestamp, + year int, + month int + ) +ROW FORMAT SERDE + 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' +STORED AS INPUTFORMAT + 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat' +OUTPUTFORMAT + 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat' +LOCATION + '/user/doris/preinstalled_data/different_types_parquet/alltypes_tiny_pages' +TBLPROPERTIES ( + 'transient_lastDdlTime'='1661955829'); + +msck repair table alltypes_tiny_pages; + + diff --git a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run11.hql b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run11.hql new file mode 100755 index 00000000000000..760271c42b571a --- /dev/null +++ b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run11.hql @@ -0,0 +1,28 @@ +CREATE TABLE `alltypes_tiny_pages_plain`( + bool_col boolean, + tinyint_col int, + smallint_col int, + int_col int, + bigint_col bigint, + float_col float, + double_col double, + id int, + date_string_col string, + string_col string, + timestamp_col timestamp, + year int, + month int + ) +ROW FORMAT SERDE + 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' +STORED AS INPUTFORMAT + 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat' +OUTPUTFORMAT + 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat' +LOCATION + '/user/doris/preinstalled_data/different_types_parquet/alltypes_tiny_pages_plain' +TBLPROPERTIES ( + 'transient_lastDdlTime'='1661955829'); + +msck repair table alltypes_tiny_pages_plain; + diff --git a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run12.hql b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run12.hql new file mode 100755 index 00000000000000..27b51014bfe7a9 --- /dev/null +++ b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run12.hql @@ -0,0 +1,20 @@ +CREATE TABLE `example_string`( + `strings` string + ) +ROW FORMAT SERDE + 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' +WITH SERDEPROPERTIES ( + 'field.delim'='\t', + 'serialization.format'='\t') +STORED AS INPUTFORMAT + 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat' +OUTPUTFORMAT + 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat' +LOCATION + '/user/doris/preinstalled_data/example_string.parquet' +TBLPROPERTIES ( + 'transient_lastDdlTime'='1661955829'); + +msck repair table example_string; + + diff --git a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run13.hql b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run13.hql new file mode 100755 index 00000000000000..1be4d8b1fb55f8 --- /dev/null +++ b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run13.hql @@ -0,0 +1,31 @@ +CREATE EXTERNAL TABLE IF NOT EXISTS `orc_all_types`( + `tinyint_col` tinyint, + `smallint_col` smallint, + `int_col` int, + `bigint_col` bigint, + `boolean_col` boolean, + `float_col` float, + `double_col` double, + `string_col` string, + `binary_col` binary, + `timestamp_col` timestamp, + `decimal_col` decimal(12,4), + `char_col` char(50), + `varchar_col` varchar(50), + `date_col` date, + `list_double_col` array, + `list_string_col` array) +PARTITIONED BY ( + `p1_col` string, + `p2_col` string) +ROW FORMAT SERDE + 'org.apache.hadoop.hive.ql.io.orc.OrcSerde' +STORED AS INPUTFORMAT + 'org.apache.hadoop.hive.ql.io.orc.OrcInputFormat' +OUTPUTFORMAT + 'org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat' +LOCATION + '/user/doris/preinstalled_data/orc/orc_all_types'; + +msck repair table orc_all_types; + diff --git a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run14.hql b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run14.hql new file mode 100755 index 00000000000000..fa4e6d3d73697d --- /dev/null +++ b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run14.hql @@ -0,0 +1,23 @@ +CREATE TABLE `lineorder` ( + `lo_orderkey` int, + `lo_linenumber` int, + `lo_custkey` int, + `lo_partkey` int, + `lo_suppkey` int, + `lo_orderdate` int, + `lo_orderpriority` varchar(16), + `lo_shippriority` int, + `lo_quantity` int, + `lo_extendedprice` int, + `lo_ordtotalprice` int, + `lo_discount` int, + `lo_revenue` int, + `lo_supplycost` int, + `lo_tax` int, + `lo_commitdate` int, + `lo_shipmode` varchar(11) +) +ROW FORMAT DELIMITED FIELDS TERMINATED by ',' +LOCATION '/user/doris/preinstalled_data/data_case/lineorder' +TBLPROPERTIES ('transient_lastDdlTime'='1658816839'); + diff --git a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run15.hql b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run15.hql new file mode 100755 index 00000000000000..e46542e8f2e5e7 --- /dev/null +++ b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run15.hql @@ -0,0 +1,16 @@ +create table t_hive ( + `k1` int, + `k2` char(10), + `k3` date, + `k5` varchar(20), + `k6` double +) +ROW FORMAT DELIMITED FIELDS TERMINATED by ',' +STORED AS INPUTFORMAT + 'org.apache.hadoop.mapred.TextInputFormat' +OUTPUTFORMAT + 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat' +LOCATION + '/user/doris/preinstalled_data/data_case/t_hive' +TBLPROPERTIES ('transient_lastDdlTime'='1658816839'); + diff --git a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run16.hql b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run16.hql new file mode 100755 index 00000000000000..c2ba60b9431581 --- /dev/null +++ b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run16.hql @@ -0,0 +1,34 @@ +CREATE external TABLE `table_with_vertical_line`( + `k1` string COMMENT 'k1', + `k2` string COMMENT 'k2', + `k3` string COMMENT 'k3', + `k4` string COMMENT 'k4', + `k5` string COMMENT 'k5', + `k6` string COMMENT 'k6', + `k7` string COMMENT 'k7', + `k8` string COMMENT 'k8', + `k9` string COMMENT 'k9', + `k10` string COMMENT 'k10', + `k11` string COMMENT 'k11', + `k12` string COMMENT 'k12', + `k13` string COMMENT 'k13', + `k14` string COMMENT 'k14', + `k15` string COMMENT 'k15') +PARTITIONED BY ( + `dt` string) +ROW FORMAT SERDE + 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' +WITH SERDEPROPERTIES ( + 'field.delim'='|', + 'serialization.format'='|') +STORED AS INPUTFORMAT + 'org.apache.hadoop.mapred.TextInputFormat' +OUTPUTFORMAT + 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' +LOCATION + '/user/doris/preinstalled_data/csv_partition_table/table_with_vertical_line/' +TBLPROPERTIES ( + 'transient_lastDdlTime'='1669304897'); + +msck repair table table_with_vertical_line; + diff --git a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run17.hql b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run17.hql new file mode 100755 index 00000000000000..fff23d1d297508 --- /dev/null +++ b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run17.hql @@ -0,0 +1,24 @@ +CREATE external TABLE `table_with_pars`( + `id` int COMMENT 'id', + `data` string COMMENT 'data') +PARTITIONED BY ( + `dt_par` date, + `time_par` timestamp, + `decimal_par1` decimal(8, 4), + `decimal_par2` decimal(18, 6), + `decimal_par3` decimal(38, 12)) +ROW FORMAT SERDE + 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' +WITH SERDEPROPERTIES ( + 'field.delim'='|', + 'serialization.format'='|') +STORED AS INPUTFORMAT + 'org.apache.hadoop.mapred.TextInputFormat' +OUTPUTFORMAT + 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' +LOCATION + '/user/doris/preinstalled_data/csv_partition_table/table_with_pars/'; + +set hive.msck.path.validation=ignore; +msck repair table table_with_pars; + diff --git a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run18.hql b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run18.hql new file mode 100755 index 00000000000000..91d099d6dc1221 --- /dev/null +++ b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run18.hql @@ -0,0 +1,31 @@ +CREATE TABLE `table_with_x01`( + `k1` string COMMENT 'k1', + `k2` string COMMENT 'k2', + `k3` string COMMENT 'k3', + `k4` string COMMENT 'k4', + `k5` string COMMENT 'k5', + `k6` string COMMENT 'k6', + `k7` string COMMENT 'k7', + `k8` string COMMENT 'k8', + `k9` string COMMENT 'k9', + `k10` string COMMENT 'k10', + `k11` string COMMENT 'k11', + `k12` string COMMENT 'k12', + `k13` string COMMENT 'k13', + `k14` string COMMENT 'k14', + `k15` string COMMENT 'k15') +PARTITIONED BY ( + `dt` string) +ROW FORMAT SERDE + 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' +STORED AS INPUTFORMAT + 'org.apache.hadoop.mapred.TextInputFormat' +OUTPUTFORMAT + 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' +LOCATION + '/user/doris/preinstalled_data/csv_partition_table/table_with_x01/' +TBLPROPERTIES ( + 'transient_lastDdlTime'='1669360080'); + +msck repair table table_with_x01; + diff --git a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run19.hql b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run19.hql new file mode 100755 index 00000000000000..dda8ca2008b8cb --- /dev/null +++ b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run19.hql @@ -0,0 +1,53 @@ +set hive.stats.column.autogather=false; + +CREATE TABLE `test_hive_orc_add_column`( + id int, + col1 int +) +stored as orc; +insert into `test_hive_orc_add_column` values(1,2),(3,4),(4,6); +alter table `test_hive_orc_add_column` ADD COLUMNS (col2 int); +insert into `test_hive_orc_add_column` values(7,8,9),(10,11,null),(12,13,null),(14,15,16); +alter table `test_hive_orc_add_column` ADD COLUMNS (col3 int,col4 string); +insert into `test_hive_orc_add_column` values(17,18,19,20,"hello world"),(21,22,23,24,"cywcywcyw"),(25,26,null,null,null),(27,28,29,null,null),(30,31,32,33,null); + +CREATE TABLE `test_hive_parquet_add_column`( + id int, + col1 int +) +stored as parquet; +insert into `test_hive_parquet_add_column` values(1,2),(3,4),(4,6); +alter table `test_hive_parquet_add_column` ADD COLUMNS (col2 int); +insert into `test_hive_parquet_add_column` values(7,8,9),(10,11,null),(12,13,null),(14,15,16); +alter table `test_hive_parquet_add_column` ADD COLUMNS (col3 int,col4 string); +insert into `test_hive_parquet_add_column` values(17,18,19,20,"hello world"),(21,22,23,24,"cywcywcyw"),(25,26,null,null,null),(27,28,29,null,null),(30,31,32,33,null); + +CREATE TABLE `schema_evo_test_text`( + id int, + name string +) +ROW FORMAT DELIMITED FIELDS TERMINATED by ','; +insert into `schema_evo_test_text` select 1, "kaka"; +alter table `schema_evo_test_text` ADD COLUMNS (`ts` timestamp); +insert into `schema_evo_test_text` select 2, "messi", from_unixtime(to_unix_timestamp('20230101 13:01:03','yyyyMMdd HH:mm:ss')); + +CREATE TABLE `schema_evo_test_parquet`( + id int, + name string +) +stored as parquet; +insert into `schema_evo_test_parquet` select 1, "kaka"; +alter table `schema_evo_test_parquet` ADD COLUMNS (`ts` timestamp); +insert into `schema_evo_test_parquet` select 2, "messi", from_unixtime(to_unix_timestamp('20230101 13:01:03','yyyyMMdd HH:mm:ss')); + +CREATE TABLE `schema_evo_test_orc`( + id int, + name string +) +stored as orc; +insert into `schema_evo_test_orc` select 1, "kaka"; +alter table `schema_evo_test_orc` ADD COLUMNS (`ts` timestamp); +insert into `schema_evo_test_orc` select 2, "messi", from_unixtime(to_unix_timestamp('20230101 13:01:03','yyyyMMdd HH:mm:ss')); + +set hive.stats.column.autogather=true; + diff --git a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run25.hql b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run25.hql new file mode 100755 index 00000000000000..2cf7197de95775 --- /dev/null +++ b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run25.hql @@ -0,0 +1,50 @@ +-- Currently docker is hive 2.x version. Hive 2.x versioned full-acid tables need to run major compaction. +SET hive.support.concurrency=true; +SET hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; + +create table orc_full_acid_empty (id INT, value STRING) +CLUSTERED BY (id) INTO 3 BUCKETS +STORED AS ORC +TBLPROPERTIES ('transactional' = 'true'); + +create table orc_full_acid_par_empty (id INT, value STRING) +PARTITIONED BY (part_col INT) +CLUSTERED BY (id) INTO 3 BUCKETS +STORED AS ORC +TBLPROPERTIES ('transactional' = 'true'); + +create table orc_full_acid (id INT, value STRING) +CLUSTERED BY (id) INTO 3 BUCKETS +STORED AS ORC +TBLPROPERTIES ('transactional' = 'true'); + +insert into orc_full_acid values +(1, 'A'), +(2, 'B'), +(3, 'C'); + +update orc_full_acid set value = 'CC' where id = 3; + +alter table orc_full_acid compact 'major'; + +create table orc_full_acid_par (id INT, value STRING) +PARTITIONED BY (part_col INT) +CLUSTERED BY (id) INTO 3 BUCKETS +STORED AS ORC +TBLPROPERTIES ('transactional' = 'true'); + +insert into orc_full_acid_par PARTITION(part_col=20230101) values +(1, 'A'), +(2, 'B'), +(3, 'C'); + +insert into orc_full_acid_par PARTITION(part_col=20230102) values +(4, 'D'), +(5, 'E'), +(6, 'F'); + +update orc_full_acid_par set value = 'BB' where id = 2; + +alter table orc_full_acid_par PARTITION(part_col=20230101) compact 'major'; +alter table orc_full_acid_par PARTITION(part_col=20230102) compact 'major'; + diff --git a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run29.hql b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run29.hql new file mode 100755 index 00000000000000..5c1b00e3e428d1 --- /dev/null +++ b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run29.hql @@ -0,0 +1,16 @@ +create table mtmv_base1 (id INT, value STRING) + PARTITIONED BY (part_col INT) + CLUSTERED BY (id) INTO 3 BUCKETS + STORED AS ORC; + +insert into mtmv_base1 PARTITION(part_col=20230101) values +(1, 'A'), +(2, 'B'), +(3, 'C'); + +insert into mtmv_base1 PARTITION(part_col=20230102) values +(4, 'D'), +(5, 'E'), +(6, 'F'); + + diff --git a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run30.hql b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run30.hql new file mode 100755 index 00000000000000..2cda09262230db --- /dev/null +++ b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run30.hql @@ -0,0 +1,10 @@ +CREATE TABLE `test_different_column_orders_orc`( + `name` string, + `id` int, + `city` string, + `age` int, + `sex` string) +STORED AS ORC +LOCATION + '/user/doris/preinstalled_data/test_different_column_orders/orc'; + diff --git a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run31.hql b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run31.hql new file mode 100755 index 00000000000000..0a7bcbf44f18f5 --- /dev/null +++ b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run31.hql @@ -0,0 +1,10 @@ +CREATE TABLE `test_different_column_orders_parquet`( + `name` string, + `id` int, + `city` string, + `age` int, + `sex` string) +STORED AS PARQUET +LOCATION + '/user/doris/preinstalled_data/test_different_column_orders/parquet'; + diff --git a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run32.hql b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run32.hql new file mode 100755 index 00000000000000..2c4a2c413b1b17 --- /dev/null +++ b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run32.hql @@ -0,0 +1,32 @@ +CREATE TABLE `parquet_partition_table`( + `l_orderkey` int, + `l_partkey` int, + `l_suppkey` int, + `l_linenumber` int, + `l_quantity` decimal(12,2), + `l_extendedprice` decimal(12,2), + `l_discount` decimal(12,2), + `l_tax` decimal(12,2), + `l_returnflag` string, + `l_linestatus` string, + `l_shipdate` date, + `l_commitdate` date, + `l_receiptdate` date, + `l_shipinstruct` string, + `l_shipmode` string, + `l_comment` string) +partitioned by (nation string, city string) +ROW FORMAT SERDE + 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' +STORED AS INPUTFORMAT + 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat' +OUTPUTFORMAT + 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat' +LOCATION + '/user/doris/preinstalled_data/parquet_table/parquet_partition_table' +TBLPROPERTIES ( + 'transient_lastDdlTime'='1661955829'); + +msck repair table parquet_partition_table; + + diff --git a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run33.hql b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run33.hql new file mode 100755 index 00000000000000..970b4c5437db17 --- /dev/null +++ b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run33.hql @@ -0,0 +1,76 @@ +CREATE EXTERNAL TABLE `parquet_delta_binary_packed`( + bitwidth0 bigint, + bitwidth1 bigint, + bitwidth2 bigint, + bitwidth3 bigint, + bitwidth4 bigint, + bitwidth5 bigint, + bitwidth6 bigint, + bitwidth7 bigint, + bitwidth8 bigint, + bitwidth9 bigint, + bitwidth10 bigint, + bitwidth11 bigint, + bitwidth12 bigint, + bitwidth13 bigint, + bitwidth14 bigint, + bitwidth15 bigint, + bitwidth16 bigint, + bitwidth17 bigint, + bitwidth18 bigint, + bitwidth19 bigint, + bitwidth20 bigint, + bitwidth21 bigint, + bitwidth22 bigint, + bitwidth23 bigint, + bitwidth24 bigint, + bitwidth25 bigint, + bitwidth26 bigint, + bitwidth27 bigint, + bitwidth28 bigint, + bitwidth29 bigint, + bitwidth30 bigint, + bitwidth31 bigint, + bitwidth32 bigint, + bitwidth33 bigint, + bitwidth34 bigint, + bitwidth35 bigint, + bitwidth36 bigint, + bitwidth37 bigint, + bitwidth38 bigint, + bitwidth39 bigint, + bitwidth40 bigint, + bitwidth41 bigint, + bitwidth42 bigint, + bitwidth43 bigint, + bitwidth44 bigint, + bitwidth45 bigint, + bitwidth46 bigint, + bitwidth47 bigint, + bitwidth48 bigint, + bitwidth49 bigint, + bitwidth50 bigint, + bitwidth51 bigint, + bitwidth52 bigint, + bitwidth53 bigint, + bitwidth54 bigint, + bitwidth55 bigint, + bitwidth56 bigint, + bitwidth57 bigint, + bitwidth58 bigint, + bitwidth59 bigint, + bitwidth60 bigint, + bitwidth61 bigint, + bitwidth62 bigint, + bitwidth63 bigint, + bitwidth64 bigint, + int_value int + ) +STORED AS parquet +LOCATION + '/user/doris/preinstalled_data/parquet_table/parquet_delta_binary_packed' +TBLPROPERTIES ( + 'transient_lastDdlTime'='1661955829'); + +msck repair table parquet_delta_binary_packed; + diff --git a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run34.hql b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run34.hql new file mode 100755 index 00000000000000..40f3f037031eb5 --- /dev/null +++ b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run34.hql @@ -0,0 +1,29 @@ +CREATE TABLE `parquet_alltypes_tiny_pages`( + bool_col boolean, + tinyint_col int, + smallint_col int, + int_col int, + bigint_col bigint, + float_col float, + double_col double, + id int, + date_string_col string, + string_col string, + timestamp_col timestamp, + year int, + month int + ) +ROW FORMAT SERDE + 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' +STORED AS INPUTFORMAT + 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat' +OUTPUTFORMAT + 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat' +LOCATION + '/user/doris/preinstalled_data/parquet_table/parquet_alltypes_tiny_pages' +TBLPROPERTIES ( + 'transient_lastDdlTime'='1661955829'); + +msck repair table parquet_alltypes_tiny_pages; + + diff --git a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run35.hql b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run35.hql new file mode 100755 index 00000000000000..bd646002dfb805 --- /dev/null +++ b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run35.hql @@ -0,0 +1,31 @@ +CREATE EXTERNAL TABLE IF NOT EXISTS `orc_all_types_partition`( + `tinyint_col` tinyint, + `smallint_col` smallint, + `int_col` int, + `bigint_col` bigint, + `boolean_col` boolean, + `float_col` float, + `double_col` double, + `string_col` string, + `binary_col` binary, + `timestamp_col` timestamp, + `decimal_col` decimal(12,4), + `char_col` char(50), + `varchar_col` varchar(50), + `date_col` date, + `list_double_col` array, + `list_string_col` array) +PARTITIONED BY ( + `p1_col` string, + `p2_col` string) +ROW FORMAT SERDE + 'org.apache.hadoop.hive.ql.io.orc.OrcSerde' +STORED AS INPUTFORMAT + 'org.apache.hadoop.hive.ql.io.orc.OrcInputFormat' +OUTPUTFORMAT + 'org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat' +LOCATION + '/user/doris/preinstalled_data/orc_table/orc_all_types_partition'; + +msck repair table orc_all_types_partition; + diff --git a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run36.hql b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run36.hql new file mode 100755 index 00000000000000..7d7511244fae93 --- /dev/null +++ b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run36.hql @@ -0,0 +1,34 @@ +CREATE external TABLE `csv_partition_table`( + `k1` string COMMENT 'k1', + `k2` string COMMENT 'k2', + `k3` string COMMENT 'k3', + `k4` string COMMENT 'k4', + `k5` string COMMENT 'k5', + `k6` string COMMENT 'k6', + `k7` string COMMENT 'k7', + `k8` string COMMENT 'k8', + `k9` string COMMENT 'k9', + `k10` string COMMENT 'k10', + `k11` string COMMENT 'k11', + `k12` string COMMENT 'k12', + `k13` string COMMENT 'k13', + `k14` string COMMENT 'k14', + `k15` string COMMENT 'k15') +PARTITIONED BY ( + `dt` string) +ROW FORMAT SERDE + 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' +WITH SERDEPROPERTIES ( + 'field.delim'='|', + 'serialization.format'='|') +STORED AS INPUTFORMAT + 'org.apache.hadoop.mapred.TextInputFormat' +OUTPUTFORMAT + 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' +LOCATION + '/user/doris/preinstalled_data/csv/csv_partition_table/' +TBLPROPERTIES ( + 'transient_lastDdlTime'='1669304897'); + +msck repair table csv_partition_table; + diff --git a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run37.hql b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run37.hql new file mode 100755 index 00000000000000..44197ec1969b89 --- /dev/null +++ b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run37.hql @@ -0,0 +1,81 @@ +CREATE TABLE `parquet_all_types`( + `t_null_string` string, + `t_null_varchar` varchar(65535), + `t_null_char` char(10), + `t_null_decimal_precision_2` decimal(2,1), + `t_null_decimal_precision_4` decimal(4,2), + `t_null_decimal_precision_8` decimal(8,4), + `t_null_decimal_precision_17` decimal(17,8), + `t_null_decimal_precision_18` decimal(18,8), + `t_null_decimal_precision_38` decimal(38,16), + `t_empty_string` string, + `t_string` string, + `t_empty_varchar` varchar(65535), + `t_varchar` varchar(65535), + `t_varchar_max_length` varchar(65535), + `t_char` char(10), + `t_int` int, + `t_bigint` bigint, + `t_float` float, + `t_double` double, + `t_boolean_true` boolean, + `t_boolean_false` boolean, + `t_decimal_precision_2` decimal(2,1), + `t_decimal_precision_4` decimal(4,2), + `t_decimal_precision_8` decimal(8,4), + `t_decimal_precision_17` decimal(17,8), + `t_decimal_precision_18` decimal(18,8), + `t_decimal_precision_38` decimal(38,16), + `t_binary` binary, + `t_map_string` map, + `t_map_varchar` map, + `t_map_char` map, + `t_map_int` map, + `t_map_bigint` map, + `t_map_float` map, + `t_map_double` map, + `t_map_boolean` map, + `t_map_decimal_precision_2` map, + `t_map_decimal_precision_4` map, + `t_map_decimal_precision_8` map, + `t_map_decimal_precision_17` map, + `t_map_decimal_precision_18` map, + `t_map_decimal_precision_38` map, + `t_array_string` array, + `t_array_int` array, + `t_array_bigint` array, + `t_array_float` array, + `t_array_double` array, + `t_array_boolean` array, + `t_array_varchar` array, + `t_array_char` array, + `t_array_decimal_precision_2` array, + `t_array_decimal_precision_4` array, + `t_array_decimal_precision_8` array, + `t_array_decimal_precision_17` array, + `t_array_decimal_precision_18` array, + `t_array_decimal_precision_38` array, + `t_struct_bigint` struct, + `t_complex` map>>, + `t_struct_nested` struct>, + `t_struct_null` struct, + `t_struct_non_nulls_after_nulls` struct, + `t_nested_struct_non_nulls_after_nulls` struct>, + `t_map_null_value` map, + `t_array_string_starting_with_nulls` array, + `t_array_string_with_nulls_in_between` array, + `t_array_string_ending_with_nulls` array, + `t_array_string_all_nulls` array +) ROW FORMAT SERDE + 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' +STORED AS INPUTFORMAT + 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat' +OUTPUTFORMAT + 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat' +LOCATION + '/user/doris/preinstalled_data/parquet_table/parquet_all_types' +TBLPROPERTIES ( + 'transient_lastDdlTime'='1681213018'); + +msck repair table parquet_all_types; + diff --git a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run38.hql b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run38.hql new file mode 100755 index 00000000000000..965dbc9ac438e1 --- /dev/null +++ b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run38.hql @@ -0,0 +1,72 @@ +CREATE TABLE IF NOT EXISTS `avro_all_types`( +`t_null_string` string, +`t_null_varchar` varchar(65535), +`t_null_char` char(10), +`t_null_array_int` array, +`t_null_decimal_precision_2` decimal(2,1), +`t_null_decimal_precision_4` decimal(4,2), +`t_null_decimal_precision_8` decimal(8,4), +`t_null_decimal_precision_17` decimal(17,8), +`t_null_decimal_precision_18` decimal(18,8), +`t_null_decimal_precision_38` decimal(38,16), +`t_empty_string` string, +`t_string` string, +`t_empty_varchar` varchar(65535), +`t_varchar` varchar(65535), +`t_varchar_max_length` varchar(65535), +`t_char` char(10), +`t_int` int, +`t_bigint` bigint, +`t_float` float, +`t_double` double, +`t_boolean_true` boolean, +`t_boolean_false` boolean, +`t_date` date, +`t_timestamp` timestamp, +`t_decimal_precision_2` decimal(2,1), +`t_decimal_precision_4` decimal(4,2), +`t_decimal_precision_8` decimal(8,4), +`t_decimal_precision_17` decimal(17,8), +`t_decimal_precision_18` decimal(18,8), +`t_decimal_precision_38` decimal(38,16), +`t_binary` binary, +`t_map_string` map, +`t_array_empty` array, +`t_array_string` array, +`t_array_int` array, +`t_array_bigint` array, +`t_array_float` array, +`t_array_double` array, +`t_array_boolean` array, +`t_array_varchar` array, +`t_array_char` array, +`t_array_date` array, +`t_array_timestamp` array, +`t_array_decimal_precision_2` array, +`t_array_decimal_precision_4` array, +`t_array_decimal_precision_8` array, +`t_array_decimal_precision_17` array, +`t_array_decimal_precision_18` array, +`t_array_decimal_precision_38` array, +`t_struct_bigint` struct, +`t_complex` map>>, +`t_struct_nested` struct>, +`t_struct_null` struct, +`t_struct_non_nulls_after_nulls` struct, +`t_array_string_starting_with_nulls` array, +`t_array_string_with_nulls_in_between` array, +`t_array_string_ending_with_nulls` array, +`t_array_string_all_nulls` array +) +ROW FORMAT SERDE + 'org.apache.hadoop.hive.serde2.avro.AvroSerDe' +STORED AS INPUTFORMAT + 'org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat' +OUTPUTFORMAT + 'org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat' +LOCATION + '/user/doris/preinstalled_data/avro/avro_all_types'; + +msck repair table avro_all_types; + + diff --git a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run39.hql b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run39.hql new file mode 100755 index 00000000000000..a9cb1f6d568dc7 --- /dev/null +++ b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run39.hql @@ -0,0 +1,94 @@ +CREATE TABLE IF NOT EXISTS `orc_all_types_t`( +`t_null_string` string, +`t_null_varchar` varchar(65535), +`t_null_char` char(10), +`t_null_array_int` array, +`t_null_decimal_precision_2` decimal(2,1), +`t_null_decimal_precision_4` decimal(4,2), +`t_null_decimal_precision_8` decimal(8,4), +`t_null_decimal_precision_17` decimal(17,8), +`t_null_decimal_precision_18` decimal(18,8), +`t_null_decimal_precision_38` decimal(38,16), +`t_empty_string` string, +`t_string` string, +`t_empty_varchar` varchar(65535), +`t_varchar` varchar(65535), +`t_varchar_max_length` varchar(65535), +`t_char` char(10), +`t_tinyint` tinyint, +`t_smallint` smallint, +`t_int` int, +`t_bigint` bigint, +`t_float` float, +`t_double` double, +`t_boolean_true` boolean, +`t_boolean_false` boolean, +`t_date` date, +`t_timestamp` timestamp, +`t_decimal_precision_2` decimal(2,1), +`t_decimal_precision_4` decimal(4,2), +`t_decimal_precision_8` decimal(8,4), +`t_decimal_precision_17` decimal(17,8), +`t_decimal_precision_18` decimal(18,8), +`t_decimal_precision_38` decimal(38,16), +`t_binary` binary, +`t_map_string` map, +`t_map_tinyint` map, +`t_map_varchar` map, +`t_map_char` map, +`t_map_smallint` map, +`t_map_int` map, +`t_map_bigint` map, +`t_map_float` map, +`t_map_double` map, +`t_map_boolean` map, +`t_map_date` map, +`t_map_timestamp` map, +`t_map_decimal_precision_2` map, +`t_map_decimal_precision_4` map, +`t_map_decimal_precision_8` map, +`t_map_decimal_precision_17` map, +`t_map_decimal_precision_18` map, +`t_map_decimal_precision_38` map, +`t_array_empty` array, +`t_array_string` array, +`t_array_tinyint` array, +`t_array_smallint` array, +`t_array_int` array, +`t_array_bigint` array, +`t_array_float` array, +`t_array_double` array, +`t_array_boolean` array, +`t_array_varchar` array, +`t_array_char` array, +`t_array_date` array, +`t_array_timestamp` array, +`t_array_decimal_precision_2` array, +`t_array_decimal_precision_4` array, +`t_array_decimal_precision_8` array, +`t_array_decimal_precision_17` array, +`t_array_decimal_precision_18` array, +`t_array_decimal_precision_38` array, +`t_struct_bigint` struct, +`t_complex` map>>, +`t_struct_nested` struct>, +`t_struct_null` struct, +`t_struct_non_nulls_after_nulls` struct, +`t_nested_struct_non_nulls_after_nulls` struct>, +`t_map_null_value` map, +`t_array_string_starting_with_nulls` array, +`t_array_string_with_nulls_in_between` array, +`t_array_string_ending_with_nulls` array, +`t_array_string_all_nulls` array +) +ROW FORMAT SERDE + 'org.apache.hadoop.hive.ql.io.orc.OrcSerde' +STORED AS INPUTFORMAT + 'org.apache.hadoop.hive.ql.io.orc.OrcInputFormat' +OUTPUTFORMAT + 'org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat' +LOCATION + '/user/doris/preinstalled_data/orc_table/orc_all_types'; + +msck repair table orc_all_types_t; + diff --git a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run40.hql b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run40.hql new file mode 100755 index 00000000000000..c557bed4b7dc56 --- /dev/null +++ b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run40.hql @@ -0,0 +1,41 @@ +CREATE TABLE IF NOT EXISTS `json_all_types`( +`t_null_string` string, +`t_null_varchar` varchar(65535), +`t_null_char` char(10), +`t_null_decimal_precision_2` decimal(2,1), +`t_null_decimal_precision_4` decimal(4,2), +`t_null_decimal_precision_8` decimal(8,4), +`t_null_decimal_precision_17` decimal(17,8), +`t_null_decimal_precision_18` decimal(18,8), +`t_null_decimal_precision_38` decimal(38,16), +`t_empty_string` string, +`t_string` string, +`t_empty_varchar` varchar(65535), +`t_varchar` varchar(65535), +`t_varchar_max_length` varchar(65535), +`t_char` char(10), +`t_tinyint` tinyint, +`t_smallint` smallint, +`t_int` int, +`t_bigint` bigint, +`t_float` float, +`t_double` double, +`t_boolean_true` boolean, +`t_boolean_false` boolean, +`t_date` date, +`t_timestamp` timestamp, +`t_decimal_precision_2` decimal(2,1), +`t_decimal_precision_4` decimal(4,2), +`t_decimal_precision_8` decimal(8,4), +`t_decimal_precision_17` decimal(17,8), +`t_decimal_precision_18` decimal(18,8) +) +ROW FORMAT SERDE + 'org.apache.hive.hcatalog.data.JsonSerDe' +STORED AS TEXTFILE +LOCATION + '/user/doris/preinstalled_data/json/json_all_types'; + +msck repair table json_all_types; + + diff --git a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run41.hql b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run41.hql new file mode 100755 index 00000000000000..4b4ff93db94cfb --- /dev/null +++ b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run41.hql @@ -0,0 +1,11 @@ +CREATE TABLE IF NOT EXISTS `csv_all_types`( +`t_empty_string` string, +`t_string` string +) +ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' +STORED AS TEXTFILE +LOCATION + '/user/doris/preinstalled_data/csv/csv_all_types'; + +msck repair table csv_all_types; + diff --git a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run42.hql b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run42.hql new file mode 100755 index 00000000000000..dc469fad77b10d --- /dev/null +++ b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run42.hql @@ -0,0 +1,42 @@ +CREATE TABLE IF NOT EXISTS `text_all_types`( +`t_null_string` string, +`t_null_varchar` varchar(65535), +`t_null_char` char(10), +`t_null_decimal_precision_2` decimal(2,1), +`t_null_decimal_precision_4` decimal(4,2), +`t_null_decimal_precision_8` decimal(8,4), +`t_null_decimal_precision_17` decimal(17,8), +`t_null_decimal_precision_18` decimal(18,8), +`t_null_decimal_precision_38` decimal(38,16), +`t_empty_string` string, +`t_string` string, +`t_empty_varchar` varchar(65535), +`t_varchar` varchar(65535), +`t_varchar_max_length` varchar(65535), +`t_char` char(10), +`t_tinyint` tinyint, +`t_smallint` smallint, +`t_int` int, +`t_bigint` bigint, +`t_float` float, +`t_double` double, +`t_boolean_true` boolean, +`t_boolean_false` boolean, +`t_date` date, +`t_timestamp` timestamp, +`t_decimal_precision_2` decimal(2,1), +`t_decimal_precision_4` decimal(4,2), +`t_decimal_precision_8` decimal(8,4), +`t_decimal_precision_17` decimal(17,8), +`t_decimal_precision_18` decimal(18,8), +`t_decimal_precision_38` decimal(38,16), +`t_binary` binary +) +ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t' +STORED AS TEXTFILE +LOCATION + '/user/doris/preinstalled_data/text/text_all_types'; + +msck repair table text_all_types; + + diff --git a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run43.hql b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run43.hql new file mode 100755 index 00000000000000..8e4bb7949b2c89 --- /dev/null +++ b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run43.hql @@ -0,0 +1,89 @@ +CREATE TABLE IF NOT EXISTS `sequence_all_types`( +`t_null_string` string, +`t_null_varchar` varchar(65535), +`t_null_char` char(10), +`t_null_array_int` array, +`t_null_decimal_precision_2` decimal(2,1), +`t_null_decimal_precision_4` decimal(4,2), +`t_null_decimal_precision_8` decimal(8,4), +`t_null_decimal_precision_17` decimal(17,8), +`t_null_decimal_precision_18` decimal(18,8), +`t_null_decimal_precision_38` decimal(38,16), +`t_empty_string` string, +`t_string` string, +`t_empty_varchar` varchar(65535), +`t_varchar` varchar(65535), +`t_varchar_max_length` varchar(65535), +`t_char` char(10), +`t_tinyint` tinyint, +`t_smallint` smallint, +`t_int` int, +`t_bigint` bigint, +`t_float` float, +`t_double` double, +`t_boolean_true` boolean, +`t_boolean_false` boolean, +`t_date` date, +`t_timestamp` timestamp, +`t_decimal_precision_2` decimal(2,1), +`t_decimal_precision_4` decimal(4,2), +`t_decimal_precision_8` decimal(8,4), +`t_decimal_precision_17` decimal(17,8), +`t_decimal_precision_18` decimal(18,8), +`t_decimal_precision_38` decimal(38,16), +`t_binary` binary, +`t_map_string` map, +`t_map_tinyint` map, +`t_map_varchar` map, +`t_map_char` map, +`t_map_smallint` map, +`t_map_int` map, +`t_map_bigint` map, +`t_map_float` map, +`t_map_double` map, +`t_map_boolean` map, +`t_map_date` map, +`t_map_timestamp` map, +`t_map_decimal_precision_2` map, +`t_map_decimal_precision_4` map, +`t_map_decimal_precision_8` map, +`t_map_decimal_precision_17` map, +`t_map_decimal_precision_18` map, +`t_map_decimal_precision_38` map, +`t_array_empty` array, +`t_array_string` array, +`t_array_tinyint` array, +`t_array_smallint` array, +`t_array_int` array, +`t_array_bigint` array, +`t_array_float` array, +`t_array_double` array, +`t_array_boolean` array, +`t_array_varchar` array, +`t_array_char` array, +`t_array_date` array, +`t_array_timestamp` array, +`t_array_decimal_precision_2` array, +`t_array_decimal_precision_4` array, +`t_array_decimal_precision_8` array, +`t_array_decimal_precision_17` array, +`t_array_decimal_precision_18` array, +`t_array_decimal_precision_38` array, +`t_struct_bigint` struct, +`t_complex` map>>, +`t_struct_nested` struct>, +`t_struct_null` struct, +`t_struct_non_nulls_after_nulls` struct, +`t_nested_struct_non_nulls_after_nulls` struct>, +`t_map_null_value` map, +`t_array_string_starting_with_nulls` array, +`t_array_string_with_nulls_in_between` array, +`t_array_string_ending_with_nulls` array, +`t_array_string_all_nulls` array +) +STORED AS SEQUENCEFILE +LOCATION + '/user/doris/preinstalled_data/sequence/sequence_all_types'; + +msck repair table sequence_all_types; + diff --git a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run44.hql b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run44.hql new file mode 100755 index 00000000000000..3b9a352dcd2729 --- /dev/null +++ b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run44.hql @@ -0,0 +1,82 @@ +CREATE TABLE `parquet_gzip_all_types`( +`t_null_string` string, +`t_null_varchar` varchar(65535), +`t_null_char` char(10), +`t_null_decimal_precision_2` decimal(2,1), +`t_null_decimal_precision_4` decimal(4,2), +`t_null_decimal_precision_8` decimal(8,4), +`t_null_decimal_precision_17` decimal(17,8), +`t_null_decimal_precision_18` decimal(18,8), +`t_null_decimal_precision_38` decimal(38,16), +`t_empty_string` string, +`t_string` string, +`t_empty_varchar` varchar(65535), +`t_varchar` varchar(65535), +`t_varchar_max_length` varchar(65535), +`t_char` char(10), +`t_int` int, +`t_bigint` bigint, +`t_float` float, +`t_double` double, +`t_boolean_true` boolean, +`t_boolean_false` boolean, +`t_decimal_precision_2` decimal(2,1), +`t_decimal_precision_4` decimal(4,2), +`t_decimal_precision_8` decimal(8,4), +`t_decimal_precision_17` decimal(17,8), +`t_decimal_precision_18` decimal(18,8), +`t_decimal_precision_38` decimal(38,16), +`t_binary` binary, +`t_map_string` map, +`t_map_varchar` map, +`t_map_char` map, +`t_map_int` map, +`t_map_bigint` map, +`t_map_float` map, +`t_map_double` map, +`t_map_boolean` map, +`t_map_decimal_precision_2` map, +`t_map_decimal_precision_4` map, +`t_map_decimal_precision_8` map, +`t_map_decimal_precision_17` map, +`t_map_decimal_precision_18` map, +`t_map_decimal_precision_38` map, +`t_array_string` array, +`t_array_int` array, +`t_array_bigint` array, +`t_array_float` array, +`t_array_double` array, +`t_array_boolean` array, +`t_array_varchar` array, +`t_array_char` array, +`t_array_decimal_precision_2` array, +`t_array_decimal_precision_4` array, +`t_array_decimal_precision_8` array, +`t_array_decimal_precision_17` array, +`t_array_decimal_precision_18` array, +`t_array_decimal_precision_38` array, +`t_struct_bigint` struct, +`t_complex` map>>, +`t_struct_nested` struct>, +`t_struct_null` struct, +`t_struct_non_nulls_after_nulls` struct, +`t_nested_struct_non_nulls_after_nulls` struct>, +`t_map_null_value` map, +`t_array_string_starting_with_nulls` array, +`t_array_string_with_nulls_in_between` array, +`t_array_string_ending_with_nulls` array, +`t_array_string_all_nulls` array +) ROW FORMAT SERDE + 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' +STORED AS INPUTFORMAT + 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat' +OUTPUTFORMAT + 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat' +LOCATION + '/user/doris/preinstalled_data/parquet_table/parquet_gzip_all_types' +TBLPROPERTIES ( + 'transient_lastDdlTime'='1681213018', + "parquet.compression"="GZIP"); + +msck repair table parquet_gzip_all_types; + diff --git a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run45.hql b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run45.hql new file mode 100755 index 00000000000000..4e7eeb60e6a66d --- /dev/null +++ b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run45.hql @@ -0,0 +1,82 @@ +CREATE TABLE `parquet_zstd_all_types`( +`t_null_string` string, +`t_null_varchar` varchar(65535), +`t_null_char` char(10), +`t_null_decimal_precision_2` decimal(2,1), +`t_null_decimal_precision_4` decimal(4,2), +`t_null_decimal_precision_8` decimal(8,4), +`t_null_decimal_precision_17` decimal(17,8), +`t_null_decimal_precision_18` decimal(18,8), +`t_null_decimal_precision_38` decimal(38,16), +`t_empty_string` string, +`t_string` string, +`t_empty_varchar` varchar(65535), +`t_varchar` varchar(65535), +`t_varchar_max_length` varchar(65535), +`t_char` char(10), +`t_int` int, +`t_bigint` bigint, +`t_float` float, +`t_double` double, +`t_boolean_true` boolean, +`t_boolean_false` boolean, +`t_decimal_precision_2` decimal(2,1), +`t_decimal_precision_4` decimal(4,2), +`t_decimal_precision_8` decimal(8,4), +`t_decimal_precision_17` decimal(17,8), +`t_decimal_precision_18` decimal(18,8), +`t_decimal_precision_38` decimal(38,16), +`t_binary` binary, +`t_map_string` map, +`t_map_varchar` map, +`t_map_char` map, +`t_map_int` map, +`t_map_bigint` map, +`t_map_float` map, +`t_map_double` map, +`t_map_boolean` map, +`t_map_decimal_precision_2` map, +`t_map_decimal_precision_4` map, +`t_map_decimal_precision_8` map, +`t_map_decimal_precision_17` map, +`t_map_decimal_precision_18` map, +`t_map_decimal_precision_38` map, +`t_array_string` array, +`t_array_int` array, +`t_array_bigint` array, +`t_array_float` array, +`t_array_double` array, +`t_array_boolean` array, +`t_array_varchar` array, +`t_array_char` array, +`t_array_decimal_precision_2` array, +`t_array_decimal_precision_4` array, +`t_array_decimal_precision_8` array, +`t_array_decimal_precision_17` array, +`t_array_decimal_precision_18` array, +`t_array_decimal_precision_38` array, +`t_struct_bigint` struct, +`t_complex` map>>, +`t_struct_nested` struct>, +`t_struct_null` struct, +`t_struct_non_nulls_after_nulls` struct, +`t_nested_struct_non_nulls_after_nulls` struct>, +`t_map_null_value` map, +`t_array_string_starting_with_nulls` array, +`t_array_string_with_nulls_in_between` array, +`t_array_string_ending_with_nulls` array, +`t_array_string_all_nulls` array +) ROW FORMAT SERDE + 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' +STORED AS INPUTFORMAT + 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat' +OUTPUTFORMAT + 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat' +LOCATION + '/user/doris/preinstalled_data/parquet_table/parquet_gzip_all_types' +TBLPROPERTIES ( + 'transient_lastDdlTime'='1681213018', + "parquet.compression"="ZSTD"); + +msck repair table parquet_zstd_all_types; + diff --git a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run46.hql b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run46.hql new file mode 100755 index 00000000000000..7b5ac7571c675c --- /dev/null +++ b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run46.hql @@ -0,0 +1,89 @@ +CREATE TABLE `rcbinary_all_types`( +`t_null_string` string, +`t_null_varchar` varchar(65535), +`t_null_char` char(10), +`t_null_array_int` array, +`t_null_decimal_precision_2` decimal(2,1), +`t_null_decimal_precision_4` decimal(4,2), +`t_null_decimal_precision_8` decimal(8,4), +`t_null_decimal_precision_17` decimal(17,8), +`t_null_decimal_precision_18` decimal(18,8), +`t_null_decimal_precision_38` decimal(38,16), +`t_empty_string` string, +`t_string` string, +`t_varchar` varchar(65535), +`t_varchar_max_length` varchar(65535), +`t_char` char(10), +`t_tinyint` tinyint, +`t_smallint` smallint, +`t_int` int, +`t_bigint` bigint, +`t_float` float, +`t_double` double, +`t_boolean_true` boolean, +`t_boolean_false` boolean, +`t_date` date, +`t_timestamp` timestamp, +`t_decimal_precision_2` decimal(2,1), +`t_decimal_precision_4` decimal(4,2), +`t_decimal_precision_8` decimal(8,4), +`t_decimal_precision_17` decimal(17,8), +`t_decimal_precision_18` decimal(18,8), +`t_decimal_precision_38` decimal(38,16), +`t_binary` binary, +`t_map_string` map, +`t_map_tinyint` map, +`t_map_varchar` map, +`t_map_char` map, +`t_map_smallint` map, +`t_map_int` map, +`t_map_bigint` map, +`t_map_float` map, +`t_map_double` map, +`t_map_boolean` map, +`t_map_date` map, +`t_map_timestamp` map, +`t_map_decimal_precision_2` map, +`t_map_decimal_precision_4` map, +`t_map_decimal_precision_8` map, +`t_map_decimal_precision_17` map, +`t_map_decimal_precision_18` map, +`t_map_decimal_precision_38` map, +`t_array_empty` array, +`t_array_string` array, +`t_array_tinyint` array, +`t_array_smallint` array, +`t_array_int` array, +`t_array_bigint` array, +`t_array_float` array, +`t_array_double` array, +`t_array_boolean` array, +`t_array_varchar` array, +`t_array_char` array, +`t_array_date` array, +`t_array_timestamp` array, +`t_array_decimal_precision_2` array, +`t_array_decimal_precision_4` array, +`t_array_decimal_precision_8` array, +`t_array_decimal_precision_17` array, +`t_array_decimal_precision_18` array, +`t_array_decimal_precision_38` array, +`t_struct_bigint` struct, +`t_complex` map>>, +`t_struct_nested` struct>, +`t_struct_null` struct, +`t_struct_non_nulls_after_nulls` struct, +`t_nested_struct_non_nulls_after_nulls` struct>, +`t_map_null_value` map, +`t_array_string_starting_with_nulls` array, +`t_array_string_with_nulls_in_between` array, +`t_array_string_ending_with_nulls` array, +`t_array_string_all_nulls` array +) +ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.columnar.LazyBinaryColumnarSerDe' +STORED AS RCFILE +LOCATION + '/user/doris/preinstalled_data/rcbinary/rcbinary_all_types'; + +msck repair table rcbinary_all_types; + diff --git a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run47.hql b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run47.hql new file mode 100755 index 00000000000000..652638d184a14a --- /dev/null +++ b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run47.hql @@ -0,0 +1,68 @@ +CREATE TABLE `bloom_parquet_table`( +`t_null_string` string, +`t_null_varchar` varchar(65535), +`t_null_char` char(10), +`t_null_decimal_precision_2` decimal(2,1), +`t_null_decimal_precision_4` decimal(4,2), +`t_null_decimal_precision_8` decimal(8,4), +`t_null_decimal_precision_17` decimal(17,8), +`t_null_decimal_precision_18` decimal(18,8), +`t_null_decimal_precision_38` decimal(38,16), +`t_empty_string` string, +`t_string` string, +`t_empty_varchar` varchar(65535), +`t_varchar` varchar(65535), +`t_varchar_max_length` varchar(65535), +`t_char` char(10), +`t_int` int, +`t_bigint` bigint, +`t_float` float, +`t_double` double, +`t_boolean_true` boolean, +`t_boolean_false` boolean, +`t_decimal_precision_2` decimal(2,1), +`t_decimal_precision_4` decimal(4,2), +`t_decimal_precision_8` decimal(8,4), +`t_decimal_precision_17` decimal(17,8), +`t_decimal_precision_18` decimal(18,8), +`t_decimal_precision_38` decimal(38,16), +`t_binary` binary, +`t_array_string` array, +`t_array_int` array, +`t_array_bigint` array, +`t_array_float` array, +`t_array_double` array, +`t_array_boolean` array, +`t_array_varchar` array, +`t_array_char` array, +`t_array_decimal_precision_2` array, +`t_array_decimal_precision_4` array, +`t_array_decimal_precision_8` array, +`t_array_decimal_precision_17` array, +`t_array_decimal_precision_18` array, +`t_array_decimal_precision_38` array, +`t_struct_bigint` struct, +`t_struct_nested` struct>, +`t_struct_null` struct, +`t_struct_non_nulls_after_nulls` struct, +`t_nested_struct_non_nulls_after_nulls` struct>, +`t_array_string_starting_with_nulls` array, +`t_array_string_with_nulls_in_between` array, +`t_array_string_ending_with_nulls` array, +`t_array_string_all_nulls` array +) ROW FORMAT SERDE + 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' +STORED AS INPUTFORMAT + 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat' +OUTPUTFORMAT + 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat' +LOCATION + '/user/doris/preinstalled_data/parquet_table/bloom_parquet_table' +TBLPROPERTIES ( + 'transient_lastDdlTime'='1681213018', + 'parquet.bloom.filter.columns'='t_int', + 'parquet.bloom.filter.fpp'='0.05'); + +msck repair table bloom_parquet_table; + + diff --git a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run48.hql b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run48.hql new file mode 100755 index 00000000000000..0d6126a77764b8 --- /dev/null +++ b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run48.hql @@ -0,0 +1,99 @@ +CREATE TABLE `bloom_orc_table`( +`t_null_string` string, +`t_null_varchar` varchar(65535), +`t_null_char` char(10), +`t_null_array_int` array, +`t_null_decimal_precision_2` decimal(2,1), +`t_null_decimal_precision_4` decimal(4,2), +`t_null_decimal_precision_8` decimal(8,4), +`t_null_decimal_precision_17` decimal(17,8), +`t_null_decimal_precision_18` decimal(18,8), +`t_null_decimal_precision_38` decimal(38,16), +`t_empty_string` string, +`t_string` string, +`t_empty_varchar` varchar(65535), +`t_varchar` varchar(65535), +`t_varchar_max_length` varchar(65535), +`t_char` char(10), +`t_tinyint` tinyint, +`t_smallint` smallint, +`t_int` int, +`t_bigint` bigint, +`t_float` float, +`t_double` double, +`t_boolean_true` boolean, +`t_boolean_false` boolean, +`t_date` date, +`t_timestamp` timestamp, +`t_decimal_precision_2` decimal(2,1), +`t_decimal_precision_4` decimal(4,2), +`t_decimal_precision_8` decimal(8,4), +`t_decimal_precision_17` decimal(17,8), +`t_decimal_precision_18` decimal(18,8), +`t_decimal_precision_38` decimal(38,16), +`t_binary` binary, +`t_map_string` map, +`t_map_tinyint` map, +`t_map_varchar` map, +`t_map_char` map, +`t_map_smallint` map, +`t_map_int` map, +`t_map_bigint` map, +`t_map_float` map, +`t_map_double` map, +`t_map_boolean` map, +`t_map_date` map, +`t_map_timestamp` map, +`t_map_decimal_precision_2` map, +`t_map_decimal_precision_4` map, +`t_map_decimal_precision_8` map, +`t_map_decimal_precision_17` map, +`t_map_decimal_precision_18` map, +`t_map_decimal_precision_38` map, +`t_array_empty` array, +`t_array_string` array, +`t_array_tinyint` array, +`t_array_smallint` array, +`t_array_int` array, +`t_array_bigint` array, +`t_array_float` array, +`t_array_double` array, +`t_array_boolean` array, +`t_array_varchar` array, +`t_array_char` array, +`t_array_date` array, +`t_array_timestamp` array, +`t_array_decimal_precision_2` array, +`t_array_decimal_precision_4` array, +`t_array_decimal_precision_8` array, +`t_array_decimal_precision_17` array, +`t_array_decimal_precision_18` array, +`t_array_decimal_precision_38` array, +`t_struct_bigint` struct, +`t_complex` map>>, +`t_struct_nested` struct>, +`t_struct_null` struct, +`t_struct_non_nulls_after_nulls` struct, +`t_nested_struct_non_nulls_after_nulls` struct>, +`t_map_null_value` map, +`t_array_string_starting_with_nulls` array, +`t_array_string_with_nulls_in_between` array, +`t_array_string_ending_with_nulls` array, +`t_array_string_all_nulls` array +) +ROW FORMAT SERDE + 'org.apache.hadoop.hive.ql.io.orc.OrcSerde' +STORED AS INPUTFORMAT + 'org.apache.hadoop.hive.ql.io.orc.OrcInputFormat' +OUTPUTFORMAT + 'org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat' +LOCATION + '/user/doris/preinstalled_data/orc_table/bloom_orc_table' +TBLPROPERTIES ( + 'transient_lastDdlTime'='1681213018', + 'orc.bloom.filter.columns'='t_int', + 'orc.bloom.filter.fpp'='0.05'); + +msck repair table bloom_orc_table; + + diff --git a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run49.hql b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run49.hql new file mode 100755 index 00000000000000..ce90ebe7de8c43 --- /dev/null +++ b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run49.hql @@ -0,0 +1,17 @@ +CREATE TABLE `orc_predicate_table`( +`column_primitive_integer` int, +`column1_struct` struct, +`column_primitive_bigint` bigint +) +ROW FORMAT SERDE + 'org.apache.hadoop.hive.ql.io.orc.OrcSerde' +STORED AS INPUTFORMAT + 'org.apache.hadoop.hive.ql.io.orc.OrcInputFormat' +OUTPUTFORMAT + 'org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat' +LOCATION + '/user/doris/preinstalled_data/orc_table/orc_predicate_table'; + +msck repair table orc_predicate_table; + + diff --git a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run50.hql b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run50.hql new file mode 100755 index 00000000000000..6e0ed5a2597f67 --- /dev/null +++ b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run50.hql @@ -0,0 +1,15 @@ +CREATE TABLE `parquet_predicate_table`( +`column_primitive_integer` int, +`column1_struct` struct, +`column_primitive_bigint` bigint +) ROW FORMAT SERDE + 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' +STORED AS INPUTFORMAT + 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat' +OUTPUTFORMAT + 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat' +LOCATION + '/user/doris/preinstalled_data/parquet_table/parquet_predicate_table'; + +msck repair table parquet_predicate_table; + diff --git a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run51.hql b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run51.hql new file mode 100755 index 00000000000000..c890f00cd1c6f5 --- /dev/null +++ b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run51.hql @@ -0,0 +1,14 @@ +CREATE TABLE `only_null`( +`x` int +) ROW FORMAT SERDE + 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' +STORED AS INPUTFORMAT + 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat' +OUTPUTFORMAT + 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat' +LOCATION + '/user/doris/preinstalled_data/parquet_table/only_null'; + +msck repair table only_null; + + diff --git a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run52.hql b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run52.hql new file mode 100755 index 00000000000000..0eb1b21fd47665 --- /dev/null +++ b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run52.hql @@ -0,0 +1,14 @@ +CREATE TABLE `parquet_timestamp_millis`( +test timestamp +) ROW FORMAT SERDE + 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' +STORED AS INPUTFORMAT + 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat' +OUTPUTFORMAT + 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat' +LOCATION + '/user/doris/preinstalled_data/parquet_table/parquet_timestamp_millis'; + +msck repair table parquet_timestamp_millis; + + diff --git a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run53.hql b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run53.hql new file mode 100755 index 00000000000000..20068857cfb824 --- /dev/null +++ b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run53.hql @@ -0,0 +1,13 @@ +CREATE TABLE `parquet_timestamp_micros`( +test timestamp +) ROW FORMAT SERDE + 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' +STORED AS INPUTFORMAT + 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat' +OUTPUTFORMAT + 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat' +LOCATION + '/user/doris/preinstalled_data/parquet_table/parquet_timestamp_micros'; + +msck repair table parquet_timestamp_micros; + diff --git a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run54.hql b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run54.hql new file mode 100755 index 00000000000000..6e7fee48e96a58 --- /dev/null +++ b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run54.hql @@ -0,0 +1,13 @@ +CREATE TABLE `parquet_timestamp_nanos`( +test timestamp +) ROW FORMAT SERDE + 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' +STORED AS INPUTFORMAT + 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat' +OUTPUTFORMAT + 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat' +LOCATION + '/user/doris/preinstalled_data/parquet_table/parquet_timestamp_nanos'; + +msck repair table parquet_timestamp_nanos; + diff --git a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run55.hql b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run55.hql new file mode 100755 index 00000000000000..a2533354e23123 --- /dev/null +++ b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run55.hql @@ -0,0 +1,19 @@ +CREATE TABLE `orc_decimal_table`( + id INT, + decimal_col1 DECIMAL(8, 4), + decimal_col2 DECIMAL(18, 6), + decimal_col3 DECIMAL(38, 12), + decimal_col4 DECIMAL(9, 0), + decimal_col5 DECIMAL(27, 9), + decimal_col6 DECIMAL(9, 0)) +ROW FORMAT SERDE + 'org.apache.hadoop.hive.ql.io.orc.OrcSerde' +STORED AS INPUTFORMAT + 'org.apache.hadoop.hive.ql.io.orc.OrcInputFormat' +OUTPUTFORMAT + 'org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat' +LOCATION + '/user/doris/preinstalled_data/orc_table/orc_decimal_table'; + +msck repair table orc_decimal_table; + diff --git a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run56.hql b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run56.hql new file mode 100755 index 00000000000000..8548f10e6e226d --- /dev/null +++ b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run56.hql @@ -0,0 +1,15 @@ +CREATE TABLE `parquet_decimal_bool`( + decimals decimal(20,3), + bool_rle boolean +) +ROW FORMAT SERDE + 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' +STORED AS INPUTFORMAT + 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat' +OUTPUTFORMAT + 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat' +LOCATION + '/user/doris/preinstalled_data/parquet_table/parquet_decimal_bool'; + +msck repair table partition_table; + diff --git a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run57.hql b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run57.hql new file mode 100755 index 00000000000000..ab46f70ece6228 --- /dev/null +++ b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run57.hql @@ -0,0 +1,13 @@ +CREATE TABLE `parquet_decimal90_table`( + `decimal_col` decimal(9,0)) +ROW FORMAT SERDE + 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' +STORED AS INPUTFORMAT + 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat' +OUTPUTFORMAT + 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat' +LOCATION + '/user/doris/preinstalled_data/parquet_table/parquet_decimal90_table'; + +msck repair table parquet_decimal90_table; + diff --git a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run58.hql b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run58.hql new file mode 100755 index 00000000000000..e00f04582295b8 --- /dev/null +++ b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run58.hql @@ -0,0 +1,19 @@ +CREATE TABLE `fixed_length_byte_array_decimal_table`( + `decimal_col1` decimal(7,2), + `decimal_col2` decimal(7,2), + `decimal_col3` decimal(7,2), + `decimal_col4` decimal(7,2), + `decimal_col5` decimal(7,2)) +ROW FORMAT SERDE + 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' +STORED AS INPUTFORMAT + 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat' +OUTPUTFORMAT + 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat' +LOCATION + '/user/doris/preinstalled_data/parquet_table/fixed_length_byte_array_decimal_table' +TBLPROPERTIES ( + 'parquet.compress'='SNAPPY'); + +msck repair table fixed_length_byte_array_decimal_table; + diff --git a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run59.hql b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run59.hql new file mode 100755 index 00000000000000..f5128d7d6df482 --- /dev/null +++ b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run59.hql @@ -0,0 +1,20 @@ +CREATE TABLE `string_col_dict_plain_mixed_orc`( + `col0` int, + `col1` string, + `col2` double, + `col3` boolean, + `col4` string, + `col5` int) +ROW FORMAT SERDE + 'org.apache.hadoop.hive.ql.io.orc.OrcSerde' +STORED AS INPUTFORMAT + 'org.apache.hadoop.hive.ql.io.orc.OrcInputFormat' +OUTPUTFORMAT + 'org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat' +LOCATION + '/user/doris/preinstalled_data/orc_table/string_col_dict_plain_mixed_orc' +TBLPROPERTIES ( + 'orc.compress'='ZLIB'); + +msck repair table string_col_dict_plain_mixed_orc; + diff --git a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run60.hql b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run60.hql new file mode 100755 index 00000000000000..022722a43b43ba --- /dev/null +++ b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run60.hql @@ -0,0 +1,21 @@ +CREATE TABLE `test_string_dict_filter_parquet`( + `o_orderkey` int, + `o_custkey` int, + `o_orderstatus` string, + `o_totalprice` decimal(15,2), + `o_orderdate` date, + `o_orderpriority` string, + `o_clerk` string, + `o_shippriority` int, + `o_comment` string) +ROW FORMAT SERDE + 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' +STORED AS INPUTFORMAT + 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat' +OUTPUTFORMAT + 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat' +LOCATION + '/user/doris/preinstalled_data/parquet_table/test_string_dict_filter_parquet'; + +msck repair table test_string_dict_filter_parquet; + diff --git a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run61.hql b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run61.hql new file mode 100755 index 00000000000000..2a8b51a0468efd --- /dev/null +++ b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run61.hql @@ -0,0 +1,22 @@ +CREATE TABLE `test_string_dict_filter_orc`( + `o_orderkey` int, + `o_custkey` int, + `o_orderstatus` string, + `o_totalprice` decimal(15,2), + `o_orderdate` date, + `o_orderpriority` string, + `o_clerk` string, + `o_shippriority` int, + `o_comment` string) +ROW FORMAT SERDE + 'org.apache.hadoop.hive.ql.io.orc.OrcSerde' +STORED AS INPUTFORMAT + 'org.apache.hadoop.hive.ql.io.orc.OrcInputFormat' +OUTPUTFORMAT + 'org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat' +LOCATION + '/user/doris/preinstalled_data/orc_table/test_string_dict_filter_orc'; + +msck repair table test_string_dict_filter_orc; + + diff --git a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run62.hql b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run62.hql new file mode 100755 index 00000000000000..fbd6a62c2299e2 --- /dev/null +++ b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run62.hql @@ -0,0 +1,17 @@ +create database stats_test; +use stats_test; +create table stats_test1 (id INT, value STRING) STORED AS ORC; +create table stats_test2 (id INT, value STRING) STORED AS PARQUET; +create table stats_test3 (id INT, value STRING) STORED AS PARQUET; + +insert into stats_test1 values (1, 'name1'), (2, 'name2'), (3, 'name3'); +INSERT INTO stats_test2 VALUES (1, ';'), (2, '\*'); + +create table employee_gz(name string,salary string) +row format serde 'org.apache.hadoop.hive.serde2.OpenCSVSerde' +with serdeproperties +('quoteChar'='\"' +,'separatorChar'=','); + +insert into employee_gz values ('a', '1.1'), ('b', '2.2'); + diff --git a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run63.hql b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run63.hql new file mode 100755 index 00000000000000..da441b2553df0d --- /dev/null +++ b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run63.hql @@ -0,0 +1,497 @@ +CREATE DATABASE write_test; +use write_test; + +CREATE TABLE `all_types_parquet_snappy_src`( + `boolean_col` boolean, + `tinyint_col` tinyint, + `smallint_col` smallint, + `int_col` int, + `bigint_col` bigint, + `float_col` float, + `double_col` double, + `decimal_col1` decimal(9,0), + `decimal_col2` decimal(8,4), + `decimal_col3` decimal(18,6), + `decimal_col4` decimal(38,12), + `string_col` string, + `binary_col` binary, + `date_col` date, + `timestamp_col1` timestamp, + `timestamp_col2` timestamp, + `timestamp_col3` timestamp, + `char_col1` char(50), + `char_col2` char(100), + `char_col3` char(255), + `varchar_col1` varchar(50), + `varchar_col2` varchar(100), + `varchar_col3` varchar(255), + `t_map_string` map, + `t_map_varchar` map, + `t_map_char` map, + `t_map_int` map, + `t_map_bigint` map, + `t_map_float` map, + `t_map_double` map, + `t_map_boolean` map, + `t_map_decimal_precision_2` map, + `t_map_decimal_precision_4` map, + `t_map_decimal_precision_8` map, + `t_map_decimal_precision_17` map, + `t_map_decimal_precision_18` map, + `t_map_decimal_precision_38` map, + `t_array_string` array, + `t_array_int` array, + `t_array_bigint` array, + `t_array_float` array, + `t_array_double` array, + `t_array_boolean` array, + `t_array_varchar` array, + `t_array_char` array, + `t_array_decimal_precision_2` array, + `t_array_decimal_precision_4` array, + `t_array_decimal_precision_8` array, + `t_array_decimal_precision_17` array, + `t_array_decimal_precision_18` array, + `t_array_decimal_precision_38` array, + `t_struct_bigint` struct, + `t_complex` map>>, + `t_struct_nested` struct>, + `t_struct_null` struct, + `t_struct_non_nulls_after_nulls` struct, + `t_nested_struct_non_nulls_after_nulls` struct>, + `t_map_null_value` map, + `t_array_string_starting_with_nulls` array, + `t_array_string_with_nulls_in_between` array, + `t_array_string_ending_with_nulls` array, + `t_array_string_all_nulls` array, + `dt` int) +stored as parquet +LOCATION + '/user/doris/preinstalled_data/parquet_table/all_types_parquet_snappy_src' +TBLPROPERTIES('parquet.compression'='SNAPPY'); + +CREATE TABLE `all_types_par_parquet_snappy_src`( + `boolean_col` boolean, + `tinyint_col` tinyint, + `smallint_col` smallint, + `int_col` int, + `bigint_col` bigint, + `float_col` float, + `double_col` double, + `decimal_col1` decimal(9,0), + `decimal_col2` decimal(8,4), + `decimal_col3` decimal(18,6), + `decimal_col4` decimal(38,12), + `string_col` string, + `binary_col` binary, + `date_col` date, + `timestamp_col1` timestamp, + `timestamp_col2` timestamp, + `timestamp_col3` timestamp, + `char_col1` char(50), + `char_col2` char(100), + `char_col3` char(255), + `varchar_col1` varchar(50), + `varchar_col2` varchar(100), + `varchar_col3` varchar(255), + `t_map_string` map, + `t_map_varchar` map, + `t_map_char` map, + `t_map_int` map, + `t_map_bigint` map, + `t_map_float` map, + `t_map_double` map, + `t_map_boolean` map, + `t_map_decimal_precision_2` map, + `t_map_decimal_precision_4` map, + `t_map_decimal_precision_8` map, + `t_map_decimal_precision_17` map, + `t_map_decimal_precision_18` map, + `t_map_decimal_precision_38` map, + `t_array_string` array, + `t_array_int` array, + `t_array_bigint` array, + `t_array_float` array, + `t_array_double` array, + `t_array_boolean` array, + `t_array_varchar` array, + `t_array_char` array, + `t_array_decimal_precision_2` array, + `t_array_decimal_precision_4` array, + `t_array_decimal_precision_8` array, + `t_array_decimal_precision_17` array, + `t_array_decimal_precision_18` array, + `t_array_decimal_precision_38` array, + `t_struct_bigint` struct, + `t_complex` map>>, + `t_struct_nested` struct>, + `t_struct_null` struct, + `t_struct_non_nulls_after_nulls` struct, + `t_nested_struct_non_nulls_after_nulls` struct>, + `t_map_null_value` map, + `t_array_string_starting_with_nulls` array, + `t_array_string_with_nulls_in_between` array, + `t_array_string_ending_with_nulls` array, + `t_array_string_all_nulls` array) +PARTITIONED BY ( + `dt` int) +stored as parquet +LOCATION + '/user/doris/preinstalled_data/parquet_table/all_types_par_parquet_snappy_src' +TBLPROPERTIES('parquet.compression'='SNAPPY'); +msck repair table all_types_par_parquet_snappy_src; + +CREATE TABLE `all_types_parquet_snappy`( + `boolean_col` boolean, + `tinyint_col` tinyint, + `smallint_col` smallint, + `int_col` int, + `bigint_col` bigint, + `float_col` float, + `double_col` double, + `decimal_col1` decimal(9,0), + `decimal_col2` decimal(8,4), + `decimal_col3` decimal(18,6), + `decimal_col4` decimal(38,12), + `string_col` string, + `binary_col` binary, + `date_col` date, + `timestamp_col1` timestamp, + `timestamp_col2` timestamp, + `timestamp_col3` timestamp, + `char_col1` char(50), + `char_col2` char(100), + `char_col3` char(255), + `varchar_col1` varchar(50), + `varchar_col2` varchar(100), + `varchar_col3` varchar(255), + `t_map_string` map, + `t_map_varchar` map, + `t_map_char` map, + `t_map_int` map, + `t_map_bigint` map, + `t_map_float` map, + `t_map_double` map, + `t_map_boolean` map, + `t_map_decimal_precision_2` map, + `t_map_decimal_precision_4` map, + `t_map_decimal_precision_8` map, + `t_map_decimal_precision_17` map, + `t_map_decimal_precision_18` map, + `t_map_decimal_precision_38` map, + `t_array_string` array, + `t_array_int` array, + `t_array_bigint` array, + `t_array_float` array, + `t_array_double` array, + `t_array_boolean` array, + `t_array_varchar` array, + `t_array_char` array, + `t_array_decimal_precision_2` array, + `t_array_decimal_precision_4` array, + `t_array_decimal_precision_8` array, + `t_array_decimal_precision_17` array, + `t_array_decimal_precision_18` array, + `t_array_decimal_precision_38` array, + `t_struct_bigint` struct, + `t_complex` map>>, + `t_struct_nested` struct>, + `t_struct_null` struct, + `t_struct_non_nulls_after_nulls` struct, + `t_nested_struct_non_nulls_after_nulls` struct>, + `t_map_null_value` map, + `t_array_string_starting_with_nulls` array, + `t_array_string_with_nulls_in_between` array, + `t_array_string_ending_with_nulls` array, + `t_array_string_all_nulls` array, + `dt` int) +stored as parquet +TBLPROPERTIES('parquet.compression'='SNAPPY'); + +CREATE TABLE `all_types_par_parquet_snappy`( + `boolean_col` boolean, + `tinyint_col` tinyint, + `smallint_col` smallint, + `int_col` int, + `bigint_col` bigint, + `float_col` float, + `double_col` double, + `decimal_col1` decimal(9,0), + `decimal_col2` decimal(8,4), + `decimal_col3` decimal(18,6), + `decimal_col4` decimal(38,12), + `string_col` string, + `binary_col` binary, + `date_col` date, + `timestamp_col1` timestamp, + `timestamp_col2` timestamp, + `timestamp_col3` timestamp, + `char_col1` char(50), + `char_col2` char(100), + `char_col3` char(255), + `varchar_col1` varchar(50), + `varchar_col2` varchar(100), + `varchar_col3` varchar(255), + `t_map_string` map, + `t_map_varchar` map, + `t_map_char` map, + `t_map_int` map, + `t_map_bigint` map, + `t_map_float` map, + `t_map_double` map, + `t_map_boolean` map, + `t_map_decimal_precision_2` map, + `t_map_decimal_precision_4` map, + `t_map_decimal_precision_8` map, + `t_map_decimal_precision_17` map, + `t_map_decimal_precision_18` map, + `t_map_decimal_precision_38` map, + `t_array_string` array, + `t_array_int` array, + `t_array_bigint` array, + `t_array_float` array, + `t_array_double` array, + `t_array_boolean` array, + `t_array_varchar` array, + `t_array_char` array, + `t_array_decimal_precision_2` array, + `t_array_decimal_precision_4` array, + `t_array_decimal_precision_8` array, + `t_array_decimal_precision_17` array, + `t_array_decimal_precision_18` array, + `t_array_decimal_precision_38` array, + `t_struct_bigint` struct, + `t_complex` map>>, + `t_struct_nested` struct>, + `t_struct_null` struct, + `t_struct_non_nulls_after_nulls` struct, + `t_nested_struct_non_nulls_after_nulls` struct>, + `t_map_null_value` map, + `t_array_string_starting_with_nulls` array, + `t_array_string_with_nulls_in_between` array, + `t_array_string_ending_with_nulls` array, + `t_array_string_all_nulls` array) +PARTITIONED BY ( + `dt` int) +stored as parquet +TBLPROPERTIES('parquet.compression'='SNAPPY'); + +CREATE TABLE `all_types_orc_zlib`( + `boolean_col` boolean, + `tinyint_col` tinyint, + `smallint_col` smallint, + `int_col` int, + `bigint_col` bigint, + `float_col` float, + `double_col` double, + `decimal_col1` decimal(9,0), + `decimal_col2` decimal(8,4), + `decimal_col3` decimal(18,6), + `decimal_col4` decimal(38,12), + `string_col` string, + `binary_col` binary, + `date_col` date, + `timestamp_col1` timestamp, + `timestamp_col2` timestamp, + `timestamp_col3` timestamp, + `char_col1` char(50), + `char_col2` char(100), + `char_col3` char(255), + `varchar_col1` varchar(50), + `varchar_col2` varchar(100), + `varchar_col3` varchar(255), + `t_map_string` map, + `t_map_varchar` map, + `t_map_char` map, + `t_map_int` map, + `t_map_bigint` map, + `t_map_float` map, + `t_map_double` map, + `t_map_boolean` map, + `t_map_decimal_precision_2` map, + `t_map_decimal_precision_4` map, + `t_map_decimal_precision_8` map, + `t_map_decimal_precision_17` map, + `t_map_decimal_precision_18` map, + `t_map_decimal_precision_38` map, + `t_array_string` array, + `t_array_int` array, + `t_array_bigint` array, + `t_array_float` array, + `t_array_double` array, + `t_array_boolean` array, + `t_array_varchar` array, + `t_array_char` array, + `t_array_decimal_precision_2` array, + `t_array_decimal_precision_4` array, + `t_array_decimal_precision_8` array, + `t_array_decimal_precision_17` array, + `t_array_decimal_precision_18` array, + `t_array_decimal_precision_38` array, + `t_struct_bigint` struct, + `t_complex` map>>, + `t_struct_nested` struct>, + `t_struct_null` struct, + `t_struct_non_nulls_after_nulls` struct, + `t_nested_struct_non_nulls_after_nulls` struct>, + `t_map_null_value` map, + `t_array_string_starting_with_nulls` array, + `t_array_string_with_nulls_in_between` array, + `t_array_string_ending_with_nulls` array, + `t_array_string_all_nulls` array, + `dt` int) +stored as orc +TBLPROPERTIES("orc.compress"="ZLIB"); + +CREATE TABLE `all_types_par_orc_zlib`( + `boolean_col` boolean, + `tinyint_col` tinyint, + `smallint_col` smallint, + `int_col` int, + `bigint_col` bigint, + `float_col` float, + `double_col` double, + `decimal_col1` decimal(9,0), + `decimal_col2` decimal(8,4), + `decimal_col3` decimal(18,6), + `decimal_col4` decimal(38,12), + `string_col` string, + `binary_col` binary, + `date_col` date, + `timestamp_col1` timestamp, + `timestamp_col2` timestamp, + `timestamp_col3` timestamp, + `char_col1` char(50), + `char_col2` char(100), + `char_col3` char(255), + `varchar_col1` varchar(50), + `varchar_col2` varchar(100), + `varchar_col3` varchar(255), + `t_map_string` map, + `t_map_varchar` map, + `t_map_char` map, + `t_map_int` map, + `t_map_bigint` map, + `t_map_float` map, + `t_map_double` map, + `t_map_boolean` map, + `t_map_decimal_precision_2` map, + `t_map_decimal_precision_4` map, + `t_map_decimal_precision_8` map, + `t_map_decimal_precision_17` map, + `t_map_decimal_precision_18` map, + `t_map_decimal_precision_38` map, + `t_array_string` array, + `t_array_int` array, + `t_array_bigint` array, + `t_array_float` array, + `t_array_double` array, + `t_array_boolean` array, + `t_array_varchar` array, + `t_array_char` array, + `t_array_decimal_precision_2` array, + `t_array_decimal_precision_4` array, + `t_array_decimal_precision_8` array, + `t_array_decimal_precision_17` array, + `t_array_decimal_precision_18` array, + `t_array_decimal_precision_38` array, + `t_struct_bigint` struct, + `t_complex` map>>, + `t_struct_nested` struct>, + `t_struct_null` struct, + `t_struct_non_nulls_after_nulls` struct, + `t_nested_struct_non_nulls_after_nulls` struct>, + `t_map_null_value` map, + `t_array_string_starting_with_nulls` array, + `t_array_string_with_nulls_in_between` array, + `t_array_string_ending_with_nulls` array, + `t_array_string_all_nulls` array) +PARTITIONED BY ( + `dt` int) +stored as orc +TBLPROPERTIES("orc.compress"="ZLIB"); + +CREATE TABLE `all_partition_types1_parquet_snappy_src`( + `id` int +) +PARTITIONED BY ( + `boolean_col` boolean, + `tinyint_col` tinyint, + `smallint_col` smallint, + `int_col` int, + `bigint_col` bigint, + `float_col` float, + `double_col` double) +stored as parquet +LOCATION + '/user/doris/preinstalled_data/parquet_table/all_partition_types1_parquet_snappy_src' +TBLPROPERTIES('parquet.compression'='SNAPPY'); +msck repair table all_partition_types1_parquet_snappy_src; + +CREATE TABLE `all_partition_types1_parquet_snappy`( + `id` int +) +PARTITIONED BY ( + `boolean_col` boolean, + `tinyint_col` tinyint, + `smallint_col` smallint, + `int_col` int, + `bigint_col` bigint, + `float_col` float, + `double_col` double) +stored as parquet +TBLPROPERTIES('parquet.compression'='SNAPPY'); + +CREATE TABLE `all_partition_types1_orc_zlib`( + `id` int +) +PARTITIONED BY ( + `boolean_col` boolean, + `tinyint_col` tinyint, + `smallint_col` smallint, + `int_col` int, + `bigint_col` bigint, + `float_col` float, + `double_col` double) +stored as orc +TBLPROPERTIES("orc.compress"="ZLIB"); + +CREATE TABLE `all_partition_types2_parquet_snappy_src`( + `id` int +) +PARTITIONED BY ( + `decimal_col` decimal(18,6), + `string_col` string, + `date_col` date, + `char_col` char(11), + `varchar_col` varchar(50)) +stored as parquet +LOCATION + '/user/doris/preinstalled_data/parquet_table/all_partition_types2_parquet_snappy_src' +TBLPROPERTIES('parquet.compression'='SNAPPY'); +msck repair table all_partition_types2_parquet_snappy_src; + +CREATE TABLE `all_partition_types2_parquet_snappy`( + `id` int +) +PARTITIONED BY ( + `decimal_col` decimal(18,6), + `string_col` string, + `date_col` date, + `char_col` char(11), + `varchar_col` varchar(50)) +stored as parquet +TBLPROPERTIES('parquet.compression'='SNAPPY'); + +CREATE TABLE `all_partition_types2_orc_zlib`( + `id` int +) +PARTITIONED BY ( + `decimal_col` decimal(18,6), + `string_col` string, + `date_col` date, + `char_col` char(11), + `varchar_col` varchar(50)) +stored as orc +TBLPROPERTIES("orc.compress"="ZLIB"); + diff --git a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_table.hql b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_table.hql deleted file mode 100644 index 38e9ebb6b07859..00000000000000 --- a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_table.hql +++ /dev/null @@ -1,2414 +0,0 @@ -use default; - -CREATE TABLE `partition_table`( - `l_orderkey` int, - `l_partkey` int, - `l_suppkey` int, - `l_linenumber` int, - `l_quantity` decimal(12,2), - `l_extendedprice` decimal(12,2), - `l_discount` decimal(12,2), - `l_tax` decimal(12,2), - `l_returnflag` string, - `l_linestatus` string, - `l_shipdate` date, - `l_commitdate` date, - `l_receiptdate` date, - `l_shipinstruct` string, - `l_shipmode` string, - `l_comment` string) -partitioned by (nation string, city string) -ROW FORMAT SERDE - 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' -STORED AS INPUTFORMAT - 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat' -OUTPUTFORMAT - 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat' -LOCATION - '/user/doris/preinstalled_data/parquet/partition_table' -TBLPROPERTIES ( - 'transient_lastDdlTime'='1661955829'); - -msck repair table partition_table; - - -CREATE TABLE `delta_byte_array`( - `c_salutation` string, - `c_first_name` string, - `c_last_name` string, - `c_preferred_cust_flag` string, - `c_birth_country` string, - `c_login` string, - `c_email_address` string, - `c_last_review_date` string, - `c_customer_id` string - ) -ROW FORMAT SERDE - 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' -STORED AS INPUTFORMAT - 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat' -OUTPUTFORMAT - 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat' -LOCATION - '/user/doris/preinstalled_data/different_types_parquet/delta_byte_array' -TBLPROPERTIES ( - 'transient_lastDdlTime'='1661955829'); - - -CREATE TABLE `delta_length_byte_array`( - `FRUIT` string - ) -ROW FORMAT SERDE - 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' -STORED AS INPUTFORMAT - 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat' -OUTPUTFORMAT - 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat' -LOCATION - '/user/doris/preinstalled_data/different_types_parquet/delta_length_byte_array' -TBLPROPERTIES ( - 'transient_lastDdlTime'='1661955829'); - -msck repair table delta_length_byte_array; - -CREATE EXTERNAL TABLE `delta_binary_packed`( - bitwidth0 bigint, - bitwidth1 bigint, - bitwidth2 bigint, - bitwidth3 bigint, - bitwidth4 bigint, - bitwidth5 bigint, - bitwidth6 bigint, - bitwidth7 bigint, - bitwidth8 bigint, - bitwidth9 bigint, - bitwidth10 bigint, - bitwidth11 bigint, - bitwidth12 bigint, - bitwidth13 bigint, - bitwidth14 bigint, - bitwidth15 bigint, - bitwidth16 bigint, - bitwidth17 bigint, - bitwidth18 bigint, - bitwidth19 bigint, - bitwidth20 bigint, - bitwidth21 bigint, - bitwidth22 bigint, - bitwidth23 bigint, - bitwidth24 bigint, - bitwidth25 bigint, - bitwidth26 bigint, - bitwidth27 bigint, - bitwidth28 bigint, - bitwidth29 bigint, - bitwidth30 bigint, - bitwidth31 bigint, - bitwidth32 bigint, - bitwidth33 bigint, - bitwidth34 bigint, - bitwidth35 bigint, - bitwidth36 bigint, - bitwidth37 bigint, - bitwidth38 bigint, - bitwidth39 bigint, - bitwidth40 bigint, - bitwidth41 bigint, - bitwidth42 bigint, - bitwidth43 bigint, - bitwidth44 bigint, - bitwidth45 bigint, - bitwidth46 bigint, - bitwidth47 bigint, - bitwidth48 bigint, - bitwidth49 bigint, - bitwidth50 bigint, - bitwidth51 bigint, - bitwidth52 bigint, - bitwidth53 bigint, - bitwidth54 bigint, - bitwidth55 bigint, - bitwidth56 bigint, - bitwidth57 bigint, - bitwidth58 bigint, - bitwidth59 bigint, - bitwidth60 bigint, - bitwidth61 bigint, - bitwidth62 bigint, - bitwidth63 bigint, - bitwidth64 bigint, - int_value int - ) -STORED AS parquet -LOCATION - '/user/doris/preinstalled_data/different_types_parquet/delta_binary_packed' -TBLPROPERTIES ( - 'transient_lastDdlTime'='1661955829'); - -msck repair table delta_binary_packed; - - -CREATE TABLE `delta_encoding_required_column`( - c_customer_sk int, - c_current_cdemo_sk int, - c_current_hdemo_sk int, - c_current_addr_sk int, - c_first_shipto_date_sk int, - c_first_sales_date_sk int, - c_birth_day int, - c_birth_month int, - c_birth_year int, - c_customer_id string, - c_salutation string, - c_first_name string, - c_last_name string, - c_preferred_cust_flag string, - c_birth_country string, - c_email_address string, - c_last_review_date string - ) -ROW FORMAT SERDE - 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' -STORED AS INPUTFORMAT - 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat' -OUTPUTFORMAT - 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat' -LOCATION - '/user/doris/preinstalled_data/different_types_parquet/delta_encoding_required_column/' -TBLPROPERTIES ( - 'transient_lastDdlTime'='1661955829'); - -msck repair table delta_encoding_required_column; - - -CREATE EXTERNAL TABLE `delta_encoding_optional_column`( - c_customer_sk int, - c_current_cdemo_sk int, - c_current_hdemo_sk int, - c_current_addr_sk int, - c_first_shipto_date_sk int, - c_first_sales_date_sk int, - c_birth_year int, - c_customer_id string, - c_salutation string, - c_first_name string, - c_last_name string, - c_preferred_cust_flag string, - c_birth_country string - ) -ROW FORMAT SERDE - 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' -STORED AS INPUTFORMAT - 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat' -OUTPUTFORMAT - 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat' -LOCATION - '/user/doris/preinstalled_data/different_types_parquet/delta_encoding_optional_column' -TBLPROPERTIES ( - 'transient_lastDdlTime'='1661955829'); - -msck repair table delta_encoding_optional_column; - - -CREATE TABLE `datapage_v1_snappy_compressed_checksum`( - `a` int, - `b` int - ) -ROW FORMAT SERDE - 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' -STORED AS INPUTFORMAT - 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat' -OUTPUTFORMAT - 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat' -LOCATION - '/user/doris/preinstalled_data/different_types_parquet/datapage_v1-snappy-compressed-checksum' -TBLPROPERTIES ( - 'transient_lastDdlTime'='1661955829'); - -msck repair table datapage_v1_snappy_compressed_checksum; - - -CREATE TABLE `overflow_i16_page_cnt`( - `inc` boolean - ) -ROW FORMAT SERDE - 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' -STORED AS INPUTFORMAT - 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat' -OUTPUTFORMAT - 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat' -LOCATION - '/user/doris/preinstalled_data/different_types_parquet/overflow_i16_page_cnt' -TBLPROPERTIES ( - 'transient_lastDdlTime'='1661955829'); - -msck repair table overflow_i16_page_cnt; - - -CREATE TABLE `alltypes_tiny_pages`( - bool_col boolean, - tinyint_col int, - smallint_col int, - int_col int, - bigint_col bigint, - float_col float, - double_col double, - id int, - date_string_col string, - string_col string, - timestamp_col timestamp, - year int, - month int - ) -ROW FORMAT SERDE - 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' -STORED AS INPUTFORMAT - 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat' -OUTPUTFORMAT - 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat' -LOCATION - '/user/doris/preinstalled_data/different_types_parquet/alltypes_tiny_pages' -TBLPROPERTIES ( - 'transient_lastDdlTime'='1661955829'); - -msck repair table alltypes_tiny_pages; - - -CREATE TABLE `alltypes_tiny_pages_plain`( - bool_col boolean, - tinyint_col int, - smallint_col int, - int_col int, - bigint_col bigint, - float_col float, - double_col double, - id int, - date_string_col string, - string_col string, - timestamp_col timestamp, - year int, - month int - ) -ROW FORMAT SERDE - 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' -STORED AS INPUTFORMAT - 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat' -OUTPUTFORMAT - 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat' -LOCATION - '/user/doris/preinstalled_data/different_types_parquet/alltypes_tiny_pages_plain' -TBLPROPERTIES ( - 'transient_lastDdlTime'='1661955829'); - -msck repair table alltypes_tiny_pages_plain; - -CREATE TABLE `example_string`( - `strings` string - ) -ROW FORMAT SERDE - 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' -WITH SERDEPROPERTIES ( - 'field.delim'='\t', - 'serialization.format'='\t') -STORED AS INPUTFORMAT - 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat' -OUTPUTFORMAT - 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat' -LOCATION - '/user/doris/preinstalled_data/example_string.parquet' -TBLPROPERTIES ( - 'transient_lastDdlTime'='1661955829'); - -msck repair table example_string; - - -CREATE EXTERNAL TABLE IF NOT EXISTS `orc_all_types`( - `tinyint_col` tinyint, - `smallint_col` smallint, - `int_col` int, - `bigint_col` bigint, - `boolean_col` boolean, - `float_col` float, - `double_col` double, - `string_col` string, - `binary_col` binary, - `timestamp_col` timestamp, - `decimal_col` decimal(12,4), - `char_col` char(50), - `varchar_col` varchar(50), - `date_col` date, - `list_double_col` array, - `list_string_col` array) -PARTITIONED BY ( - `p1_col` string, - `p2_col` string) -ROW FORMAT SERDE - 'org.apache.hadoop.hive.ql.io.orc.OrcSerde' -STORED AS INPUTFORMAT - 'org.apache.hadoop.hive.ql.io.orc.OrcInputFormat' -OUTPUTFORMAT - 'org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat' -LOCATION - '/user/doris/preinstalled_data/orc/orc_all_types'; - -msck repair table orc_all_types; - -CREATE TABLE `lineorder` ( - `lo_orderkey` int, - `lo_linenumber` int, - `lo_custkey` int, - `lo_partkey` int, - `lo_suppkey` int, - `lo_orderdate` int, - `lo_orderpriority` varchar(16), - `lo_shippriority` int, - `lo_quantity` int, - `lo_extendedprice` int, - `lo_ordtotalprice` int, - `lo_discount` int, - `lo_revenue` int, - `lo_supplycost` int, - `lo_tax` int, - `lo_commitdate` int, - `lo_shipmode` varchar(11) -) -ROW FORMAT DELIMITED FIELDS TERMINATED by ',' -LOCATION '/user/doris/preinstalled_data/data_case/lineorder' -TBLPROPERTIES ('transient_lastDdlTime'='1658816839'); - -create table t_hive ( - `k1` int, - `k2` char(10), - `k3` date, - `k5` varchar(20), - `k6` double -) -ROW FORMAT DELIMITED FIELDS TERMINATED by ',' -STORED AS INPUTFORMAT - 'org.apache.hadoop.mapred.TextInputFormat' -OUTPUTFORMAT - 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat' -LOCATION - '/user/doris/preinstalled_data/data_case/t_hive' -TBLPROPERTIES ('transient_lastDdlTime'='1658816839'); - -CREATE external TABLE `table_with_vertical_line`( - `k1` string COMMENT 'k1', - `k2` string COMMENT 'k2', - `k3` string COMMENT 'k3', - `k4` string COMMENT 'k4', - `k5` string COMMENT 'k5', - `k6` string COMMENT 'k6', - `k7` string COMMENT 'k7', - `k8` string COMMENT 'k8', - `k9` string COMMENT 'k9', - `k10` string COMMENT 'k10', - `k11` string COMMENT 'k11', - `k12` string COMMENT 'k12', - `k13` string COMMENT 'k13', - `k14` string COMMENT 'k14', - `k15` string COMMENT 'k15') -PARTITIONED BY ( - `dt` string) -ROW FORMAT SERDE - 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' -WITH SERDEPROPERTIES ( - 'field.delim'='|', - 'serialization.format'='|') -STORED AS INPUTFORMAT - 'org.apache.hadoop.mapred.TextInputFormat' -OUTPUTFORMAT - 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' -LOCATION - '/user/doris/preinstalled_data/csv_partition_table/table_with_vertical_line/' -TBLPROPERTIES ( - 'transient_lastDdlTime'='1669304897'); - -msck repair table table_with_vertical_line; - -CREATE external TABLE `table_with_pars`( - `id` int COMMENT 'id', - `data` string COMMENT 'data') -PARTITIONED BY ( - `dt_par` date, - `time_par` timestamp, - `decimal_par1` decimal(8, 4), - `decimal_par2` decimal(18, 6), - `decimal_par3` decimal(38, 12)) -ROW FORMAT SERDE - 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' -WITH SERDEPROPERTIES ( - 'field.delim'='|', - 'serialization.format'='|') -STORED AS INPUTFORMAT - 'org.apache.hadoop.mapred.TextInputFormat' -OUTPUTFORMAT - 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' -LOCATION - '/user/doris/preinstalled_data/csv_partition_table/table_with_pars/'; - -set hive.msck.path.validation=ignore; -msck repair table table_with_pars; - -CREATE TABLE `table_with_x01`( - `k1` string COMMENT 'k1', - `k2` string COMMENT 'k2', - `k3` string COMMENT 'k3', - `k4` string COMMENT 'k4', - `k5` string COMMENT 'k5', - `k6` string COMMENT 'k6', - `k7` string COMMENT 'k7', - `k8` string COMMENT 'k8', - `k9` string COMMENT 'k9', - `k10` string COMMENT 'k10', - `k11` string COMMENT 'k11', - `k12` string COMMENT 'k12', - `k13` string COMMENT 'k13', - `k14` string COMMENT 'k14', - `k15` string COMMENT 'k15') -PARTITIONED BY ( - `dt` string) -ROW FORMAT SERDE - 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' -STORED AS INPUTFORMAT - 'org.apache.hadoop.mapred.TextInputFormat' -OUTPUTFORMAT - 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' -LOCATION - '/user/doris/preinstalled_data/csv_partition_table/table_with_x01/' -TBLPROPERTIES ( - 'transient_lastDdlTime'='1669360080'); - -msck repair table table_with_x01; - -CREATE TABLE `unsupported_type_table`( - k1 int, - k2 string, - k3 double, - k4 map, - k5 STRUCT< - houseno: STRING - ,streetname: STRING - >, - k6 int -); - -set hive.stats.column.autogather=false; - -CREATE TABLE `test_hive_orc_add_column`( - id int, - col1 int -) -stored as orc; -insert into `test_hive_orc_add_column` values(1,2); -insert into `test_hive_orc_add_column` values(3,4),(4,6); -alter table `test_hive_orc_add_column` ADD COLUMNS (col2 int); -insert into `test_hive_orc_add_column` values(7,8,9); -insert into `test_hive_orc_add_column` values(10,11,null); -insert into `test_hive_orc_add_column` values(12,13,null); -insert into `test_hive_orc_add_column` values(14,15,16); -alter table `test_hive_orc_add_column` ADD COLUMNS (col3 int,col4 string); -insert into `test_hive_orc_add_column` values(17,18,19,20,"hello world"); -insert into `test_hive_orc_add_column` values(21,22,23,24,"cywcywcyw"); -insert into `test_hive_orc_add_column` values(25,26,null,null,null); -insert into `test_hive_orc_add_column` values(27,28,29,null,null); -insert into `test_hive_orc_add_column` values(30,31,32,33,null); - - -CREATE TABLE `schema_evo_test_text`( - id int, - name string -) -ROW FORMAT DELIMITED FIELDS TERMINATED by ','; -insert into `schema_evo_test_text` select 1, "kaka"; -alter table `schema_evo_test_text` ADD COLUMNS (`ts` timestamp); -insert into `schema_evo_test_text` select 2, "messi", from_unixtime(to_unix_timestamp('20230101 13:01:03','yyyyMMdd HH:mm:ss')); - -CREATE TABLE `schema_evo_test_parquet`( - id int, - name string -) -stored as parquet; -insert into `schema_evo_test_parquet` select 1, "kaka"; -alter table `schema_evo_test_parquet` ADD COLUMNS (`ts` timestamp); -insert into `schema_evo_test_parquet` select 2, "messi", from_unixtime(to_unix_timestamp('20230101 13:01:03','yyyyMMdd HH:mm:ss')); - -CREATE TABLE `schema_evo_test_orc`( - id int, - name string -) -stored as orc; -insert into `schema_evo_test_orc` select 1, "kaka"; -alter table `schema_evo_test_orc` ADD COLUMNS (`ts` timestamp); -insert into `schema_evo_test_orc` select 2, "messi", from_unixtime(to_unix_timestamp('20230101 13:01:03','yyyyMMdd HH:mm:ss')); - -set hive.stats.column.autogather=true; - --- Currently docker is hive 2.x version. Hive 2.x versioned full-acid tables need to run major compaction. -SET hive.support.concurrency=true; -SET hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; - -create table orc_full_acid_empty (id INT, value STRING) -CLUSTERED BY (id) INTO 3 BUCKETS -STORED AS ORC -TBLPROPERTIES ('transactional' = 'true'); - -create table orc_full_acid_par_empty (id INT, value STRING) -PARTITIONED BY (part_col INT) -CLUSTERED BY (id) INTO 3 BUCKETS -STORED AS ORC -TBLPROPERTIES ('transactional' = 'true'); - -create table orc_full_acid (id INT, value STRING) -CLUSTERED BY (id) INTO 3 BUCKETS -STORED AS ORC -TBLPROPERTIES ('transactional' = 'true'); - -insert into orc_full_acid values -(1, 'A'), -(2, 'B'), -(3, 'C'); - -update orc_full_acid set value = 'CC' where id = 3; - -alter table orc_full_acid compact 'major'; - -create table orc_full_acid_par (id INT, value STRING) -PARTITIONED BY (part_col INT) -CLUSTERED BY (id) INTO 3 BUCKETS -STORED AS ORC -TBLPROPERTIES ('transactional' = 'true'); - -insert into orc_full_acid_par PARTITION(part_col=20230101) values -(1, 'A'), -(2, 'B'), -(3, 'C'); - -insert into orc_full_acid_par PARTITION(part_col=20230102) values -(4, 'D'), -(5, 'E'), -(6, 'F'); - -update orc_full_acid_par set value = 'BB' where id = 2; - -alter table orc_full_acid_par PARTITION(part_col=20230101) compact 'major'; -alter table orc_full_acid_par PARTITION(part_col=20230102) compact 'major'; - -create table mtmv_base1 (id INT, value STRING) - PARTITIONED BY (part_col INT) - CLUSTERED BY (id) INTO 3 BUCKETS - STORED AS ORC; - -insert into mtmv_base1 PARTITION(part_col=20230101) values -(1, 'A'), -(2, 'B'), -(3, 'C'); - -insert into mtmv_base1 PARTITION(part_col=20230102) values -(4, 'D'), -(5, 'E'), -(6, 'F'); - - -CREATE TABLE `test_different_column_orders_orc`( - `name` string, - `id` int, - `city` string, - `age` int, - `sex` string) -STORED AS ORC -LOCATION - '/user/doris/preinstalled_data/test_different_column_orders/orc'; - -CREATE TABLE `test_different_column_orders_parquet`( - `name` string, - `id` int, - `city` string, - `age` int, - `sex` string) -STORED AS PARQUET -LOCATION - '/user/doris/preinstalled_data/test_different_column_orders/parquet'; - -CREATE TABLE `parquet_partition_table`( - `l_orderkey` int, - `l_partkey` int, - `l_suppkey` int, - `l_linenumber` int, - `l_quantity` decimal(12,2), - `l_extendedprice` decimal(12,2), - `l_discount` decimal(12,2), - `l_tax` decimal(12,2), - `l_returnflag` string, - `l_linestatus` string, - `l_shipdate` date, - `l_commitdate` date, - `l_receiptdate` date, - `l_shipinstruct` string, - `l_shipmode` string, - `l_comment` string) -partitioned by (nation string, city string) -ROW FORMAT SERDE - 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' -STORED AS INPUTFORMAT - 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat' -OUTPUTFORMAT - 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat' -LOCATION - '/user/doris/preinstalled_data/parquet_table/parquet_partition_table' -TBLPROPERTIES ( - 'transient_lastDdlTime'='1661955829'); - -msck repair table parquet_partition_table; - - -CREATE EXTERNAL TABLE `parquet_delta_binary_packed`( - bitwidth0 bigint, - bitwidth1 bigint, - bitwidth2 bigint, - bitwidth3 bigint, - bitwidth4 bigint, - bitwidth5 bigint, - bitwidth6 bigint, - bitwidth7 bigint, - bitwidth8 bigint, - bitwidth9 bigint, - bitwidth10 bigint, - bitwidth11 bigint, - bitwidth12 bigint, - bitwidth13 bigint, - bitwidth14 bigint, - bitwidth15 bigint, - bitwidth16 bigint, - bitwidth17 bigint, - bitwidth18 bigint, - bitwidth19 bigint, - bitwidth20 bigint, - bitwidth21 bigint, - bitwidth22 bigint, - bitwidth23 bigint, - bitwidth24 bigint, - bitwidth25 bigint, - bitwidth26 bigint, - bitwidth27 bigint, - bitwidth28 bigint, - bitwidth29 bigint, - bitwidth30 bigint, - bitwidth31 bigint, - bitwidth32 bigint, - bitwidth33 bigint, - bitwidth34 bigint, - bitwidth35 bigint, - bitwidth36 bigint, - bitwidth37 bigint, - bitwidth38 bigint, - bitwidth39 bigint, - bitwidth40 bigint, - bitwidth41 bigint, - bitwidth42 bigint, - bitwidth43 bigint, - bitwidth44 bigint, - bitwidth45 bigint, - bitwidth46 bigint, - bitwidth47 bigint, - bitwidth48 bigint, - bitwidth49 bigint, - bitwidth50 bigint, - bitwidth51 bigint, - bitwidth52 bigint, - bitwidth53 bigint, - bitwidth54 bigint, - bitwidth55 bigint, - bitwidth56 bigint, - bitwidth57 bigint, - bitwidth58 bigint, - bitwidth59 bigint, - bitwidth60 bigint, - bitwidth61 bigint, - bitwidth62 bigint, - bitwidth63 bigint, - bitwidth64 bigint, - int_value int - ) -STORED AS parquet -LOCATION - '/user/doris/preinstalled_data/parquet_table/parquet_delta_binary_packed' -TBLPROPERTIES ( - 'transient_lastDdlTime'='1661955829'); - -msck repair table parquet_delta_binary_packed; - -CREATE TABLE `parquet_alltypes_tiny_pages`( - bool_col boolean, - tinyint_col int, - smallint_col int, - int_col int, - bigint_col bigint, - float_col float, - double_col double, - id int, - date_string_col string, - string_col string, - timestamp_col timestamp, - year int, - month int - ) -ROW FORMAT SERDE - 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' -STORED AS INPUTFORMAT - 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat' -OUTPUTFORMAT - 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat' -LOCATION - '/user/doris/preinstalled_data/parquet_table/parquet_alltypes_tiny_pages' -TBLPROPERTIES ( - 'transient_lastDdlTime'='1661955829'); - -msck repair table parquet_alltypes_tiny_pages; - - -CREATE EXTERNAL TABLE IF NOT EXISTS `orc_all_types_partition`( - `tinyint_col` tinyint, - `smallint_col` smallint, - `int_col` int, - `bigint_col` bigint, - `boolean_col` boolean, - `float_col` float, - `double_col` double, - `string_col` string, - `binary_col` binary, - `timestamp_col` timestamp, - `decimal_col` decimal(12,4), - `char_col` char(50), - `varchar_col` varchar(50), - `date_col` date, - `list_double_col` array, - `list_string_col` array) -PARTITIONED BY ( - `p1_col` string, - `p2_col` string) -ROW FORMAT SERDE - 'org.apache.hadoop.hive.ql.io.orc.OrcSerde' -STORED AS INPUTFORMAT - 'org.apache.hadoop.hive.ql.io.orc.OrcInputFormat' -OUTPUTFORMAT - 'org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat' -LOCATION - '/user/doris/preinstalled_data/orc_table/orc_all_types_partition'; - -msck repair table orc_all_types_partition; - -CREATE external TABLE `csv_partition_table`( - `k1` string COMMENT 'k1', - `k2` string COMMENT 'k2', - `k3` string COMMENT 'k3', - `k4` string COMMENT 'k4', - `k5` string COMMENT 'k5', - `k6` string COMMENT 'k6', - `k7` string COMMENT 'k7', - `k8` string COMMENT 'k8', - `k9` string COMMENT 'k9', - `k10` string COMMENT 'k10', - `k11` string COMMENT 'k11', - `k12` string COMMENT 'k12', - `k13` string COMMENT 'k13', - `k14` string COMMENT 'k14', - `k15` string COMMENT 'k15') -PARTITIONED BY ( - `dt` string) -ROW FORMAT SERDE - 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' -WITH SERDEPROPERTIES ( - 'field.delim'='|', - 'serialization.format'='|') -STORED AS INPUTFORMAT - 'org.apache.hadoop.mapred.TextInputFormat' -OUTPUTFORMAT - 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' -LOCATION - '/user/doris/preinstalled_data/csv/csv_partition_table/' -TBLPROPERTIES ( - 'transient_lastDdlTime'='1669304897'); - -msck repair table csv_partition_table; - -CREATE TABLE `parquet_all_types`( - `t_null_string` string, - `t_null_varchar` varchar(65535), - `t_null_char` char(10), - `t_null_decimal_precision_2` decimal(2,1), - `t_null_decimal_precision_4` decimal(4,2), - `t_null_decimal_precision_8` decimal(8,4), - `t_null_decimal_precision_17` decimal(17,8), - `t_null_decimal_precision_18` decimal(18,8), - `t_null_decimal_precision_38` decimal(38,16), - `t_empty_string` string, - `t_string` string, - `t_empty_varchar` varchar(65535), - `t_varchar` varchar(65535), - `t_varchar_max_length` varchar(65535), - `t_char` char(10), - `t_int` int, - `t_bigint` bigint, - `t_float` float, - `t_double` double, - `t_boolean_true` boolean, - `t_boolean_false` boolean, - `t_decimal_precision_2` decimal(2,1), - `t_decimal_precision_4` decimal(4,2), - `t_decimal_precision_8` decimal(8,4), - `t_decimal_precision_17` decimal(17,8), - `t_decimal_precision_18` decimal(18,8), - `t_decimal_precision_38` decimal(38,16), - `t_binary` binary, - `t_map_string` map, - `t_map_varchar` map, - `t_map_char` map, - `t_map_int` map, - `t_map_bigint` map, - `t_map_float` map, - `t_map_double` map, - `t_map_boolean` map, - `t_map_decimal_precision_2` map, - `t_map_decimal_precision_4` map, - `t_map_decimal_precision_8` map, - `t_map_decimal_precision_17` map, - `t_map_decimal_precision_18` map, - `t_map_decimal_precision_38` map, - `t_array_string` array, - `t_array_int` array, - `t_array_bigint` array, - `t_array_float` array, - `t_array_double` array, - `t_array_boolean` array, - `t_array_varchar` array, - `t_array_char` array, - `t_array_decimal_precision_2` array, - `t_array_decimal_precision_4` array, - `t_array_decimal_precision_8` array, - `t_array_decimal_precision_17` array, - `t_array_decimal_precision_18` array, - `t_array_decimal_precision_38` array, - `t_struct_bigint` struct, - `t_complex` map>>, - `t_struct_nested` struct>, - `t_struct_null` struct, - `t_struct_non_nulls_after_nulls` struct, - `t_nested_struct_non_nulls_after_nulls` struct>, - `t_map_null_value` map, - `t_array_string_starting_with_nulls` array, - `t_array_string_with_nulls_in_between` array, - `t_array_string_ending_with_nulls` array, - `t_array_string_all_nulls` array -) ROW FORMAT SERDE - 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' -STORED AS INPUTFORMAT - 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat' -OUTPUTFORMAT - 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat' -LOCATION - '/user/doris/preinstalled_data/parquet_table/parquet_all_types' -TBLPROPERTIES ( - 'transient_lastDdlTime'='1681213018'); - -msck repair table parquet_all_types; - -CREATE TABLE IF NOT EXISTS `avro_all_types`( -`t_null_string` string, -`t_null_varchar` varchar(65535), -`t_null_char` char(10), -`t_null_array_int` array, -`t_null_decimal_precision_2` decimal(2,1), -`t_null_decimal_precision_4` decimal(4,2), -`t_null_decimal_precision_8` decimal(8,4), -`t_null_decimal_precision_17` decimal(17,8), -`t_null_decimal_precision_18` decimal(18,8), -`t_null_decimal_precision_38` decimal(38,16), -`t_empty_string` string, -`t_string` string, -`t_empty_varchar` varchar(65535), -`t_varchar` varchar(65535), -`t_varchar_max_length` varchar(65535), -`t_char` char(10), -`t_int` int, -`t_bigint` bigint, -`t_float` float, -`t_double` double, -`t_boolean_true` boolean, -`t_boolean_false` boolean, -`t_date` date, -`t_timestamp` timestamp, -`t_decimal_precision_2` decimal(2,1), -`t_decimal_precision_4` decimal(4,2), -`t_decimal_precision_8` decimal(8,4), -`t_decimal_precision_17` decimal(17,8), -`t_decimal_precision_18` decimal(18,8), -`t_decimal_precision_38` decimal(38,16), -`t_binary` binary, -`t_map_string` map, -`t_array_empty` array, -`t_array_string` array, -`t_array_int` array, -`t_array_bigint` array, -`t_array_float` array, -`t_array_double` array, -`t_array_boolean` array, -`t_array_varchar` array, -`t_array_char` array, -`t_array_date` array, -`t_array_timestamp` array, -`t_array_decimal_precision_2` array, -`t_array_decimal_precision_4` array, -`t_array_decimal_precision_8` array, -`t_array_decimal_precision_17` array, -`t_array_decimal_precision_18` array, -`t_array_decimal_precision_38` array, -`t_struct_bigint` struct, -`t_complex` map>>, -`t_struct_nested` struct>, -`t_struct_null` struct, -`t_struct_non_nulls_after_nulls` struct, -`t_array_string_starting_with_nulls` array, -`t_array_string_with_nulls_in_between` array, -`t_array_string_ending_with_nulls` array, -`t_array_string_all_nulls` array -) -ROW FORMAT SERDE - 'org.apache.hadoop.hive.serde2.avro.AvroSerDe' -STORED AS INPUTFORMAT - 'org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat' -OUTPUTFORMAT - 'org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat' -LOCATION - '/user/doris/preinstalled_data/avro/avro_all_types'; - -msck repair table avro_all_types; - - -CREATE TABLE IF NOT EXISTS `orc_all_types_t`( -`t_null_string` string, -`t_null_varchar` varchar(65535), -`t_null_char` char(10), -`t_null_array_int` array, -`t_null_decimal_precision_2` decimal(2,1), -`t_null_decimal_precision_4` decimal(4,2), -`t_null_decimal_precision_8` decimal(8,4), -`t_null_decimal_precision_17` decimal(17,8), -`t_null_decimal_precision_18` decimal(18,8), -`t_null_decimal_precision_38` decimal(38,16), -`t_empty_string` string, -`t_string` string, -`t_empty_varchar` varchar(65535), -`t_varchar` varchar(65535), -`t_varchar_max_length` varchar(65535), -`t_char` char(10), -`t_tinyint` tinyint, -`t_smallint` smallint, -`t_int` int, -`t_bigint` bigint, -`t_float` float, -`t_double` double, -`t_boolean_true` boolean, -`t_boolean_false` boolean, -`t_date` date, -`t_timestamp` timestamp, -`t_decimal_precision_2` decimal(2,1), -`t_decimal_precision_4` decimal(4,2), -`t_decimal_precision_8` decimal(8,4), -`t_decimal_precision_17` decimal(17,8), -`t_decimal_precision_18` decimal(18,8), -`t_decimal_precision_38` decimal(38,16), -`t_binary` binary, -`t_map_string` map, -`t_map_tinyint` map, -`t_map_varchar` map, -`t_map_char` map, -`t_map_smallint` map, -`t_map_int` map, -`t_map_bigint` map, -`t_map_float` map, -`t_map_double` map, -`t_map_boolean` map, -`t_map_date` map, -`t_map_timestamp` map, -`t_map_decimal_precision_2` map, -`t_map_decimal_precision_4` map, -`t_map_decimal_precision_8` map, -`t_map_decimal_precision_17` map, -`t_map_decimal_precision_18` map, -`t_map_decimal_precision_38` map, -`t_array_empty` array, -`t_array_string` array, -`t_array_tinyint` array, -`t_array_smallint` array, -`t_array_int` array, -`t_array_bigint` array, -`t_array_float` array, -`t_array_double` array, -`t_array_boolean` array, -`t_array_varchar` array, -`t_array_char` array, -`t_array_date` array, -`t_array_timestamp` array, -`t_array_decimal_precision_2` array, -`t_array_decimal_precision_4` array, -`t_array_decimal_precision_8` array, -`t_array_decimal_precision_17` array, -`t_array_decimal_precision_18` array, -`t_array_decimal_precision_38` array, -`t_struct_bigint` struct, -`t_complex` map>>, -`t_struct_nested` struct>, -`t_struct_null` struct, -`t_struct_non_nulls_after_nulls` struct, -`t_nested_struct_non_nulls_after_nulls` struct>, -`t_map_null_value` map, -`t_array_string_starting_with_nulls` array, -`t_array_string_with_nulls_in_between` array, -`t_array_string_ending_with_nulls` array, -`t_array_string_all_nulls` array -) -ROW FORMAT SERDE - 'org.apache.hadoop.hive.ql.io.orc.OrcSerde' -STORED AS INPUTFORMAT - 'org.apache.hadoop.hive.ql.io.orc.OrcInputFormat' -OUTPUTFORMAT - 'org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat' -LOCATION - '/user/doris/preinstalled_data/orc_table/orc_all_types'; - -msck repair table orc_all_types_t; - -CREATE TABLE IF NOT EXISTS `json_all_types`( -`t_null_string` string, -`t_null_varchar` varchar(65535), -`t_null_char` char(10), -`t_null_decimal_precision_2` decimal(2,1), -`t_null_decimal_precision_4` decimal(4,2), -`t_null_decimal_precision_8` decimal(8,4), -`t_null_decimal_precision_17` decimal(17,8), -`t_null_decimal_precision_18` decimal(18,8), -`t_null_decimal_precision_38` decimal(38,16), -`t_empty_string` string, -`t_string` string, -`t_empty_varchar` varchar(65535), -`t_varchar` varchar(65535), -`t_varchar_max_length` varchar(65535), -`t_char` char(10), -`t_tinyint` tinyint, -`t_smallint` smallint, -`t_int` int, -`t_bigint` bigint, -`t_float` float, -`t_double` double, -`t_boolean_true` boolean, -`t_boolean_false` boolean, -`t_date` date, -`t_timestamp` timestamp, -`t_decimal_precision_2` decimal(2,1), -`t_decimal_precision_4` decimal(4,2), -`t_decimal_precision_8` decimal(8,4), -`t_decimal_precision_17` decimal(17,8), -`t_decimal_precision_18` decimal(18,8) -) -ROW FORMAT SERDE - 'org.apache.hive.hcatalog.data.JsonSerDe' -STORED AS TEXTFILE -LOCATION - '/user/doris/preinstalled_data/json/json_all_types'; - -msck repair table json_all_types; - - -CREATE TABLE IF NOT EXISTS `csv_all_types`( -`t_empty_string` string, -`t_string` string -) -ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' -STORED AS TEXTFILE -LOCATION - '/user/doris/preinstalled_data/csv/csv_all_types'; - -msck repair table csv_all_types; - -CREATE TABLE IF NOT EXISTS `text_all_types`( -`t_null_string` string, -`t_null_varchar` varchar(65535), -`t_null_char` char(10), -`t_null_decimal_precision_2` decimal(2,1), -`t_null_decimal_precision_4` decimal(4,2), -`t_null_decimal_precision_8` decimal(8,4), -`t_null_decimal_precision_17` decimal(17,8), -`t_null_decimal_precision_18` decimal(18,8), -`t_null_decimal_precision_38` decimal(38,16), -`t_empty_string` string, -`t_string` string, -`t_empty_varchar` varchar(65535), -`t_varchar` varchar(65535), -`t_varchar_max_length` varchar(65535), -`t_char` char(10), -`t_tinyint` tinyint, -`t_smallint` smallint, -`t_int` int, -`t_bigint` bigint, -`t_float` float, -`t_double` double, -`t_boolean_true` boolean, -`t_boolean_false` boolean, -`t_date` date, -`t_timestamp` timestamp, -`t_decimal_precision_2` decimal(2,1), -`t_decimal_precision_4` decimal(4,2), -`t_decimal_precision_8` decimal(8,4), -`t_decimal_precision_17` decimal(17,8), -`t_decimal_precision_18` decimal(18,8), -`t_decimal_precision_38` decimal(38,16), -`t_binary` binary -) -ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t' -STORED AS TEXTFILE -LOCATION - '/user/doris/preinstalled_data/text/text_all_types'; - -msck repair table text_all_types; - - -CREATE TABLE IF NOT EXISTS `sequence_all_types`( -`t_null_string` string, -`t_null_varchar` varchar(65535), -`t_null_char` char(10), -`t_null_array_int` array, -`t_null_decimal_precision_2` decimal(2,1), -`t_null_decimal_precision_4` decimal(4,2), -`t_null_decimal_precision_8` decimal(8,4), -`t_null_decimal_precision_17` decimal(17,8), -`t_null_decimal_precision_18` decimal(18,8), -`t_null_decimal_precision_38` decimal(38,16), -`t_empty_string` string, -`t_string` string, -`t_empty_varchar` varchar(65535), -`t_varchar` varchar(65535), -`t_varchar_max_length` varchar(65535), -`t_char` char(10), -`t_tinyint` tinyint, -`t_smallint` smallint, -`t_int` int, -`t_bigint` bigint, -`t_float` float, -`t_double` double, -`t_boolean_true` boolean, -`t_boolean_false` boolean, -`t_date` date, -`t_timestamp` timestamp, -`t_decimal_precision_2` decimal(2,1), -`t_decimal_precision_4` decimal(4,2), -`t_decimal_precision_8` decimal(8,4), -`t_decimal_precision_17` decimal(17,8), -`t_decimal_precision_18` decimal(18,8), -`t_decimal_precision_38` decimal(38,16), -`t_binary` binary, -`t_map_string` map, -`t_map_tinyint` map, -`t_map_varchar` map, -`t_map_char` map, -`t_map_smallint` map, -`t_map_int` map, -`t_map_bigint` map, -`t_map_float` map, -`t_map_double` map, -`t_map_boolean` map, -`t_map_date` map, -`t_map_timestamp` map, -`t_map_decimal_precision_2` map, -`t_map_decimal_precision_4` map, -`t_map_decimal_precision_8` map, -`t_map_decimal_precision_17` map, -`t_map_decimal_precision_18` map, -`t_map_decimal_precision_38` map, -`t_array_empty` array, -`t_array_string` array, -`t_array_tinyint` array, -`t_array_smallint` array, -`t_array_int` array, -`t_array_bigint` array, -`t_array_float` array, -`t_array_double` array, -`t_array_boolean` array, -`t_array_varchar` array, -`t_array_char` array, -`t_array_date` array, -`t_array_timestamp` array, -`t_array_decimal_precision_2` array, -`t_array_decimal_precision_4` array, -`t_array_decimal_precision_8` array, -`t_array_decimal_precision_17` array, -`t_array_decimal_precision_18` array, -`t_array_decimal_precision_38` array, -`t_struct_bigint` struct, -`t_complex` map>>, -`t_struct_nested` struct>, -`t_struct_null` struct, -`t_struct_non_nulls_after_nulls` struct, -`t_nested_struct_non_nulls_after_nulls` struct>, -`t_map_null_value` map, -`t_array_string_starting_with_nulls` array, -`t_array_string_with_nulls_in_between` array, -`t_array_string_ending_with_nulls` array, -`t_array_string_all_nulls` array -) -STORED AS SEQUENCEFILE -LOCATION - '/user/doris/preinstalled_data/sequence/sequence_all_types'; - -msck repair table sequence_all_types; - -CREATE TABLE `parquet_gzip_all_types`( -`t_null_string` string, -`t_null_varchar` varchar(65535), -`t_null_char` char(10), -`t_null_decimal_precision_2` decimal(2,1), -`t_null_decimal_precision_4` decimal(4,2), -`t_null_decimal_precision_8` decimal(8,4), -`t_null_decimal_precision_17` decimal(17,8), -`t_null_decimal_precision_18` decimal(18,8), -`t_null_decimal_precision_38` decimal(38,16), -`t_empty_string` string, -`t_string` string, -`t_empty_varchar` varchar(65535), -`t_varchar` varchar(65535), -`t_varchar_max_length` varchar(65535), -`t_char` char(10), -`t_int` int, -`t_bigint` bigint, -`t_float` float, -`t_double` double, -`t_boolean_true` boolean, -`t_boolean_false` boolean, -`t_decimal_precision_2` decimal(2,1), -`t_decimal_precision_4` decimal(4,2), -`t_decimal_precision_8` decimal(8,4), -`t_decimal_precision_17` decimal(17,8), -`t_decimal_precision_18` decimal(18,8), -`t_decimal_precision_38` decimal(38,16), -`t_binary` binary, -`t_map_string` map, -`t_map_varchar` map, -`t_map_char` map, -`t_map_int` map, -`t_map_bigint` map, -`t_map_float` map, -`t_map_double` map, -`t_map_boolean` map, -`t_map_decimal_precision_2` map, -`t_map_decimal_precision_4` map, -`t_map_decimal_precision_8` map, -`t_map_decimal_precision_17` map, -`t_map_decimal_precision_18` map, -`t_map_decimal_precision_38` map, -`t_array_string` array, -`t_array_int` array, -`t_array_bigint` array, -`t_array_float` array, -`t_array_double` array, -`t_array_boolean` array, -`t_array_varchar` array, -`t_array_char` array, -`t_array_decimal_precision_2` array, -`t_array_decimal_precision_4` array, -`t_array_decimal_precision_8` array, -`t_array_decimal_precision_17` array, -`t_array_decimal_precision_18` array, -`t_array_decimal_precision_38` array, -`t_struct_bigint` struct, -`t_complex` map>>, -`t_struct_nested` struct>, -`t_struct_null` struct, -`t_struct_non_nulls_after_nulls` struct, -`t_nested_struct_non_nulls_after_nulls` struct>, -`t_map_null_value` map, -`t_array_string_starting_with_nulls` array, -`t_array_string_with_nulls_in_between` array, -`t_array_string_ending_with_nulls` array, -`t_array_string_all_nulls` array -) ROW FORMAT SERDE - 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' -STORED AS INPUTFORMAT - 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat' -OUTPUTFORMAT - 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat' -LOCATION - '/user/doris/preinstalled_data/parquet_table/parquet_gzip_all_types' -TBLPROPERTIES ( - 'transient_lastDdlTime'='1681213018', - "parquet.compression"="GZIP"); - -msck repair table parquet_gzip_all_types; - -CREATE TABLE `parquet_zstd_all_types`( -`t_null_string` string, -`t_null_varchar` varchar(65535), -`t_null_char` char(10), -`t_null_decimal_precision_2` decimal(2,1), -`t_null_decimal_precision_4` decimal(4,2), -`t_null_decimal_precision_8` decimal(8,4), -`t_null_decimal_precision_17` decimal(17,8), -`t_null_decimal_precision_18` decimal(18,8), -`t_null_decimal_precision_38` decimal(38,16), -`t_empty_string` string, -`t_string` string, -`t_empty_varchar` varchar(65535), -`t_varchar` varchar(65535), -`t_varchar_max_length` varchar(65535), -`t_char` char(10), -`t_int` int, -`t_bigint` bigint, -`t_float` float, -`t_double` double, -`t_boolean_true` boolean, -`t_boolean_false` boolean, -`t_decimal_precision_2` decimal(2,1), -`t_decimal_precision_4` decimal(4,2), -`t_decimal_precision_8` decimal(8,4), -`t_decimal_precision_17` decimal(17,8), -`t_decimal_precision_18` decimal(18,8), -`t_decimal_precision_38` decimal(38,16), -`t_binary` binary, -`t_map_string` map, -`t_map_varchar` map, -`t_map_char` map, -`t_map_int` map, -`t_map_bigint` map, -`t_map_float` map, -`t_map_double` map, -`t_map_boolean` map, -`t_map_decimal_precision_2` map, -`t_map_decimal_precision_4` map, -`t_map_decimal_precision_8` map, -`t_map_decimal_precision_17` map, -`t_map_decimal_precision_18` map, -`t_map_decimal_precision_38` map, -`t_array_string` array, -`t_array_int` array, -`t_array_bigint` array, -`t_array_float` array, -`t_array_double` array, -`t_array_boolean` array, -`t_array_varchar` array, -`t_array_char` array, -`t_array_decimal_precision_2` array, -`t_array_decimal_precision_4` array, -`t_array_decimal_precision_8` array, -`t_array_decimal_precision_17` array, -`t_array_decimal_precision_18` array, -`t_array_decimal_precision_38` array, -`t_struct_bigint` struct, -`t_complex` map>>, -`t_struct_nested` struct>, -`t_struct_null` struct, -`t_struct_non_nulls_after_nulls` struct, -`t_nested_struct_non_nulls_after_nulls` struct>, -`t_map_null_value` map, -`t_array_string_starting_with_nulls` array, -`t_array_string_with_nulls_in_between` array, -`t_array_string_ending_with_nulls` array, -`t_array_string_all_nulls` array -) ROW FORMAT SERDE - 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' -STORED AS INPUTFORMAT - 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat' -OUTPUTFORMAT - 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat' -LOCATION - '/user/doris/preinstalled_data/parquet_table/parquet_gzip_all_types' -TBLPROPERTIES ( - 'transient_lastDdlTime'='1681213018', - "parquet.compression"="ZSTD"); - -msck repair table parquet_zstd_all_types; - -CREATE TABLE `rcbinary_all_types`( -`t_null_string` string, -`t_null_varchar` varchar(65535), -`t_null_char` char(10), -`t_null_array_int` array, -`t_null_decimal_precision_2` decimal(2,1), -`t_null_decimal_precision_4` decimal(4,2), -`t_null_decimal_precision_8` decimal(8,4), -`t_null_decimal_precision_17` decimal(17,8), -`t_null_decimal_precision_18` decimal(18,8), -`t_null_decimal_precision_38` decimal(38,16), -`t_empty_string` string, -`t_string` string, -`t_varchar` varchar(65535), -`t_varchar_max_length` varchar(65535), -`t_char` char(10), -`t_tinyint` tinyint, -`t_smallint` smallint, -`t_int` int, -`t_bigint` bigint, -`t_float` float, -`t_double` double, -`t_boolean_true` boolean, -`t_boolean_false` boolean, -`t_date` date, -`t_timestamp` timestamp, -`t_decimal_precision_2` decimal(2,1), -`t_decimal_precision_4` decimal(4,2), -`t_decimal_precision_8` decimal(8,4), -`t_decimal_precision_17` decimal(17,8), -`t_decimal_precision_18` decimal(18,8), -`t_decimal_precision_38` decimal(38,16), -`t_binary` binary, -`t_map_string` map, -`t_map_tinyint` map, -`t_map_varchar` map, -`t_map_char` map, -`t_map_smallint` map, -`t_map_int` map, -`t_map_bigint` map, -`t_map_float` map, -`t_map_double` map, -`t_map_boolean` map, -`t_map_date` map, -`t_map_timestamp` map, -`t_map_decimal_precision_2` map, -`t_map_decimal_precision_4` map, -`t_map_decimal_precision_8` map, -`t_map_decimal_precision_17` map, -`t_map_decimal_precision_18` map, -`t_map_decimal_precision_38` map, -`t_array_empty` array, -`t_array_string` array, -`t_array_tinyint` array, -`t_array_smallint` array, -`t_array_int` array, -`t_array_bigint` array, -`t_array_float` array, -`t_array_double` array, -`t_array_boolean` array, -`t_array_varchar` array, -`t_array_char` array, -`t_array_date` array, -`t_array_timestamp` array, -`t_array_decimal_precision_2` array, -`t_array_decimal_precision_4` array, -`t_array_decimal_precision_8` array, -`t_array_decimal_precision_17` array, -`t_array_decimal_precision_18` array, -`t_array_decimal_precision_38` array, -`t_struct_bigint` struct, -`t_complex` map>>, -`t_struct_nested` struct>, -`t_struct_null` struct, -`t_struct_non_nulls_after_nulls` struct, -`t_nested_struct_non_nulls_after_nulls` struct>, -`t_map_null_value` map, -`t_array_string_starting_with_nulls` array, -`t_array_string_with_nulls_in_between` array, -`t_array_string_ending_with_nulls` array, -`t_array_string_all_nulls` array -) -ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.columnar.LazyBinaryColumnarSerDe' -STORED AS RCFILE -LOCATION - '/user/doris/preinstalled_data/rcbinary/rcbinary_all_types'; - -msck repair table rcbinary_all_types; - -CREATE TABLE `bloom_parquet_table`( -`t_null_string` string, -`t_null_varchar` varchar(65535), -`t_null_char` char(10), -`t_null_decimal_precision_2` decimal(2,1), -`t_null_decimal_precision_4` decimal(4,2), -`t_null_decimal_precision_8` decimal(8,4), -`t_null_decimal_precision_17` decimal(17,8), -`t_null_decimal_precision_18` decimal(18,8), -`t_null_decimal_precision_38` decimal(38,16), -`t_empty_string` string, -`t_string` string, -`t_empty_varchar` varchar(65535), -`t_varchar` varchar(65535), -`t_varchar_max_length` varchar(65535), -`t_char` char(10), -`t_int` int, -`t_bigint` bigint, -`t_float` float, -`t_double` double, -`t_boolean_true` boolean, -`t_boolean_false` boolean, -`t_decimal_precision_2` decimal(2,1), -`t_decimal_precision_4` decimal(4,2), -`t_decimal_precision_8` decimal(8,4), -`t_decimal_precision_17` decimal(17,8), -`t_decimal_precision_18` decimal(18,8), -`t_decimal_precision_38` decimal(38,16), -`t_binary` binary, -`t_array_string` array, -`t_array_int` array, -`t_array_bigint` array, -`t_array_float` array, -`t_array_double` array, -`t_array_boolean` array, -`t_array_varchar` array, -`t_array_char` array, -`t_array_decimal_precision_2` array, -`t_array_decimal_precision_4` array, -`t_array_decimal_precision_8` array, -`t_array_decimal_precision_17` array, -`t_array_decimal_precision_18` array, -`t_array_decimal_precision_38` array, -`t_struct_bigint` struct, -`t_struct_nested` struct>, -`t_struct_null` struct, -`t_struct_non_nulls_after_nulls` struct, -`t_nested_struct_non_nulls_after_nulls` struct>, -`t_array_string_starting_with_nulls` array, -`t_array_string_with_nulls_in_between` array, -`t_array_string_ending_with_nulls` array, -`t_array_string_all_nulls` array -) ROW FORMAT SERDE - 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' -STORED AS INPUTFORMAT - 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat' -OUTPUTFORMAT - 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat' -LOCATION - '/user/doris/preinstalled_data/parquet_table/bloom_parquet_table' -TBLPROPERTIES ( - 'transient_lastDdlTime'='1681213018', - 'parquet.bloom.filter.columns'='t_int', - 'parquet.bloom.filter.fpp'='0.05'); - -msck repair table bloom_parquet_table; - - -CREATE TABLE `bloom_orc_table`( -`t_null_string` string, -`t_null_varchar` varchar(65535), -`t_null_char` char(10), -`t_null_array_int` array, -`t_null_decimal_precision_2` decimal(2,1), -`t_null_decimal_precision_4` decimal(4,2), -`t_null_decimal_precision_8` decimal(8,4), -`t_null_decimal_precision_17` decimal(17,8), -`t_null_decimal_precision_18` decimal(18,8), -`t_null_decimal_precision_38` decimal(38,16), -`t_empty_string` string, -`t_string` string, -`t_empty_varchar` varchar(65535), -`t_varchar` varchar(65535), -`t_varchar_max_length` varchar(65535), -`t_char` char(10), -`t_tinyint` tinyint, -`t_smallint` smallint, -`t_int` int, -`t_bigint` bigint, -`t_float` float, -`t_double` double, -`t_boolean_true` boolean, -`t_boolean_false` boolean, -`t_date` date, -`t_timestamp` timestamp, -`t_decimal_precision_2` decimal(2,1), -`t_decimal_precision_4` decimal(4,2), -`t_decimal_precision_8` decimal(8,4), -`t_decimal_precision_17` decimal(17,8), -`t_decimal_precision_18` decimal(18,8), -`t_decimal_precision_38` decimal(38,16), -`t_binary` binary, -`t_map_string` map, -`t_map_tinyint` map, -`t_map_varchar` map, -`t_map_char` map, -`t_map_smallint` map, -`t_map_int` map, -`t_map_bigint` map, -`t_map_float` map, -`t_map_double` map, -`t_map_boolean` map, -`t_map_date` map, -`t_map_timestamp` map, -`t_map_decimal_precision_2` map, -`t_map_decimal_precision_4` map, -`t_map_decimal_precision_8` map, -`t_map_decimal_precision_17` map, -`t_map_decimal_precision_18` map, -`t_map_decimal_precision_38` map, -`t_array_empty` array, -`t_array_string` array, -`t_array_tinyint` array, -`t_array_smallint` array, -`t_array_int` array, -`t_array_bigint` array, -`t_array_float` array, -`t_array_double` array, -`t_array_boolean` array, -`t_array_varchar` array, -`t_array_char` array, -`t_array_date` array, -`t_array_timestamp` array, -`t_array_decimal_precision_2` array, -`t_array_decimal_precision_4` array, -`t_array_decimal_precision_8` array, -`t_array_decimal_precision_17` array, -`t_array_decimal_precision_18` array, -`t_array_decimal_precision_38` array, -`t_struct_bigint` struct, -`t_complex` map>>, -`t_struct_nested` struct>, -`t_struct_null` struct, -`t_struct_non_nulls_after_nulls` struct, -`t_nested_struct_non_nulls_after_nulls` struct>, -`t_map_null_value` map, -`t_array_string_starting_with_nulls` array, -`t_array_string_with_nulls_in_between` array, -`t_array_string_ending_with_nulls` array, -`t_array_string_all_nulls` array -) -ROW FORMAT SERDE - 'org.apache.hadoop.hive.ql.io.orc.OrcSerde' -STORED AS INPUTFORMAT - 'org.apache.hadoop.hive.ql.io.orc.OrcInputFormat' -OUTPUTFORMAT - 'org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat' -LOCATION - '/user/doris/preinstalled_data/orc_table/bloom_orc_table' -TBLPROPERTIES ( - 'transient_lastDdlTime'='1681213018', - 'orc.bloom.filter.columns'='t_int', - 'orc.bloom.filter.fpp'='0.05'); - -msck repair table bloom_orc_table; - - -CREATE TABLE `orc_predicate_table`( -`column_primitive_integer` int, -`column1_struct` struct, -`column_primitive_bigint` bigint -) -ROW FORMAT SERDE - 'org.apache.hadoop.hive.ql.io.orc.OrcSerde' -STORED AS INPUTFORMAT - 'org.apache.hadoop.hive.ql.io.orc.OrcInputFormat' -OUTPUTFORMAT - 'org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat' -LOCATION - '/user/doris/preinstalled_data/orc_table/orc_predicate_table'; - -msck repair table orc_predicate_table; - - -CREATE TABLE `parquet_predicate_table`( -`column_primitive_integer` int, -`column1_struct` struct, -`column_primitive_bigint` bigint -) ROW FORMAT SERDE - 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' -STORED AS INPUTFORMAT - 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat' -OUTPUTFORMAT - 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat' -LOCATION - '/user/doris/preinstalled_data/parquet_table/parquet_predicate_table'; - -msck repair table parquet_predicate_table; - -CREATE TABLE `only_null`( -`x` int -) ROW FORMAT SERDE - 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' -STORED AS INPUTFORMAT - 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat' -OUTPUTFORMAT - 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat' -LOCATION - '/user/doris/preinstalled_data/parquet_table/only_null'; - -msck repair table only_null; - - -CREATE TABLE `parquet_timestamp_millis`( -test timestamp -) ROW FORMAT SERDE - 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' -STORED AS INPUTFORMAT - 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat' -OUTPUTFORMAT - 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat' -LOCATION - '/user/doris/preinstalled_data/parquet_table/parquet_timestamp_millis'; - -msck repair table parquet_timestamp_millis; - - -CREATE TABLE `parquet_timestamp_micros`( -test timestamp -) ROW FORMAT SERDE - 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' -STORED AS INPUTFORMAT - 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat' -OUTPUTFORMAT - 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat' -LOCATION - '/user/doris/preinstalled_data/parquet_table/parquet_timestamp_micros'; - -msck repair table parquet_timestamp_micros; - -CREATE TABLE `parquet_timestamp_nanos`( -test timestamp -) ROW FORMAT SERDE - 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' -STORED AS INPUTFORMAT - 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat' -OUTPUTFORMAT - 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat' -LOCATION - '/user/doris/preinstalled_data/parquet_table/parquet_timestamp_nanos'; - -msck repair table parquet_timestamp_nanos; - -CREATE TABLE `orc_decimal_table`( - id INT, - decimal_col1 DECIMAL(8, 4), - decimal_col2 DECIMAL(18, 6), - decimal_col3 DECIMAL(38, 12), - decimal_col4 DECIMAL(9, 0), - decimal_col5 DECIMAL(27, 9), - decimal_col6 DECIMAL(9, 0)) -ROW FORMAT SERDE - 'org.apache.hadoop.hive.ql.io.orc.OrcSerde' -STORED AS INPUTFORMAT - 'org.apache.hadoop.hive.ql.io.orc.OrcInputFormat' -OUTPUTFORMAT - 'org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat' -LOCATION - '/user/doris/preinstalled_data/orc_table/orc_decimal_table'; - -msck repair table orc_decimal_table; - -CREATE TABLE `parquet_decimal_bool`( - decimals decimal(20,3), - bool_rle boolean -) -ROW FORMAT SERDE - 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' -STORED AS INPUTFORMAT - 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat' -OUTPUTFORMAT - 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat' -LOCATION - '/user/doris/preinstalled_data/parquet_table/parquet_decimal_bool'; - -msck repair table partition_table; - -CREATE TABLE `parquet_decimal90_table`( - `decimal_col` decimal(9,0)) -ROW FORMAT SERDE - 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' -STORED AS INPUTFORMAT - 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat' -OUTPUTFORMAT - 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat' -LOCATION - '/user/doris/preinstalled_data/parquet_table/parquet_decimal90_table'; - -msck repair table parquet_decimal90_table; - -CREATE TABLE `fixed_length_byte_array_decimal_table`( - `decimal_col1` decimal(7,2), - `decimal_col2` decimal(7,2), - `decimal_col3` decimal(7,2), - `decimal_col4` decimal(7,2), - `decimal_col5` decimal(7,2)) -ROW FORMAT SERDE - 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' -STORED AS INPUTFORMAT - 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat' -OUTPUTFORMAT - 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat' -LOCATION - '/user/doris/preinstalled_data/parquet_table/fixed_length_byte_array_decimal_table' -TBLPROPERTIES ( - 'parquet.compress'='SNAPPY'); - -msck repair table fixed_length_byte_array_decimal_table; - -CREATE TABLE `string_col_dict_plain_mixed_orc`( - `col0` int, - `col1` string, - `col2` double, - `col3` boolean, - `col4` string, - `col5` int) -ROW FORMAT SERDE - 'org.apache.hadoop.hive.ql.io.orc.OrcSerde' -STORED AS INPUTFORMAT - 'org.apache.hadoop.hive.ql.io.orc.OrcInputFormat' -OUTPUTFORMAT - 'org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat' -LOCATION - '/user/doris/preinstalled_data/orc_table/string_col_dict_plain_mixed_orc' -TBLPROPERTIES ( - 'orc.compress'='ZLIB'); - -msck repair table string_col_dict_plain_mixed_orc; - -CREATE TABLE `test_string_dict_filter_parquet`( - `o_orderkey` int, - `o_custkey` int, - `o_orderstatus` string, - `o_totalprice` decimal(15,2), - `o_orderdate` date, - `o_orderpriority` string, - `o_clerk` string, - `o_shippriority` int, - `o_comment` string) -ROW FORMAT SERDE - 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' -STORED AS INPUTFORMAT - 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat' -OUTPUTFORMAT - 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat' -LOCATION - '/user/doris/preinstalled_data/parquet_table/test_string_dict_filter_parquet'; - -msck repair table test_string_dict_filter_parquet; - -CREATE TABLE `test_string_dict_filter_orc`( - `o_orderkey` int, - `o_custkey` int, - `o_orderstatus` string, - `o_totalprice` decimal(15,2), - `o_orderdate` date, - `o_orderpriority` string, - `o_clerk` string, - `o_shippriority` int, - `o_comment` string) -ROW FORMAT SERDE - 'org.apache.hadoop.hive.ql.io.orc.OrcSerde' -STORED AS INPUTFORMAT - 'org.apache.hadoop.hive.ql.io.orc.OrcInputFormat' -OUTPUTFORMAT - 'org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat' -LOCATION - '/user/doris/preinstalled_data/orc_table/test_string_dict_filter_orc'; - -msck repair table test_string_dict_filter_orc; - -show tables; - - -create database stats_test; -use stats_test; -create table stats_test1 (id INT, value STRING) STORED AS ORC; -create table stats_test2 (id INT, value STRING) STORED AS PARQUET; -create table stats_test3 (id INT, value STRING) STORED AS PARQUET; - -insert into stats_test1 values (1, 'name1'), (2, 'name2'), (3, 'name3'); -INSERT INTO stats_test2 VALUES (1, ';'), (2, '\*'); - -create table employee_gz(name string,salary string) -row format serde 'org.apache.hadoop.hive.serde2.OpenCSVSerde' -with serdeproperties -('quoteChar'='\"' -,'separatorChar'=','); - -insert into employee_gz values ('a', '1.1'), ('b', '2.2'); - -CREATE DATABASE write_test; -use write_test; - -CREATE TABLE `all_types_parquet_snappy_src`( - `boolean_col` boolean, - `tinyint_col` tinyint, - `smallint_col` smallint, - `int_col` int, - `bigint_col` bigint, - `float_col` float, - `double_col` double, - `decimal_col1` decimal(9,0), - `decimal_col2` decimal(8,4), - `decimal_col3` decimal(18,6), - `decimal_col4` decimal(38,12), - `string_col` string, - `binary_col` binary, - `date_col` date, - `timestamp_col1` timestamp, - `timestamp_col2` timestamp, - `timestamp_col3` timestamp, - `char_col1` char(50), - `char_col2` char(100), - `char_col3` char(255), - `varchar_col1` varchar(50), - `varchar_col2` varchar(100), - `varchar_col3` varchar(255), - `t_map_string` map, - `t_map_varchar` map, - `t_map_char` map, - `t_map_int` map, - `t_map_bigint` map, - `t_map_float` map, - `t_map_double` map, - `t_map_boolean` map, - `t_map_decimal_precision_2` map, - `t_map_decimal_precision_4` map, - `t_map_decimal_precision_8` map, - `t_map_decimal_precision_17` map, - `t_map_decimal_precision_18` map, - `t_map_decimal_precision_38` map, - `t_array_string` array, - `t_array_int` array, - `t_array_bigint` array, - `t_array_float` array, - `t_array_double` array, - `t_array_boolean` array, - `t_array_varchar` array, - `t_array_char` array, - `t_array_decimal_precision_2` array, - `t_array_decimal_precision_4` array, - `t_array_decimal_precision_8` array, - `t_array_decimal_precision_17` array, - `t_array_decimal_precision_18` array, - `t_array_decimal_precision_38` array, - `t_struct_bigint` struct, - `t_complex` map>>, - `t_struct_nested` struct>, - `t_struct_null` struct, - `t_struct_non_nulls_after_nulls` struct, - `t_nested_struct_non_nulls_after_nulls` struct>, - `t_map_null_value` map, - `t_array_string_starting_with_nulls` array, - `t_array_string_with_nulls_in_between` array, - `t_array_string_ending_with_nulls` array, - `t_array_string_all_nulls` array, - `dt` int) -stored as parquet -LOCATION - '/user/doris/preinstalled_data/parquet_table/all_types_parquet_snappy_src' -TBLPROPERTIES('parquet.compression'='SNAPPY'); - -CREATE TABLE `all_types_par_parquet_snappy_src`( - `boolean_col` boolean, - `tinyint_col` tinyint, - `smallint_col` smallint, - `int_col` int, - `bigint_col` bigint, - `float_col` float, - `double_col` double, - `decimal_col1` decimal(9,0), - `decimal_col2` decimal(8,4), - `decimal_col3` decimal(18,6), - `decimal_col4` decimal(38,12), - `string_col` string, - `binary_col` binary, - `date_col` date, - `timestamp_col1` timestamp, - `timestamp_col2` timestamp, - `timestamp_col3` timestamp, - `char_col1` char(50), - `char_col2` char(100), - `char_col3` char(255), - `varchar_col1` varchar(50), - `varchar_col2` varchar(100), - `varchar_col3` varchar(255), - `t_map_string` map, - `t_map_varchar` map, - `t_map_char` map, - `t_map_int` map, - `t_map_bigint` map, - `t_map_float` map, - `t_map_double` map, - `t_map_boolean` map, - `t_map_decimal_precision_2` map, - `t_map_decimal_precision_4` map, - `t_map_decimal_precision_8` map, - `t_map_decimal_precision_17` map, - `t_map_decimal_precision_18` map, - `t_map_decimal_precision_38` map, - `t_array_string` array, - `t_array_int` array, - `t_array_bigint` array, - `t_array_float` array, - `t_array_double` array, - `t_array_boolean` array, - `t_array_varchar` array, - `t_array_char` array, - `t_array_decimal_precision_2` array, - `t_array_decimal_precision_4` array, - `t_array_decimal_precision_8` array, - `t_array_decimal_precision_17` array, - `t_array_decimal_precision_18` array, - `t_array_decimal_precision_38` array, - `t_struct_bigint` struct, - `t_complex` map>>, - `t_struct_nested` struct>, - `t_struct_null` struct, - `t_struct_non_nulls_after_nulls` struct, - `t_nested_struct_non_nulls_after_nulls` struct>, - `t_map_null_value` map, - `t_array_string_starting_with_nulls` array, - `t_array_string_with_nulls_in_between` array, - `t_array_string_ending_with_nulls` array, - `t_array_string_all_nulls` array) -PARTITIONED BY ( - `dt` int) -stored as parquet -LOCATION - '/user/doris/preinstalled_data/parquet_table/all_types_par_parquet_snappy_src' -TBLPROPERTIES('parquet.compression'='SNAPPY'); -msck repair table all_types_par_parquet_snappy_src; - -CREATE TABLE `all_types_parquet_snappy`( - `boolean_col` boolean, - `tinyint_col` tinyint, - `smallint_col` smallint, - `int_col` int, - `bigint_col` bigint, - `float_col` float, - `double_col` double, - `decimal_col1` decimal(9,0), - `decimal_col2` decimal(8,4), - `decimal_col3` decimal(18,6), - `decimal_col4` decimal(38,12), - `string_col` string, - `binary_col` binary, - `date_col` date, - `timestamp_col1` timestamp, - `timestamp_col2` timestamp, - `timestamp_col3` timestamp, - `char_col1` char(50), - `char_col2` char(100), - `char_col3` char(255), - `varchar_col1` varchar(50), - `varchar_col2` varchar(100), - `varchar_col3` varchar(255), - `t_map_string` map, - `t_map_varchar` map, - `t_map_char` map, - `t_map_int` map, - `t_map_bigint` map, - `t_map_float` map, - `t_map_double` map, - `t_map_boolean` map, - `t_map_decimal_precision_2` map, - `t_map_decimal_precision_4` map, - `t_map_decimal_precision_8` map, - `t_map_decimal_precision_17` map, - `t_map_decimal_precision_18` map, - `t_map_decimal_precision_38` map, - `t_array_string` array, - `t_array_int` array, - `t_array_bigint` array, - `t_array_float` array, - `t_array_double` array, - `t_array_boolean` array, - `t_array_varchar` array, - `t_array_char` array, - `t_array_decimal_precision_2` array, - `t_array_decimal_precision_4` array, - `t_array_decimal_precision_8` array, - `t_array_decimal_precision_17` array, - `t_array_decimal_precision_18` array, - `t_array_decimal_precision_38` array, - `t_struct_bigint` struct, - `t_complex` map>>, - `t_struct_nested` struct>, - `t_struct_null` struct, - `t_struct_non_nulls_after_nulls` struct, - `t_nested_struct_non_nulls_after_nulls` struct>, - `t_map_null_value` map, - `t_array_string_starting_with_nulls` array, - `t_array_string_with_nulls_in_between` array, - `t_array_string_ending_with_nulls` array, - `t_array_string_all_nulls` array, - `dt` int) -stored as parquet -TBLPROPERTIES('parquet.compression'='SNAPPY'); - -CREATE TABLE `all_types_par_parquet_snappy`( - `boolean_col` boolean, - `tinyint_col` tinyint, - `smallint_col` smallint, - `int_col` int, - `bigint_col` bigint, - `float_col` float, - `double_col` double, - `decimal_col1` decimal(9,0), - `decimal_col2` decimal(8,4), - `decimal_col3` decimal(18,6), - `decimal_col4` decimal(38,12), - `string_col` string, - `binary_col` binary, - `date_col` date, - `timestamp_col1` timestamp, - `timestamp_col2` timestamp, - `timestamp_col3` timestamp, - `char_col1` char(50), - `char_col2` char(100), - `char_col3` char(255), - `varchar_col1` varchar(50), - `varchar_col2` varchar(100), - `varchar_col3` varchar(255), - `t_map_string` map, - `t_map_varchar` map, - `t_map_char` map, - `t_map_int` map, - `t_map_bigint` map, - `t_map_float` map, - `t_map_double` map, - `t_map_boolean` map, - `t_map_decimal_precision_2` map, - `t_map_decimal_precision_4` map, - `t_map_decimal_precision_8` map, - `t_map_decimal_precision_17` map, - `t_map_decimal_precision_18` map, - `t_map_decimal_precision_38` map, - `t_array_string` array, - `t_array_int` array, - `t_array_bigint` array, - `t_array_float` array, - `t_array_double` array, - `t_array_boolean` array, - `t_array_varchar` array, - `t_array_char` array, - `t_array_decimal_precision_2` array, - `t_array_decimal_precision_4` array, - `t_array_decimal_precision_8` array, - `t_array_decimal_precision_17` array, - `t_array_decimal_precision_18` array, - `t_array_decimal_precision_38` array, - `t_struct_bigint` struct, - `t_complex` map>>, - `t_struct_nested` struct>, - `t_struct_null` struct, - `t_struct_non_nulls_after_nulls` struct, - `t_nested_struct_non_nulls_after_nulls` struct>, - `t_map_null_value` map, - `t_array_string_starting_with_nulls` array, - `t_array_string_with_nulls_in_between` array, - `t_array_string_ending_with_nulls` array, - `t_array_string_all_nulls` array) -PARTITIONED BY ( - `dt` int) -stored as parquet -TBLPROPERTIES('parquet.compression'='SNAPPY'); - -CREATE TABLE `all_types_orc_zlib`( - `boolean_col` boolean, - `tinyint_col` tinyint, - `smallint_col` smallint, - `int_col` int, - `bigint_col` bigint, - `float_col` float, - `double_col` double, - `decimal_col1` decimal(9,0), - `decimal_col2` decimal(8,4), - `decimal_col3` decimal(18,6), - `decimal_col4` decimal(38,12), - `string_col` string, - `binary_col` binary, - `date_col` date, - `timestamp_col1` timestamp, - `timestamp_col2` timestamp, - `timestamp_col3` timestamp, - `char_col1` char(50), - `char_col2` char(100), - `char_col3` char(255), - `varchar_col1` varchar(50), - `varchar_col2` varchar(100), - `varchar_col3` varchar(255), - `t_map_string` map, - `t_map_varchar` map, - `t_map_char` map, - `t_map_int` map, - `t_map_bigint` map, - `t_map_float` map, - `t_map_double` map, - `t_map_boolean` map, - `t_map_decimal_precision_2` map, - `t_map_decimal_precision_4` map, - `t_map_decimal_precision_8` map, - `t_map_decimal_precision_17` map, - `t_map_decimal_precision_18` map, - `t_map_decimal_precision_38` map, - `t_array_string` array, - `t_array_int` array, - `t_array_bigint` array, - `t_array_float` array, - `t_array_double` array, - `t_array_boolean` array, - `t_array_varchar` array, - `t_array_char` array, - `t_array_decimal_precision_2` array, - `t_array_decimal_precision_4` array, - `t_array_decimal_precision_8` array, - `t_array_decimal_precision_17` array, - `t_array_decimal_precision_18` array, - `t_array_decimal_precision_38` array, - `t_struct_bigint` struct, - `t_complex` map>>, - `t_struct_nested` struct>, - `t_struct_null` struct, - `t_struct_non_nulls_after_nulls` struct, - `t_nested_struct_non_nulls_after_nulls` struct>, - `t_map_null_value` map, - `t_array_string_starting_with_nulls` array, - `t_array_string_with_nulls_in_between` array, - `t_array_string_ending_with_nulls` array, - `t_array_string_all_nulls` array, - `dt` int) -stored as orc -TBLPROPERTIES("orc.compress"="ZLIB"); - -CREATE TABLE `all_types_par_orc_zlib`( - `boolean_col` boolean, - `tinyint_col` tinyint, - `smallint_col` smallint, - `int_col` int, - `bigint_col` bigint, - `float_col` float, - `double_col` double, - `decimal_col1` decimal(9,0), - `decimal_col2` decimal(8,4), - `decimal_col3` decimal(18,6), - `decimal_col4` decimal(38,12), - `string_col` string, - `binary_col` binary, - `date_col` date, - `timestamp_col1` timestamp, - `timestamp_col2` timestamp, - `timestamp_col3` timestamp, - `char_col1` char(50), - `char_col2` char(100), - `char_col3` char(255), - `varchar_col1` varchar(50), - `varchar_col2` varchar(100), - `varchar_col3` varchar(255), - `t_map_string` map, - `t_map_varchar` map, - `t_map_char` map, - `t_map_int` map, - `t_map_bigint` map, - `t_map_float` map, - `t_map_double` map, - `t_map_boolean` map, - `t_map_decimal_precision_2` map, - `t_map_decimal_precision_4` map, - `t_map_decimal_precision_8` map, - `t_map_decimal_precision_17` map, - `t_map_decimal_precision_18` map, - `t_map_decimal_precision_38` map, - `t_array_string` array, - `t_array_int` array, - `t_array_bigint` array, - `t_array_float` array, - `t_array_double` array, - `t_array_boolean` array, - `t_array_varchar` array, - `t_array_char` array, - `t_array_decimal_precision_2` array, - `t_array_decimal_precision_4` array, - `t_array_decimal_precision_8` array, - `t_array_decimal_precision_17` array, - `t_array_decimal_precision_18` array, - `t_array_decimal_precision_38` array, - `t_struct_bigint` struct, - `t_complex` map>>, - `t_struct_nested` struct>, - `t_struct_null` struct, - `t_struct_non_nulls_after_nulls` struct, - `t_nested_struct_non_nulls_after_nulls` struct>, - `t_map_null_value` map, - `t_array_string_starting_with_nulls` array, - `t_array_string_with_nulls_in_between` array, - `t_array_string_ending_with_nulls` array, - `t_array_string_all_nulls` array) -PARTITIONED BY ( - `dt` int) -stored as orc -TBLPROPERTIES("orc.compress"="ZLIB"); - -CREATE TABLE `all_partition_types1_parquet_snappy_src`( - `id` int -) -PARTITIONED BY ( - `boolean_col` boolean, - `tinyint_col` tinyint, - `smallint_col` smallint, - `int_col` int, - `bigint_col` bigint, - `float_col` float, - `double_col` double) -stored as parquet -LOCATION - '/user/doris/preinstalled_data/parquet_table/all_partition_types1_parquet_snappy_src' -TBLPROPERTIES('parquet.compression'='SNAPPY'); -msck repair table all_partition_types1_parquet_snappy_src; - -CREATE TABLE `all_partition_types1_parquet_snappy`( - `id` int -) -PARTITIONED BY ( - `boolean_col` boolean, - `tinyint_col` tinyint, - `smallint_col` smallint, - `int_col` int, - `bigint_col` bigint, - `float_col` float, - `double_col` double) -stored as parquet -TBLPROPERTIES('parquet.compression'='SNAPPY'); - -CREATE TABLE `all_partition_types1_orc_zlib`( - `id` int -) -PARTITIONED BY ( - `boolean_col` boolean, - `tinyint_col` tinyint, - `smallint_col` smallint, - `int_col` int, - `bigint_col` bigint, - `float_col` float, - `double_col` double) -stored as orc -TBLPROPERTIES("orc.compress"="ZLIB"); - -CREATE TABLE `all_partition_types2_parquet_snappy_src`( - `id` int -) -PARTITIONED BY ( - `decimal_col` decimal(18,6), - `string_col` string, - `date_col` date, - `char_col` char(11), - `varchar_col` varchar(50)) -stored as parquet -LOCATION - '/user/doris/preinstalled_data/parquet_table/all_partition_types2_parquet_snappy_src' -TBLPROPERTIES('parquet.compression'='SNAPPY'); -msck repair table all_partition_types2_parquet_snappy_src; - -CREATE TABLE `all_partition_types2_parquet_snappy`( - `id` int -) -PARTITIONED BY ( - `decimal_col` decimal(18,6), - `string_col` string, - `date_col` date, - `char_col` char(11), - `varchar_col` varchar(50)) -stored as parquet -TBLPROPERTIES('parquet.compression'='SNAPPY'); - -CREATE TABLE `all_partition_types2_orc_zlib`( - `id` int -) -PARTITIONED BY ( - `decimal_col` decimal(18,6), - `string_col` string, - `date_col` date, - `char_col` char(11), - `varchar_col` varchar(50)) -stored as orc -TBLPROPERTIES("orc.compress"="ZLIB"); diff --git a/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/hive_text_complex_type/run.sh b/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/hive_text_complex_type/run.sh old mode 100644 new mode 100755 diff --git a/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/hive_text_complex_type2/run.sh b/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/hive_text_complex_type2/run.sh old mode 100644 new mode 100755 diff --git a/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/hive_text_complex_type3/run.sh b/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/hive_text_complex_type3/run.sh old mode 100644 new mode 100755 diff --git a/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/hive_text_complex_type_delimiter/run.sh b/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/hive_text_complex_type_delimiter/run.sh old mode 100644 new mode 100755 diff --git a/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/hive_text_complex_type_delimiter2/run.sh b/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/hive_text_complex_type_delimiter2/run.sh old mode 100644 new mode 100755 diff --git a/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/hive_text_complex_type_delimiter3/run.sh b/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/hive_text_complex_type_delimiter3/run.sh old mode 100644 new mode 100755 diff --git a/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/logs1_parquet/run.sh b/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/logs1_parquet/run.sh old mode 100644 new mode 100755 diff --git a/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/orc_partitioned_columns/run.sh b/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/orc_partitioned_columns/run.sh old mode 100644 new mode 100755 diff --git a/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/orc_partitioned_one_column/run.sh b/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/orc_partitioned_one_column/run.sh old mode 100644 new mode 100755 diff --git a/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/parquet_alter_column_to_bigint/run.sh b/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/parquet_alter_column_to_bigint/run.sh old mode 100644 new mode 100755 diff --git a/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/parquet_alter_column_to_boolean/run.sh b/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/parquet_alter_column_to_boolean/run.sh old mode 100644 new mode 100755 diff --git a/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/parquet_alter_column_to_char/run.sh b/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/parquet_alter_column_to_char/run.sh old mode 100644 new mode 100755 diff --git a/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/parquet_alter_column_to_date/run.sh b/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/parquet_alter_column_to_date/run.sh old mode 100644 new mode 100755 diff --git a/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/parquet_alter_column_to_decimal/run.sh b/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/parquet_alter_column_to_decimal/run.sh old mode 100644 new mode 100755 diff --git a/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/parquet_alter_column_to_double/run.sh b/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/parquet_alter_column_to_double/run.sh old mode 100644 new mode 100755 diff --git a/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/parquet_alter_column_to_float/run.sh b/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/parquet_alter_column_to_float/run.sh old mode 100644 new mode 100755 diff --git a/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/parquet_alter_column_to_int/run.sh b/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/parquet_alter_column_to_int/run.sh old mode 100644 new mode 100755 diff --git a/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/parquet_alter_column_to_smallint/run.sh b/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/parquet_alter_column_to_smallint/run.sh old mode 100644 new mode 100755 diff --git a/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/parquet_alter_column_to_string/run.sh b/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/parquet_alter_column_to_string/run.sh old mode 100644 new mode 100755 diff --git a/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/parquet_alter_column_to_timestamp/run.sh b/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/parquet_alter_column_to_timestamp/run.sh old mode 100644 new mode 100755 diff --git a/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/parquet_alter_column_to_tinyint/run.sh b/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/parquet_alter_column_to_tinyint/run.sh old mode 100644 new mode 100755 diff --git a/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/parquet_alter_column_to_varchar/run.sh b/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/parquet_alter_column_to_varchar/run.sh old mode 100644 new mode 100755 diff --git a/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/parquet_partitioned_columns/run.sh b/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/parquet_partitioned_columns/run.sh old mode 100644 new mode 100755 diff --git a/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/parquet_partitioned_one_column/run.sh b/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/parquet_partitioned_one_column/run.sh old mode 100644 new mode 100755 diff --git a/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/parquet_predicate_table/run.sh b/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/parquet_predicate_table/run.sh old mode 100644 new mode 100755 diff --git a/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/test_chinese_orc/run.sh b/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/test_chinese_orc/run.sh old mode 100644 new mode 100755 diff --git a/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/test_chinese_parquet/run.sh b/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/test_chinese_parquet/run.sh old mode 100644 new mode 100755 diff --git a/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/test_chinese_text/run.sh b/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/test_chinese_text/run.sh old mode 100644 new mode 100755 diff --git a/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/test_hive_same_db_table_name/run.sh b/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/test_hive_same_db_table_name/run.sh old mode 100644 new mode 100755 diff --git a/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/test_hive_special_char_partition/run.sh b/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/test_hive_special_char_partition/run.sh old mode 100644 new mode 100755 diff --git a/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/test_mixed_par_locations_orc/run.sh b/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/test_mixed_par_locations_orc/run.sh old mode 100644 new mode 100755 diff --git a/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/test_mixed_par_locations_parquet/run.sh b/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/test_mixed_par_locations_parquet/run.sh old mode 100644 new mode 100755 diff --git a/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/test_multi_langs_orc/run.sh b/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/test_multi_langs_orc/run.sh old mode 100644 new mode 100755 diff --git a/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/test_multi_langs_parquet/run.sh b/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/test_multi_langs_parquet/run.sh old mode 100644 new mode 100755 diff --git a/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/test_multi_langs_text/run.sh b/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/test_multi_langs_text/run.sh old mode 100644 new mode 100755 diff --git a/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/test_truncate_char_or_varchar_columns_orc/run.sh b/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/test_truncate_char_or_varchar_columns_orc/run.sh old mode 100644 new mode 100755 diff --git a/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/test_truncate_char_or_varchar_columns_parquet/run.sh b/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/test_truncate_char_or_varchar_columns_parquet/run.sh old mode 100644 new mode 100755 diff --git a/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/test_truncate_char_or_varchar_columns_text/run.sh b/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/test_truncate_char_or_varchar_columns_text/run.sh old mode 100644 new mode 100755 diff --git a/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/text_partitioned_columns/run.sh b/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/text_partitioned_columns/run.sh old mode 100644 new mode 100755 diff --git a/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/text_partitioned_one_column/run.sh b/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/text_partitioned_one_column/run.sh old mode 100644 new mode 100755 diff --git a/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/type_change_orc/run.sh b/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/type_change_orc/run.sh old mode 100644 new mode 100755 diff --git a/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/type_change_origin/run.sh b/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/type_change_origin/run.sh old mode 100644 new mode 100755 diff --git a/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/type_change_parquet/run.sh b/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/type_change_parquet/run.sh old mode 100644 new mode 100755 diff --git a/docker/thirdparties/docker-compose/hive/scripts/data/partition_type/bigint_partition/run.sh b/docker/thirdparties/docker-compose/hive/scripts/data/partition_type/bigint_partition/run.sh old mode 100644 new mode 100755 diff --git a/docker/thirdparties/docker-compose/hive/scripts/data/partition_type/char_partition/run.sh b/docker/thirdparties/docker-compose/hive/scripts/data/partition_type/char_partition/run.sh old mode 100644 new mode 100755 diff --git a/docker/thirdparties/docker-compose/hive/scripts/data/partition_type/date_partition/run.sh b/docker/thirdparties/docker-compose/hive/scripts/data/partition_type/date_partition/run.sh old mode 100644 new mode 100755 diff --git a/docker/thirdparties/docker-compose/hive/scripts/data/partition_type/decimal_partition/run.sh b/docker/thirdparties/docker-compose/hive/scripts/data/partition_type/decimal_partition/run.sh old mode 100644 new mode 100755 diff --git a/docker/thirdparties/docker-compose/hive/scripts/data/partition_type/double_partition/run.sh b/docker/thirdparties/docker-compose/hive/scripts/data/partition_type/double_partition/run.sh old mode 100644 new mode 100755 diff --git a/docker/thirdparties/docker-compose/hive/scripts/data/partition_type/float_partition/run.sh b/docker/thirdparties/docker-compose/hive/scripts/data/partition_type/float_partition/run.sh old mode 100644 new mode 100755 diff --git a/docker/thirdparties/docker-compose/hive/scripts/data/partition_type/int_partition/run.sh b/docker/thirdparties/docker-compose/hive/scripts/data/partition_type/int_partition/run.sh old mode 100644 new mode 100755 diff --git a/docker/thirdparties/docker-compose/hive/scripts/data/partition_type/smallint_partition/run.sh b/docker/thirdparties/docker-compose/hive/scripts/data/partition_type/smallint_partition/run.sh old mode 100644 new mode 100755 diff --git a/docker/thirdparties/docker-compose/hive/scripts/data/partition_type/string_partition/run.sh b/docker/thirdparties/docker-compose/hive/scripts/data/partition_type/string_partition/run.sh old mode 100644 new mode 100755 diff --git a/docker/thirdparties/docker-compose/hive/scripts/data/partition_type/tinyint_partition/run.sh b/docker/thirdparties/docker-compose/hive/scripts/data/partition_type/tinyint_partition/run.sh old mode 100644 new mode 100755 diff --git a/docker/thirdparties/docker-compose/hive/scripts/data/partition_type/varchar_partition/run.sh b/docker/thirdparties/docker-compose/hive/scripts/data/partition_type/varchar_partition/run.sh old mode 100644 new mode 100755 diff --git a/docker/thirdparties/docker-compose/hive/scripts/data/statistics/empty_table/run.sh b/docker/thirdparties/docker-compose/hive/scripts/data/statistics/empty_table/run.sh old mode 100644 new mode 100755 diff --git a/docker/thirdparties/docker-compose/hive/scripts/data/statistics/stats/run.sh b/docker/thirdparties/docker-compose/hive/scripts/data/statistics/stats/run.sh old mode 100644 new mode 100755 diff --git a/docker/thirdparties/docker-compose/hive/scripts/data/tpch_1000_parquet/part/run.sh b/docker/thirdparties/docker-compose/hive/scripts/data/tpch_1000_parquet/part/run.sh old mode 100644 new mode 100755 diff --git a/docker/thirdparties/docker-compose/hive/scripts/data/tvf/test_hdfs_tvf_compression/run.sh b/docker/thirdparties/docker-compose/hive/scripts/data/tvf/test_hdfs_tvf_compression/run.sh old mode 100644 new mode 100755 diff --git a/docker/thirdparties/docker-compose/hive/scripts/data/tvf/test_tvf/run.sh b/docker/thirdparties/docker-compose/hive/scripts/data/tvf/test_tvf/run.sh old mode 100644 new mode 100755 diff --git a/docker/thirdparties/docker-compose/hive/scripts/hive-metastore.sh b/docker/thirdparties/docker-compose/hive/scripts/hive-metastore.sh index abf4ef869583af..8338f467952831 100755 --- a/docker/thirdparties/docker-compose/hive/scripts/hive-metastore.sh +++ b/docker/thirdparties/docker-compose/hive/scripts/hive-metastore.sh @@ -28,13 +28,19 @@ sleep 10s hadoop fs -mkdir -p /user/doris/suites/ DATA_DIR="/mnt/scripts/data/" -find "${DATA_DIR}" -type f -name "run.sh" -exec chmod +x {} \; -exec {} \; +find "${DATA_DIR}" -type f -name "run.sh" -print0 | xargs -0 -n 1 -P 10 -I {} sh -c ' + START_TIME=$(date +%s) + chmod +x "{}" && "{}" + END_TIME=$(date +%s) + EXECUTION_TIME=$((END_TIME - START_TIME)) + echo "Script: {} executed in $EXECUTION_TIME seconds" +' # if you test in your local，better use # to annotation section about tpch1.db if [[ ! -d "/mnt/scripts/tpch1.db" ]]; then echo "/mnt/scripts/tpch1.db does not exist" cd /mnt/scripts/ - curl -O https://s3BucketName.s3Endpoint/regression/datalake/pipeline_data/tpch1.db.tar.gz + curl -O https://doris-build-hk-1308700295.cos.ap-hongkong.myqcloud.com/regression/datalake/pipeline_data/tpch1.db.tar.gz tar -zxf tpch1.db.tar.gz rm -rf tpch1.db.tar.gz cd - @@ -47,12 +53,11 @@ fi hadoop fs -mkdir -p /user/doris/ hadoop fs -put /mnt/scripts/tpch1.db /user/doris/ - # if you test in your local，better use # to annotation section about paimon if [[ ! -d "/mnt/scripts/paimon1" ]]; then echo "/mnt/scripts/paimon1 does not exist" cd /mnt/scripts/ - curl -O https://s3BucketName.s3Endpoint/regression/datalake/pipeline_data/paimon1.tar.gz + curl -O https://doris-build-hk-1308700295.cos.ap-hongkong.myqcloud.com/regression/datalake/pipeline_data/paimon1.tar.gz tar -zxf paimon1.tar.gz rm -rf paimon1.tar.gz cd - @@ -66,10 +71,14 @@ hadoop fs -put /mnt/scripts/paimon1 /user/doris/ ## put other preinstalled data hadoop fs -put /mnt/scripts/preinstalled_data /user/doris/ -# create table -hive -f /mnt/scripts/create_tpch1_orc.hql -hive -f /mnt/scripts/create_tpch1_parquet.hql -hive -f /mnt/scripts/create_preinstalled_table.hql +# create tables +ls /mnt/scripts/create_preinstalled_scripts/*.hql | xargs -n 1 -P 10 -I {} bash -c ' + START_TIME=$(date +%s) + hive -f {} + END_TIME=$(date +%s) + EXECUTION_TIME=$((END_TIME - START_TIME)) + echo "Script: {} executed in $EXECUTION_TIME seconds" +' touch /mnt/SUCCESS