From ae01646136e630c9046e3aa0c538a44f551f0507 Mon Sep 17 00:00:00 2001 From: avogar Date: Thu, 14 Apr 2022 19:05:20 +0000 Subject: [PATCH 1/2] Place new logic of schema inference in insert select from table function under setting --- src/Core/Settings.h | 2 ++ src/Interpreters/Context.cpp | 2 +- .../02249_insert_select_from_input_schema_inference.sql | 2 ++ .../02250_insert_select_from_file_schema_inference.sql | 2 ++ 4 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index aa78456702ce..8461c1cb3735 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -566,6 +566,8 @@ class IColumn; M(Bool, enable_filesystem_cache_on_write_operations, false, "Write into cache on write operations. To actually work this setting requires be added to disk config too", 0) \ M(Bool, read_from_filesystem_cache_if_exists_otherwise_bypass_cache, false, "", 0) \ \ + M(Bool, use_structure_from_insertion_table_in_table_functions, false, "Use structure from insertion table instead of schema inference from data", 0) \ + \ M(UInt64, http_max_tries, 10, "Max attempts to read via http.", 0) \ M(UInt64, http_retry_initial_backoff_ms, 100, "Min milliseconds for backoff, when retrying read via http", 0) \ M(UInt64, http_retry_max_backoff_ms, 10000, "Max milliseconds for backoff, when retrying read via http", 0) \ diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 40ed4d589933..effcfd507300 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -1101,7 +1101,7 @@ StoragePtr Context::executeTableFunction(const ASTPtr & table_expression) if (!res) { TableFunctionPtr table_function_ptr = TableFunctionFactory::instance().get(table_expression, shared_from_this()); - if (table_function_ptr->needStructureHint()) + if (getSettingsRef().use_structure_from_insertion_table_in_table_functions && table_function_ptr->needStructureHint()) { const auto & insertion_table = getInsertionTable(); if (!insertion_table.empty()) diff --git a/tests/queries/0_stateless/02249_insert_select_from_input_schema_inference.sql b/tests/queries/0_stateless/02249_insert_select_from_input_schema_inference.sql index da3bccfd7453..eb83309f1171 100644 --- a/tests/queries/0_stateless/02249_insert_select_from_input_schema_inference.sql +++ b/tests/queries/0_stateless/02249_insert_select_from_input_schema_inference.sql @@ -1,3 +1,5 @@ +set use_structure_from_insertion_table_in_table_functions = 1; + drop table if exists test_02249; create table test_02249 (x UInt32, y String) engine=Memory(); insert into test_02249 select * from input() format JSONEachRow {"x" : 1, "y" : "string1"}, {"y" : "string2", "x" : 2}; diff --git a/tests/queries/0_stateless/02250_insert_select_from_file_schema_inference.sql b/tests/queries/0_stateless/02250_insert_select_from_file_schema_inference.sql index 2c04045463e2..55dfaf877175 100644 --- a/tests/queries/0_stateless/02250_insert_select_from_file_schema_inference.sql +++ b/tests/queries/0_stateless/02250_insert_select_from_file_schema_inference.sql @@ -1,3 +1,5 @@ +set use_structure_from_insertion_table_in_table_functions = 1; + insert into table function file('data_02250.jsonl') select NULL as x settings engine_file_truncate_on_insert=1; drop table if exists test_02250; create table test_02250 (x Nullable(UInt32)) engine=Memory(); From 996be9fe54efc19c1d6d2e58cdd79da2ff2a938c Mon Sep 17 00:00:00 2001 From: avogar Date: Tue, 19 Apr 2022 12:16:45 +0000 Subject: [PATCH 2/2] Fix tests --- .../0_stateless/01825_type_json_schema_inference.reference | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01825_type_json_schema_inference.reference b/tests/queries/0_stateless/01825_type_json_schema_inference.reference index 6f1a65c6af3e..c2c18b5a2ff5 100644 --- a/tests/queries/0_stateless/01825_type_json_schema_inference.reference +++ b/tests/queries/0_stateless/01825_type_json_schema_inference.reference @@ -5,4 +5,4 @@ Tuple(k1 Int8, k2 Tuple(k3 String, k4 Nested(k5 Int8, k6 Int8)), some Int8) {"id":"2","obj":"bbb","s":"bar"} {"map":{"k1":1,"k2":2},"obj":{"k1":1,"k2.k3":2},"map_type":"Map(String, Nullable(Float64))","obj_type":"Object('json')"} {"obj":{"k1":1,"k2":2},"map":{"k1":"1","k2":"2"}} -Tuple(k1 Int8, k2 Int8) +Tuple(k1 Float64, k2 Float64)