Retain result type after UNION as base type in T-SQL (#3175) (#3268)

Description
-----------
When a base type such as `money` is UNIONed with another integer type,
the result type will be `fixeddecimal` instead of `money`, which is
inconsistent with SQL Server's behaviour. And could also cause some
characters not displaying well.

Analysis
--------
The root cause of the issue is Postgres will consider the base type of
domain type as the result type. In Babelfish, `money` is implemented
as a domain type of `fixeddecimal`. Actually, `money` is a base type
in T-SQL, thus Babelfish should retain `money` type after UNION.

Fix
---
After a final type has determined by the optimizer, check if it's a
base type in T-SQL. If it is, try to use the domain type as result
type if the domain type has higher precision than other types.

Task: BABEL-5242

Signed-off-by: Bo Li <[email protected]>

contrib/babelfishpg_tsql/src/pltsql_coerce.c

@@ -54,6 +54,7 @@
 /* Hooks for engine*/
 extern find_coercion_pathway_hook_type find_coercion_pathway_hook;
 extern determine_datatype_precedence_hook_type determine_datatype_precedence_hook;
+extern is_tsql_base_datatype_hook_type is_tsql_base_datatype_hook;
 extern func_select_candidate_hook_type func_select_candidate_hook;
 extern coerce_string_literal_hook_type coerce_string_literal_hook;
 extern select_common_type_hook_type select_common_type_hook;
@@ -846,6 +847,15 @@ tsql_has_higher_precedence(Oid typeId1, Oid typeId2)
 	return type1_precedence < type2_precedence;
 }
 
+/*
+ * Returns if given data type is a base type in T-SQL.
+ */
+static bool
+is_tsql_base_datatype(Oid typeId)
+{
+	return tsql_get_type_precedence(typeId) != -1;
+}
+
 static bool
 is_vectorized_binary_operator(FuncCandidateList candidate)
 {
@@ -2135,6 +2145,7 @@ init_tsql_datatype_precedence_hash_tab(PG_FUNCTION_ARGS)
 
 	/* Register Hooks */
 	determine_datatype_precedence_hook = tsql_has_higher_precedence;
+	is_tsql_base_datatype_hook = is_tsql_base_datatype;
 	func_select_candidate_hook = tsql_func_select_candidate;
 	coerce_string_literal_hook = tsql_coerce_string_literal_hook;
 	select_common_type_hook = tsql_select_common_type_hook;

test/JDBC/expected/BABEL-3392-vu-verify.out

@@ -1637,7 +1637,7 @@ SELECT name, max_length FROM sys.columns WHERE name = 'babel4157_c1'
 GO
 ~~START~~
 varchar#!#smallint
-babel4157_c1#!#<NULL>
+babel4157_c1#!#3
 ~~END~~
 
 
@@ -1890,14 +1890,14 @@ select cast(17 as varbinary(1)) UNION select cast(10 as varbinary(2)) union sele
 GO
 ~~ERROR (Code: 33557097)~~
 
-~~ERROR (Message: UNION could not convert type "nchar" to bbf_varbinary)~~
+~~ERROR (Message: UNION could not convert type "nchar" to varbinary)~~
 
 
 SELECT CAST(N'ΘЖऌฒ' AS NCHAR(15)) UNION select cast(10 as varbinary(2));
 GO
 ~~ERROR (Code: 33557097)~~
 
-~~ERROR (Message: UNION could not convert type varbinary to bpchar)~~
+~~ERROR (Message: UNION could not convert type varbinary to "nchar")~~
 
 
 SELECT CAST(1 as BIT) UNION SELECT cast(N'ab' as NCHAR(10)) UNION SELECT CAST('bar' as CHAR(10));