From 982793f67b05a2fae0f933594d92d2cb3e127a14 Mon Sep 17 00:00:00 2001 From: Tanzeel Khan Date: Thu, 26 Oct 2023 04:43:34 +0000 Subject: [PATCH 01/28] initial commit Signed-off-by: Tanzeel Khan --- contrib/babelfishpg_common/src/varbinary.c | 43 ++++++++++++++------ contrib/babelfishpg_tsql/src/pltsql_coerce.c | 32 ++++++++++++++- 2 files changed, 62 insertions(+), 13 deletions(-) diff --git a/contrib/babelfishpg_common/src/varbinary.c b/contrib/babelfishpg_common/src/varbinary.c index 8e6e4aee16..e3b1a33ba0 100644 --- a/contrib/babelfishpg_common/src/varbinary.c +++ b/contrib/babelfishpg_common/src/varbinary.c @@ -16,7 +16,9 @@ #include "access/hash.h" #include "catalog/pg_collation.h" #include "catalog/pg_type.h" +#include "collation.h" #include "common/int.h" +#include "encoding/encoding.h" #include "lib/hyperloglog.h" #include "libpq/pqformat.h" #include "miscadmin.h" @@ -621,13 +623,16 @@ Datum varcharvarbinary(PG_FUNCTION_ARGS) { VarChar *source = PG_GETARG_VARCHAR_PP(0); - char *data = VARDATA_ANY(source); + char *data = VARDATA_ANY(source); /* Source string is UTF-8 */ + char *encoded_data; char *rp; size_t len = VARSIZE_ANY_EXHDR(source); int32 typmod = PG_GETARG_INT32(1); bool isExplicit = PG_GETARG_BOOL(2); int32 maxlen; bytea *result; + coll_info collInfo; + int encodedByteLen; if (!isExplicit) ereport(ERROR, @@ -636,20 +641,26 @@ varcharvarbinary(PG_FUNCTION_ARGS) "varbinary is not allowed. Use the CONVERT function " "to run this query."))); - /* If typmod is -1 (or invalid), use the actual length */ + collInfo = lookup_collation_table(get_server_collation_oid_internal(false)); + encoded_data = encoding_conv_util(data, len, PG_UTF8, collInfo.enc, &encodedByteLen); + + /* + * If typmod is -1 (or invalid), use the actual length + * Length should be checked after encoding into server enc + */ if (typmod < (int32) VARHDRSZ) - maxlen = len; + maxlen = encodedByteLen; else maxlen = typmod - VARHDRSZ; - if (len > maxlen) - len = maxlen; + if (encodedByteLen > maxlen) + encodedByteLen = maxlen; - result = (bytea *) palloc(len + VARHDRSZ); - SET_VARSIZE(result, len + VARHDRSZ); + result = (bytea *) palloc(encodedByteLen + VARHDRSZ); + SET_VARSIZE(result, encodedByteLen + VARHDRSZ); rp = VARDATA(result); - memcpy(rp, data, len); + memcpy(rp, encoded_data, encodedByteLen); PG_RETURN_BYTEA_P(result); } @@ -697,21 +708,29 @@ Datum varbinaryvarchar(PG_FUNCTION_ARGS) { bytea *source = PG_GETARG_BYTEA_PP(0); - char *data = VARDATA_ANY(source); + char *data = VARDATA_ANY(source); /* Source data is server encoded */ + VarChar *result; + char *encoded_result; size_t len = VARSIZE_ANY_EXHDR(source); int32 typmod = PG_GETARG_INT32(1); int32 maxlen = typmod - VARHDRSZ; - VarChar *result; + coll_info collInfo; + int encodedByteLen; + + collInfo = lookup_collation_table(get_server_collation_oid_internal(false)); /* * Cast the entire input binary data if maxlen is invalid or supplied data * fits it */ if (maxlen < 0 || len <= maxlen) - result = (VarChar *) cstring_to_text_with_len(data, len); + encoded_result = encoding_conv_util(data, len, collInfo.enc, PG_UTF8, &encodedByteLen); /* Else truncate it */ else - result = (VarChar *) cstring_to_text_with_len(data, maxlen); + encoded_result = encoding_conv_util(data, maxlen, collInfo.enc, PG_UTF8, &encodedByteLen); + + result = (VarChar *) cstring_to_text_with_len(encoded_result, encodedByteLen); + PG_RETURN_VARCHAR_P(result); } diff --git a/contrib/babelfishpg_tsql/src/pltsql_coerce.c b/contrib/babelfishpg_tsql/src/pltsql_coerce.c index 2232a95ec9..6186509413 100644 --- a/contrib/babelfishpg_tsql/src/pltsql_coerce.c +++ b/contrib/babelfishpg_tsql/src/pltsql_coerce.c @@ -16,6 +16,7 @@ #include "catalog/pg_type.h" #include "catalog/pg_proc.h" #include "catalog/pg_namespace.h" +#include "collation.h" #include "executor/spi.h" #include "mb/pg_wchar.h" #include "nodes/makefuncs.h" @@ -32,6 +33,7 @@ #include "utils/lsyscache.h" #include "utils/syscache.h" #include "pltsql_instr.h" +#include "parser/parse_target.h" #include @@ -972,7 +974,7 @@ tsql_coerce_string_literal_hook(ParseCallbackState *pcbstate, Oid targetTypeId, if (ccontext != COERCION_EXPLICIT) { /* - * T-SQL may forbid casting from string literal to certain + * T-SQL forbids implicit casting from string literal to certain * datatypes (i.e. binary, varbinary) */ if ((*common_utility_plugin_ptr->is_tsql_binary_datatype) (baseTypeId)) @@ -1100,6 +1102,34 @@ tsql_coerce_string_literal_hook(ParseCallbackState *pcbstate, Oid targetTypeId, newcon->constvalue = stringTypeDatum(baseType, value, inputTypeMod); } } + else if ((*common_utility_plugin_ptr->is_tsql_binary_datatype) (baseTypeId) || + (*common_utility_plugin_ptr->is_tsql_varbinary_datatype) (baseTypeId)) + { + /* + * binary datatype should be passed in client encoding + * when explicit cast is called + */ + + TypeName *varcharTypeName = makeTypeName("varchar"); + Node *result; + Const *tempcon; + + typenameTypeIdAndMod(NULL, (const TypeName *)varcharTypeName, &baseTypeId, &baseTypeMod); + baseType = typeidType(baseTypeId); + pfree(varcharTypeName); + + tempcon = makeConst(TEXTOID, -1, + tsql_get_server_collation_oid_internal(false), + -1, PointerGetDatum(cstring_to_text(value)), + false, false); + + result = coerce_to_target_type(NULL, (Node *) tempcon, baseTypeId, + targetTypeId, targetTypeMod, + COERCION_EXPLICIT, + COERCE_EXPLICIT_CAST, + location); + return result; + } else { newcon->constvalue = stringTypeDatum(baseType, value, inputTypeMod); From 745aaebb4e5b837e32fc60d0ef45c066ae1e626d Mon Sep 17 00:00:00 2001 From: Tanzeel Khan Date: Thu, 26 Oct 2023 04:48:32 +0000 Subject: [PATCH 02/28] fix indent Signed-off-by: Tanzeel Khan --- contrib/babelfishpg_common/src/varbinary.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/contrib/babelfishpg_common/src/varbinary.c b/contrib/babelfishpg_common/src/varbinary.c index e3b1a33ba0..d4c8972061 100644 --- a/contrib/babelfishpg_common/src/varbinary.c +++ b/contrib/babelfishpg_common/src/varbinary.c @@ -644,10 +644,10 @@ varcharvarbinary(PG_FUNCTION_ARGS) collInfo = lookup_collation_table(get_server_collation_oid_internal(false)); encoded_data = encoding_conv_util(data, len, PG_UTF8, collInfo.enc, &encodedByteLen); - /* - * If typmod is -1 (or invalid), use the actual length - * Length should be checked after encoding into server enc - */ + /* + * If typmod is -1 (or invalid), use the actual length + * Length should be checked after encoding into server enc + */ if (typmod < (int32) VARHDRSZ) maxlen = encodedByteLen; else From d9391a2512cb127bb6878bf0426b4a2cab783cc5 Mon Sep 17 00:00:00 2001 From: Tanzeel Khan Date: Thu, 26 Oct 2023 09:15:21 +0000 Subject: [PATCH 03/28] allow 0x00 when encoding to UTF-8 Signed-off-by: Tanzeel Khan --- contrib/babelfishpg_common/src/encoding/mb/conv.c | 4 ---- test/JDBC/expected/babel_function_string-vu-verify.out | 2 +- test/JDBC/expected/babel_function_string.out | 2 +- 3 files changed, 2 insertions(+), 6 deletions(-) diff --git a/contrib/babelfishpg_common/src/encoding/mb/conv.c b/contrib/babelfishpg_common/src/encoding/mb/conv.c index 087a408b0f..7a4561fcc2 100644 --- a/contrib/babelfishpg_common/src/encoding/mb/conv.c +++ b/contrib/babelfishpg_common/src/encoding/mb/conv.c @@ -401,10 +401,6 @@ TsqlLocalToUtf(const unsigned char *iso, int len, unsigned char b3 = 0; unsigned char b4 = 0; - /* "break" cases all represent errors */ - if (*iso == '\0') - break; - if (!IS_HIGHBIT_SET(*iso)) { /* ASCII case is easy, assume it's one-to-one conversion */ diff --git a/test/JDBC/expected/babel_function_string-vu-verify.out b/test/JDBC/expected/babel_function_string-vu-verify.out index 4e0207f9ff..b63ef75e2d 100644 --- a/test/JDBC/expected/babel_function_string-vu-verify.out +++ b/test/JDBC/expected/babel_function_string-vu-verify.out @@ -151,7 +151,7 @@ SELECT datalength(a), datalength(b),datalength(c),datalength(d),datalength(e), GO ~~START~~ int#!#int#!#int#!#int#!#int#!#int#!#int#!#int#!#int -3#!#3#!#3#!#10#!#3#!#3#!#10#!#3#!#3 +10#!#3#!#3#!#10#!#3#!#3#!#10#!#3#!#3 #!##!##!##!##!##!##!##!# ~~END~~ diff --git a/test/JDBC/expected/babel_function_string.out b/test/JDBC/expected/babel_function_string.out index 13bd5d712b..82cd9cddf3 100644 --- a/test/JDBC/expected/babel_function_string.out +++ b/test/JDBC/expected/babel_function_string.out @@ -165,7 +165,7 @@ SELECT datalength(a), datalength(b),datalength(c),datalength(d),datalength(e), GO ~~START~~ int#!#int#!#int#!#int#!#int#!#int#!#int#!#int#!#int -3#!#3#!#3#!#10#!#3#!#3#!#10#!#3#!#3 +10#!#3#!#3#!#10#!#3#!#3#!#10#!#3#!#3 #!##!##!##!##!##!##!##!# ~~END~~ From cd3bbfc8af12087ddec9ec1b22625d468955e7f3 Mon Sep 17 00:00:00 2001 From: Tanzeel Khan Date: Thu, 26 Oct 2023 14:00:08 +0000 Subject: [PATCH 04/28] add tests Signed-off-by: Tanzeel Khan --- test/JDBC/expected/BABEL_1940.out | 217 +++++ ..._string-before-15-5-or-14-10-vu-verify.out | 789 ++++++++++++++++++ test/JDBC/input/BABEL_1940.sql | 76 ++ 3 files changed, 1082 insertions(+) create mode 100644 test/JDBC/expected/BABEL_1940.out create mode 100644 test/JDBC/expected/babel_function_string-before-15-5-or-14-10-vu-verify.out create mode 100644 test/JDBC/input/BABEL_1940.sql diff --git a/test/JDBC/expected/BABEL_1940.out b/test/JDBC/expected/BABEL_1940.out new file mode 100644 index 0000000000..6e3b515b88 --- /dev/null +++ b/test/JDBC/expected/BABEL_1940.out @@ -0,0 +1,217 @@ +SELECT CONVERT(VARCHAR(10), 0x123456789) +GO +~~START~~ +varchar +#Eg‰ +~~END~~ + + +SELECT CONVERT(VARCHAR(10), 0x80) +GO +~~START~~ +varchar +€ +~~END~~ + + +SELECT CONVERT(VARCHAR(10), 0xaaa) +GO +~~START~~ +varchar +ª +~~END~~ + + +SELECT CONVERT(VARCHAR(10), 0x330033) +GO +~~START~~ +varchar +3 +~~END~~ + + +SELECT CONVERT(VARBINARY(10), 'ｳ') +GO +~~START~~ +varbinary +3F +~~END~~ + + +SELECT CONVERT(VARBINARY(10), 'ﾊﾟ') +GO +~~START~~ +varbinary +3F3F +~~END~~ + + +SELECT CONVERT(VARBINARY(10), 'A') +GO +~~START~~ +varbinary +41 +~~END~~ + + +SELECT CONVERT(VARBINARY(10), 'ア') +GO +~~START~~ +varbinary +3F +~~END~~ + + +SELECT CONVERT(VARBINARY(10), 0x81) +GO +~~START~~ +varbinary +81 +~~END~~ + + +SELECT CONVERT(VARBINARY(10), 0x330033) +GO +~~START~~ +varbinary +330033 +~~END~~ + + +DECLARE @key varchar(20) = 'part1' +DECLARE @email varchar(20) = 'part2' +SELECT CONVERT(VARCHAR(10), HASHBYTES('SHA1', @key + LOWER(@email))) +GO +~~START~~ +varchar +æ/fact¢+Ó +~~END~~ + + + +CREATE TABLE babel_1940_t1 (a VARBINARY(9)) +GO + +INSERT INTO babel_1940_t1 VALUES(0x80) +INSERT INTO babel_1940_t1 VALUES(0xaaa) +INSERT INTO babel_1940_t1 VALUES(0x123456789) +GO +~~ROW COUNT: 1~~ + +~~ROW COUNT: 1~~ + +~~ROW COUNT: 1~~ + + +SELECT * FROM babel_1940_t1 +GO +~~START~~ +varbinary +80 +0AAA +0123456789 +~~END~~ + + +SELECT CONVERT(VARCHAR(9), a) FROM babel_1940_t1 +GO +~~START~~ +varchar +€ +ª +#Eg‰ +~~END~~ + + +SELECT CAST(a as VARCHAR(9)) FROM babel_1940_t1 +GO +~~START~~ +varchar +€ +ª +#Eg‰ +~~END~~ + + +SELECT CAST(a as VARCHAR(10)) FROM babel_1940_t1 +GO +~~START~~ +varchar +€ +ª +#Eg‰ +~~END~~ + + + +CREATE TABLE babel_1940_t2(a varchar(10) collate japanese_cs_as); +GO + +insert into babel_1940_t2 values ('ｳ'), ('C'), ('ﾊﾟ'), ('３'), ('c'), ('ｲ'), ('Ｃ'),('ﾊ'),('1'), +('ｱ'),('パ'), ('b'), ('2'), ('B'),('１'), ('Ａ'),('ア'),('A'), ('a'),('AbC'), ('aBc'); +GO +~~ROW COUNT: 21~~ + + +SELECT CONVERT(VARBINARY(10), a) FROM babel_1940_t2 +GO +~~START~~ +varbinary +3F +43 +3F3F +3F +63 +3F +3F +3F +31 +3F +3F +62 +32 +42 +3F +3F +3F +41 +61 +416243 +614263 +~~END~~ + + +SELECT CONVERT(VARBINARY(10), CONVERT(VARCHAR(10), CONVERT(VARCHAR(10), a))) FROM babel_1940_t2 +GO +~~START~~ +varbinary +3F +43 +3F3F +3F +63 +3F +3F +3F +31 +3F +3F +62 +32 +42 +3F +3F +3F +41 +61 +416243 +614263 +~~END~~ + + + +DROP TABLE babel_1940_t2 +GO + +DROP TABLE babel_1940_t1 +GO diff --git a/test/JDBC/expected/babel_function_string-before-15-5-or-14-10-vu-verify.out b/test/JDBC/expected/babel_function_string-before-15-5-or-14-10-vu-verify.out new file mode 100644 index 0000000000..4e0207f9ff --- /dev/null +++ b/test/JDBC/expected/babel_function_string-before-15-5-or-14-10-vu-verify.out @@ -0,0 +1,789 @@ +-- test REPLICATE function +SELECT REPLICATE(' abc ', 3) +GO +~~START~~ +varchar + abc abc abc +~~END~~ + + +SELECT REPLICATE(N'abc', 3) +GO +~~START~~ +varchar +abcabcabc +~~END~~ + + +SELECT REPLICATE(' abc ', 0) +GO +~~START~~ +varchar + +~~END~~ + + +-- test null condition +SELECT REPLICATE('abc', -3) +GO +~~START~~ +varchar + +~~END~~ + + +SELECT REPLICATE(null, 1) +GO +~~START~~ +varchar + +~~END~~ + + +-- test LEN and DATALENGTH functions +SELECT LEN(N'123') +GO +~~START~~ +int +3 +~~END~~ + + +SELECT LEN(N'123 ') +GO +~~START~~ +int +3 +~~END~~ + + +SELECT LEN(N' 123 ') +GO +~~START~~ +int +6 +~~END~~ + + +SELECT LEN(CAST('123' as char(25))) +GO +~~START~~ +int +3 +~~END~~ + + +SELECT LEN('abc') +GO +~~START~~ +int +3 +~~END~~ + + +SELECT LEN('abc' + 'def') +GO +~~START~~ +int +6 +~~END~~ + + +SELECT LEN('tamaño') +GO +~~START~~ +int +6 +~~END~~ + + +SELECT DATALENGTH(N'123') +GO +~~START~~ +int +3 +~~END~~ + + +SELECT DATALENGTH(N'123 ') +GO +~~START~~ +int +6 +~~END~~ + + +SELECT DATALENGTH(N' 123 ') +GO +~~START~~ +int +9 +~~END~~ + + +SELECT DATALENGTH(CAST('123' as char(25))) +GO +~~START~~ +int +25 +~~END~~ + + +SELECT DATALENGTH('ab' + 'def') +GO +~~START~~ +int +5 +~~END~~ + + +SELECT DATALENGTH('哈哈12345') +GO +~~START~~ +int +11 +~~END~~ + + +-- additional tests for DATALENGTH (more types, nullvalues) +SELECT datalength(a), datalength(b),datalength(c),datalength(d),datalength(e), + datalength(f),datalength(g),datalength(h),datalength(i) FROM babel_function_string_vu_prepare_1 +GO +~~START~~ +int#!#int#!#int#!#int#!#int#!#int#!#int#!#int#!#int +3#!#3#!#3#!#10#!#3#!#3#!#10#!#3#!#3 +#!##!##!##!##!##!##!##!# +~~END~~ + + +SELECT datalength(a), datalength(b),datalength(c),datalength(d),datalength(e), datalength(f),datalength(g),datalength(h),datalength(i) FROM babel_function_string_vu_prepare_2 +GO +~~START~~ +int#!#int#!#int#!#int#!#int#!#int#!#int#!#int#!#int +4#!#8#!#1#!#2#!#2#!#4#!#4#!#8#!#4 +#!##!##!##!##!##!##!##!# +~~END~~ + + +SELECT datalength(a), datalength(b),datalength(c),datalength(d),datalength(e), datalength(f),datalength(g),datalength(h) FROM babel_function_string_vu_prepare_3 +GO +~~START~~ +int#!#int#!#int#!#int#!#int#!#int#!#int#!#int +8#!#8#!#4#!#8#!#8#!#8#!#8#!#16 +#!##!##!##!##!##!##!# +~~END~~ + + +-- test quotename function +SELECT quotename('hardrada', ']') +GO +~~START~~ +nvarchar +[hardrada] +~~END~~ + + +SELECT quotename('gershwin', '<') +GO +~~START~~ +nvarchar +>gershwin< +~~END~~ + + +SELECT quotename('faulkner', '>') +GO +~~START~~ +nvarchar +>faulkner< +~~END~~ + + +SELECT quotename('edgerton', '(') +GO +~~START~~ +nvarchar +(edgerton) +~~END~~ + + +SELECT quotename('denali', ')') +GO +~~START~~ +nvarchar +(denali) +~~END~~ + + +SELECT quotename('charisma', '{') +GO +~~START~~ +nvarchar +{charisma} +~~END~~ + + +SELECT quotename('banana', '}') +GO +~~START~~ +nvarchar +{banana} +~~END~~ + + +SELECT quotename('aardvark', '`') +GO +~~START~~ +nvarchar +`aardvark` +~~END~~ + + +SELECT quotename('128 characters exactly----------------------------------------------------------------------------------------------------------') +GO +~~START~~ +nvarchar +[128 characters exactly----------------------------------------------------------------------------------------------------------] +~~END~~ + + +SELECT +quotename(']]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]') +GO +~~START~~ +nvarchar +[]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]] +~~END~~ + + +SELECT +quotename('""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""') +GO +~~START~~ +nvarchar +[""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""] +~~END~~ + + +SELECT quotename('''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''') +GO +~~START~~ +nvarchar +[''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''] +~~END~~ + + +SELECT quotename('') +GO +~~START~~ +nvarchar +[] +~~END~~ + + +-- regtype error expected pending BABEL-883 +SELECT pg_typeof(quotename('a')) +GO +~~ERROR (Code: 33557097)~~ + +~~ERROR (Message: data type regtype is not supported yet)~~ + + +SELECT quotename(CAST('abc' as varchar)) +GO +~~START~~ +nvarchar +[abc] +~~END~~ + + +SELECT quotename(CAST('abc' as sys.nvarchar)) +GO +~~START~~ +nvarchar +[abc] +~~END~~ + + +SELECT quotename(CAST('abc' as text)) +GO +~~START~~ +nvarchar +[abc] +~~END~~ + + +SELECT quotename('invalid char', 'F') +GO +~~START~~ +nvarchar + +~~END~~ + + +SELECT quotename('too long char', 'aa') +GO +~~START~~ +nvarchar + +~~END~~ + + +SELECT quotename('129 characters exactly-----------------------------------------------------------------------------------------------------------') +GO +~~START~~ +nvarchar + +~~END~~ + + +SELECT quotename('default should be bracket') +GO +~~START~~ +nvarchar +[default should be bracket] +~~END~~ + + +SELECT quotename('abc [] def') +GO +~~START~~ +nvarchar +[abc []] def] +~~END~~ + + +SELECT quotename(NULL) +GO +~~START~~ +nvarchar + +~~END~~ + + +SELECT quotename(NULL, NULL) +GO +~~START~~ +nvarchar + +~~END~~ + + +SELECT quotename('hey', NULL) +GO +~~START~~ +nvarchar + +~~END~~ + + +SELECT quotename(NULL, '[') +GO +~~START~~ +nvarchar + +~~END~~ + + + +-- test unicode function +SELECT unicode(null) +GO +~~START~~ +int + +~~END~~ + + +SELECT unicode('Åkergatan 24') +GO +~~START~~ +int +197 +~~END~~ + + +SELECT nchar(unicode('Åkergatan 24')) +GO +~~START~~ +nvarchar +Å +~~END~~ + + +SELECT unicode(cast('Āmazon' AS nvarchar)) +GO +~~START~~ +int +256 +~~END~~ + + +SELECT unicode(CAST('Āmazon' as nvarchar)) +GO +~~START~~ +int +256 +~~END~~ + + +SELECT unicode(cast('Ƃ' as nchar)) +GO +~~START~~ +int +386 +~~END~~ + + +SELECT unicode(CAST('Ƃ' as nchar)) +GO +~~START~~ +int +386 +~~END~~ + + +SELECT STRING_SPLIT('Lorem ipsum dolor sit amet.', ' ') +GO +~~START~~ +varchar +Lorem +ipsum +dolor +sit +amet. +~~END~~ + + +SELECT STRING_SPLIT('clothing,road,,touring,bike', ',') +GO +~~START~~ +varchar +clothing +road + +touring +bike +~~END~~ + + +SELECT STRING_SPLIT('||||||||', '|') +GO +~~START~~ +varchar + + + + + + + + + +~~END~~ + + +SELECT STRING_SPLIT(NULL, ' ') +GO +~~START~~ +varchar +~~END~~ + + +-- test invalid separator +SELECT STRING_SPLIT('asdf', '') +GO +~~START~~ +varchar +~~ERROR (Code: 33557097)~~ + +~~ERROR (Message: Invalid separator: )~~ + + +SELECT STRING_SPLIT('asdf', NULL) +GO +~~START~~ +varchar +~~END~~ + + +SELECT STRING_SPLIT(NULL, NULL) +GO +~~START~~ +varchar +~~END~~ + + +SELECT STRING_SPLIT(CAST('nvarchar nvarchar nvarchar' as nvarchar), CAST(' ' as nvarchar)) +GO +~~START~~ +varchar +nvarchar +nvarchar +nvarchar +~~END~~ + + +SELECT STRING_SPLIT(CAST('varchar varchar varchar' as varchar), CAST(' ' as varchar)) +GO +~~START~~ +varchar +varchar +varchar +varchar +~~END~~ + + +SELECT STRING_SPLIT('char char char', ' ') +GO +~~START~~ +varchar +char +char +char +~~END~~ + + +SELECT STRING_SPLIT('a,b,c,d', ',') +GO +~~START~~ +varchar +a +b +c +d +~~END~~ + + +SELECT STRING_SPLIT('mississippi island lives in igloo', 'i') +GO +~~START~~ +varchar +m +ss +ss +pp + +sland l +ves +n +gloo +~~END~~ + + +SELECT STRING_SPLIT(CAST('asdf' as nchar(4)), ' ') +GO +~~START~~ +varchar +asdf +~~END~~ + + +SELECT STRING_SPLIT(CAST('asdf' as char(4)), ' ') +GO +~~START~~ +varchar +asdf +~~END~~ + + +-- test invalid separator +SELECT STRING_SPLIT('Lorem ipsum', 'too many chars') +GO +~~START~~ +varchar +~~ERROR (Code: 33557097)~~ + +~~ERROR (Message: Invalid separator: too many chars)~~ + + +SELECT value FROM STRING_SPLIT('Lorem ipsum dolor sit amet.', ' ') +GO +~~START~~ +varchar +Lorem +ipsum +dolor +sit +amet. +~~END~~ + + +SELECT mycol FROM STRING_SPLIT('Lorem ipsum dolor sit amet.', ' ') +GO +~~ERROR (Code: 33557097)~~ + +~~ERROR (Message: column "mycol" does not exist)~~ + + + +-- STRING_ESCAPE tests +SELECT STRING_ESCAPE('foo', 'notjson') +GO +~~START~~ +nvarchar + +~~END~~ + + +SELECT STRING_ESCAPE('foo', '') +GO +~~START~~ +nvarchar + +~~END~~ + + +SELECT STRING_ESCAPE('foo', NULL) +GO +~~START~~ +nvarchar + +~~END~~ + + +SELECT STRING_ESCAPE(NULL, '') +GO +~~START~~ +nvarchar + +~~END~~ + + +SELECT STRING_ESCAPE(NULL, NULL) +GO +~~START~~ +nvarchar + +~~END~~ + + +SELECT STRING_ESCAPE(NULL, 'json') +GO +~~START~~ +nvarchar + +~~END~~ + + +SELECT STRING_ESCAPE(' ', 'json') +GO +~~START~~ +nvarchar +\t +~~END~~ + + +SELECT STRING_ESCAPE('"', 'json') +GO +~~START~~ +nvarchar +\" +~~END~~ + + +SELECT STRING_ESCAPE('\', 'json') +GO +~~START~~ +nvarchar +\\ +~~END~~ + + +SELECT STRING_ESCAPE('/', 'json') +GO +~~START~~ +nvarchar +\/ +~~END~~ + + +SELECT STRING_ESCAPE(chr(1), 'json') +GO +~~START~~ +nvarchar +\u0001 +~~END~~ + + +SELECT STRING_ESCAPE(chr(2), 'json') +GO +~~START~~ +nvarchar +\u0002 +~~END~~ + + +SELECT STRING_ESCAPE(chr(8), 'json') +GO +~~START~~ +nvarchar +\b +~~END~~ + + +SELECT STRING_ESCAPE(chr(9), 'json') +GO +~~START~~ +nvarchar +\t +~~END~~ + + +SELECT STRING_ESCAPE(' +', 'json') +GO +~~START~~ +nvarchar +\n +~~END~~ + + +SELECT STRING_ESCAPE(chr(10), 'json') +GO +~~START~~ +nvarchar +\n +~~END~~ + + +SELECT STRING_ESCAPE(chr(11), 'json') +GO +~~START~~ +nvarchar +\u000b +~~END~~ + + +SELECT STRING_ESCAPE(chr(12), 'json') +GO +~~START~~ +nvarchar +\f +~~END~~ + + +SELECT STRING_ESCAPE(chr(13), 'json') +GO +~~START~~ +nvarchar +\r +~~END~~ + + +SELECT STRING_ESCAPE(chr(31), 'json') +GO +~~START~~ +nvarchar +\u001f +~~END~~ + + +SELECT STRING_ESCAPE('lorem ipsum dolor amet +consectetur adipiscing elit', 'json') +GO +~~START~~ +nvarchar +lorem ipsum dolor amet\t\nconsectetur adipiscing elit +~~END~~ + diff --git a/test/JDBC/input/BABEL_1940.sql b/test/JDBC/input/BABEL_1940.sql new file mode 100644 index 0000000000..13bab80027 --- /dev/null +++ b/test/JDBC/input/BABEL_1940.sql @@ -0,0 +1,76 @@ +SELECT CONVERT(VARCHAR(10), 0x123456789) +GO + +SELECT CONVERT(VARCHAR(10), 0x80) +GO + +SELECT CONVERT(VARCHAR(10), 0xaaa) +GO + +SELECT CONVERT(VARCHAR(10), 0x330033) +GO + +SELECT CONVERT(VARBINARY(10), 'ｳ') +GO + +SELECT CONVERT(VARBINARY(10), 'ﾊﾟ') +GO + +SELECT CONVERT(VARBINARY(10), 'A') +GO + +SELECT CONVERT(VARBINARY(10), 'ア') +GO + +SELECT CONVERT(VARBINARY(10), 0x81) +GO + +SELECT CONVERT(VARBINARY(10), 0x330033) +GO + +DECLARE @key varchar(20) = 'part1' +DECLARE @email varchar(20) = 'part2' +SELECT CONVERT(VARCHAR(10), HASHBYTES('SHA1', @key + LOWER(@email))) +GO + + +CREATE TABLE babel_1940_t1 (a VARBINARY(9)) +GO + +INSERT INTO babel_1940_t1 VALUES(0x80) +INSERT INTO babel_1940_t1 VALUES(0xaaa) +INSERT INTO babel_1940_t1 VALUES(0x123456789) +GO + +SELECT * FROM babel_1940_t1 +GO + +SELECT CONVERT(VARCHAR(9), a) FROM babel_1940_t1 +GO + +SELECT CAST(a as VARCHAR(9)) FROM babel_1940_t1 +GO + +SELECT CAST(a as VARCHAR(10)) FROM babel_1940_t1 +GO + + +CREATE TABLE babel_1940_t2(a varchar(10) collate japanese_cs_as); +GO + +insert into babel_1940_t2 values ('ｳ'), ('C'), ('ﾊﾟ'), ('３'), ('c'), ('ｲ'), ('Ｃ'),('ﾊ'),('1'), +('ｱ'),('パ'), ('b'), ('2'), ('B'),('１'), ('Ａ'),('ア'),('A'), ('a'),('AbC'), ('aBc'); +GO + +SELECT CONVERT(VARBINARY(10), a) FROM babel_1940_t2 +GO + +SELECT CONVERT(VARBINARY(10), CONVERT(VARCHAR(10), CONVERT(VARCHAR(10), a))) FROM babel_1940_t2 +GO + + +DROP TABLE babel_1940_t2 +GO + +DROP TABLE babel_1940_t1 +GO \ No newline at end of file From d2db3fbc02702cebbcc2a10fd82b7a97f7eb52f2 Mon Sep 17 00:00:00 2001 From: Tanzeel Khan Date: Thu, 26 Oct 2023 15:11:32 +0000 Subject: [PATCH 05/28] correct upgrade tests for babel_function_string Signed-off-by: Tanzeel Khan --- ...string-before-15-5-or-14-10-vu-prepare.out | 42 +++ ...string-before-15-5-or-14-10-vu-prepare.sql | 30 ++ ..._string-before-15-5-or-14-10-vu-verify.sql | 292 ++++++++++++++++++ test/JDBC/jdbc_schedule | 2 + test/JDBC/upgrade/13_4/schedule | 2 +- test/JDBC/upgrade/13_5/schedule | 2 +- test/JDBC/upgrade/13_6/schedule | 2 +- test/JDBC/upgrade/13_7/schedule | 2 +- test/JDBC/upgrade/13_8/schedule | 2 +- test/JDBC/upgrade/13_9/schedule | 2 +- test/JDBC/upgrade/14_10/schedule | 2 +- test/JDBC/upgrade/14_3/schedule | 2 +- test/JDBC/upgrade/14_5/schedule | 2 +- test/JDBC/upgrade/14_6/schedule | 2 +- test/JDBC/upgrade/14_7/schedule | 2 +- test/JDBC/upgrade/14_8/schedule | 2 +- test/JDBC/upgrade/14_9/schedule | 2 +- test/JDBC/upgrade/15_1/schedule | 2 +- test/JDBC/upgrade/15_2/schedule | 2 +- test/JDBC/upgrade/15_3/schedule | 2 +- test/JDBC/upgrade/15_4/schedule | 2 +- 21 files changed, 383 insertions(+), 17 deletions(-) create mode 100644 test/JDBC/expected/babel_function_string-before-15-5-or-14-10-vu-prepare.out create mode 100644 test/JDBC/input/babel_function_string-before-15-5-or-14-10-vu-prepare.sql create mode 100644 test/JDBC/input/babel_function_string-before-15-5-or-14-10-vu-verify.sql diff --git a/test/JDBC/expected/babel_function_string-before-15-5-or-14-10-vu-prepare.out b/test/JDBC/expected/babel_function_string-before-15-5-or-14-10-vu-prepare.out new file mode 100644 index 0000000000..e16f0ffe48 --- /dev/null +++ b/test/JDBC/expected/babel_function_string-before-15-5-or-14-10-vu-prepare.out @@ -0,0 +1,42 @@ +-- additional tests for DATALENGTH (more types, nullvalues) +CREATE table babel_function_string_vu_prepare_1 (a binary(10), b image, c varbinary(10), d char(10), + e varchar(10), f text, g nchar(10), h nvarchar(10), i ntext) +GO + +INSERT into babel_function_string_vu_prepare_1 values (cast('abc' as binary(10)), cast('abc' as image), cast('abc' as varbinary(10)),'abc','abc','abc','abc','abc','abc') +GO +~~ROW COUNT: 1~~ + + +INSERT into babel_function_string_vu_prepare_1 values (null, null, null, null, null, null,null, null, null) +GO +~~ROW COUNT: 1~~ + + +CREATE table babel_function_string_vu_prepare_2 (a integer, b bigint, c bit, d smallint, e tinyint, f decimal, g numeric, h float, i real) +GO + +INSERT into babel_function_string_vu_prepare_2 values (1, 1, 1, 1, 1, 1, 1, 1, 1) +GO +~~ROW COUNT: 1~~ + + +INSERT into babel_function_string_vu_prepare_2 values (null, null, null, null, null, null,null, null, null) +GO +~~ROW COUNT: 1~~ + + +CREATE table babel_function_string_vu_prepare_3 (a smallmoney, b money, c date, d datetime, e datetime2, f smalldatetime, g time, h uniqueidentifier) +GO + +INSERT into babel_function_string_vu_prepare_3 values (cast(1 as smallmoney), cast(1 as money), cast('2020-02-20' as date), cast('2020-02-20 20:20:20.888' as datetime), + cast('2020-02-20 20:20:20.88888' as datetime2), cast('2020-02-20 20:20:20' as smalldatetime), cast('20:20:20.888' as time), + cast('6F9619FF-8B86-D011-B42D-00C04FC964FF' as uniqueidentifier)) +GO +~~ROW COUNT: 1~~ + + +INSERT into babel_function_string_vu_prepare_3 values (null, null, null, null, null, null,null, null) +GO +~~ROW COUNT: 1~~ + diff --git a/test/JDBC/input/babel_function_string-before-15-5-or-14-10-vu-prepare.sql b/test/JDBC/input/babel_function_string-before-15-5-or-14-10-vu-prepare.sql new file mode 100644 index 0000000000..759440425e --- /dev/null +++ b/test/JDBC/input/babel_function_string-before-15-5-or-14-10-vu-prepare.sql @@ -0,0 +1,30 @@ +-- additional tests for DATALENGTH (more types, nullvalues) +CREATE table babel_function_string_vu_prepare_1 (a binary(10), b image, c varbinary(10), d char(10), + e varchar(10), f text, g nchar(10), h nvarchar(10), i ntext) +GO + +INSERT into babel_function_string_vu_prepare_1 values (cast('abc' as binary(10)), cast('abc' as image), cast('abc' as varbinary(10)),'abc','abc','abc','abc','abc','abc') +GO + +INSERT into babel_function_string_vu_prepare_1 values (null, null, null, null, null, null,null, null, null) +GO + +CREATE table babel_function_string_vu_prepare_2 (a integer, b bigint, c bit, d smallint, e tinyint, f decimal, g numeric, h float, i real) +GO + +INSERT into babel_function_string_vu_prepare_2 values (1, 1, 1, 1, 1, 1, 1, 1, 1) +GO + +INSERT into babel_function_string_vu_prepare_2 values (null, null, null, null, null, null,null, null, null) +GO + +CREATE table babel_function_string_vu_prepare_3 (a smallmoney, b money, c date, d datetime, e datetime2, f smalldatetime, g time, h uniqueidentifier) +GO + +INSERT into babel_function_string_vu_prepare_3 values (cast(1 as smallmoney), cast(1 as money), cast('2020-02-20' as date), cast('2020-02-20 20:20:20.888' as datetime), + cast('2020-02-20 20:20:20.88888' as datetime2), cast('2020-02-20 20:20:20' as smalldatetime), cast('20:20:20.888' as time), + cast('6F9619FF-8B86-D011-B42D-00C04FC964FF' as uniqueidentifier)) +GO + +INSERT into babel_function_string_vu_prepare_3 values (null, null, null, null, null, null,null, null) +GO \ No newline at end of file diff --git a/test/JDBC/input/babel_function_string-before-15-5-or-14-10-vu-verify.sql b/test/JDBC/input/babel_function_string-before-15-5-or-14-10-vu-verify.sql new file mode 100644 index 0000000000..204c3a90e1 --- /dev/null +++ b/test/JDBC/input/babel_function_string-before-15-5-or-14-10-vu-verify.sql @@ -0,0 +1,292 @@ +-- test REPLICATE function +SELECT REPLICATE(' abc ', 3) +GO + +SELECT REPLICATE(N'abc', 3) +GO + +SELECT REPLICATE(' abc ', 0) +GO + +-- test null condition +SELECT REPLICATE('abc', -3) +GO + +SELECT REPLICATE(null, 1) +GO + +-- test LEN and DATALENGTH functions +SELECT LEN(N'123') +GO + +SELECT LEN(N'123 ') +GO + +SELECT LEN(N' 123 ') +GO + +SELECT LEN(CAST('123' as char(25))) +GO + +SELECT LEN('abc') +GO + +SELECT LEN('abc' + 'def') +GO + +SELECT LEN('tamaño') +GO + +SELECT DATALENGTH(N'123') +GO + +SELECT DATALENGTH(N'123 ') +GO + +SELECT DATALENGTH(N' 123 ') +GO + +SELECT DATALENGTH(CAST('123' as char(25))) +GO + +SELECT DATALENGTH('ab' + 'def') +GO + +SELECT DATALENGTH('哈哈12345') +GO + +-- additional tests for DATALENGTH (more types, nullvalues) +SELECT datalength(a), datalength(b),datalength(c),datalength(d),datalength(e), + datalength(f),datalength(g),datalength(h),datalength(i) FROM babel_function_string_vu_prepare_1 +GO + +SELECT datalength(a), datalength(b),datalength(c),datalength(d),datalength(e), datalength(f),datalength(g),datalength(h),datalength(i) FROM babel_function_string_vu_prepare_2 +GO + +SELECT datalength(a), datalength(b),datalength(c),datalength(d),datalength(e), datalength(f),datalength(g),datalength(h) FROM babel_function_string_vu_prepare_3 +GO + +-- test quotename function +SELECT quotename('hardrada', ']') +GO + +SELECT quotename('gershwin', '<') +GO + +SELECT quotename('faulkner', '>') +GO + +SELECT quotename('edgerton', '(') +GO + +SELECT quotename('denali', ')') +GO + +SELECT quotename('charisma', '{') +GO + +SELECT quotename('banana', '}') +GO + +SELECT quotename('aardvark', '`') +GO + +SELECT quotename('128 characters exactly----------------------------------------------------------------------------------------------------------') +GO + +SELECT +quotename(']]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]') +GO + +SELECT +quotename('""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""') +GO + +SELECT quotename('''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''') +GO + +SELECT quotename('') +GO + +-- regtype error expected pending BABEL-883 +SELECT pg_typeof(quotename('a')) +GO + +SELECT quotename(CAST('abc' as varchar)) +GO + +SELECT quotename(CAST('abc' as sys.nvarchar)) +GO + +SELECT quotename(CAST('abc' as text)) +GO + +SELECT quotename('invalid char', 'F') +GO + +SELECT quotename('too long char', 'aa') +GO + +SELECT quotename('129 characters exactly-----------------------------------------------------------------------------------------------------------') +GO + +SELECT quotename('default should be bracket') +GO + +SELECT quotename('abc [] def') +GO + +SELECT quotename(NULL) +GO + +SELECT quotename(NULL, NULL) +GO + +SELECT quotename('hey', NULL) +GO + +SELECT quotename(NULL, '[') +GO + +-- test unicode function + +SELECT unicode(null) +GO + +SELECT unicode('Åkergatan 24') +GO + +SELECT nchar(unicode('Åkergatan 24')) +GO + +SELECT unicode(cast('Āmazon' AS nvarchar)) +GO + +SELECT unicode(CAST('Āmazon' as nvarchar)) +GO + +SELECT unicode(cast('Ƃ' as nchar)) +GO + +SELECT unicode(CAST('Ƃ' as nchar)) +GO + +SELECT STRING_SPLIT('Lorem ipsum dolor sit amet.', ' ') +GO + +SELECT STRING_SPLIT('clothing,road,,touring,bike', ',') +GO + +SELECT STRING_SPLIT('||||||||', '|') +GO + +SELECT STRING_SPLIT(NULL, ' ') +GO + +-- test invalid separator +SELECT STRING_SPLIT('asdf', '') +GO + +SELECT STRING_SPLIT('asdf', NULL) +GO + +SELECT STRING_SPLIT(NULL, NULL) +GO + +SELECT STRING_SPLIT(CAST('nvarchar nvarchar nvarchar' as nvarchar), CAST(' ' as nvarchar)) +GO + +SELECT STRING_SPLIT(CAST('varchar varchar varchar' as varchar), CAST(' ' as varchar)) +GO + +SELECT STRING_SPLIT('char char char', ' ') +GO + +SELECT STRING_SPLIT('a,b,c,d', ',') +GO + +SELECT STRING_SPLIT('mississippi island lives in igloo', 'i') +GO + +SELECT STRING_SPLIT(CAST('asdf' as nchar(4)), ' ') +GO + +SELECT STRING_SPLIT(CAST('asdf' as char(4)), ' ') +GO + +-- test invalid separator +SELECT STRING_SPLIT('Lorem ipsum', 'too many chars') +GO + +SELECT value FROM STRING_SPLIT('Lorem ipsum dolor sit amet.', ' ') +GO + +SELECT mycol FROM STRING_SPLIT('Lorem ipsum dolor sit amet.', ' ') +GO + +-- STRING_ESCAPE tests + +SELECT STRING_ESCAPE('foo', 'notjson') +GO + +SELECT STRING_ESCAPE('foo', '') +GO + +SELECT STRING_ESCAPE('foo', NULL) +GO + +SELECT STRING_ESCAPE(NULL, '') +GO + +SELECT STRING_ESCAPE(NULL, NULL) +GO + +SELECT STRING_ESCAPE(NULL, 'json') +GO + +SELECT STRING_ESCAPE(' ', 'json') +GO + +SELECT STRING_ESCAPE('"', 'json') +GO + +SELECT STRING_ESCAPE('\', 'json') +GO + +SELECT STRING_ESCAPE('/', 'json') +GO + +SELECT STRING_ESCAPE(chr(1), 'json') +GO + +SELECT STRING_ESCAPE(chr(2), 'json') +GO + +SELECT STRING_ESCAPE(chr(8), 'json') +GO + +SELECT STRING_ESCAPE(chr(9), 'json') +GO + +SELECT STRING_ESCAPE(' +', 'json') +GO + +SELECT STRING_ESCAPE(chr(10), 'json') +GO + +SELECT STRING_ESCAPE(chr(11), 'json') +GO + +SELECT STRING_ESCAPE(chr(12), 'json') +GO + +SELECT STRING_ESCAPE(chr(13), 'json') +GO + +SELECT STRING_ESCAPE(chr(31), 'json') +GO + +SELECT STRING_ESCAPE('lorem ipsum dolor amet +consectetur adipiscing elit', 'json') +GO \ No newline at end of file diff --git a/test/JDBC/jdbc_schedule b/test/JDBC/jdbc_schedule index c21361e06f..7b66ef0441 100644 --- a/test/JDBC/jdbc_schedule +++ b/test/JDBC/jdbc_schedule @@ -89,6 +89,8 @@ ignore#!#forjson-datatypes-before-14_10-or-15_5-vu-cleanup ignore#!#orderby-before-15_3-vu-prepare ignore#!#orderby-before-15_3-vu-verify ignore#!#orderby-before-15_3-vu-cleanup +ignore#!#babel_function_string-before-15-5-or-14-10-vu-prepare +ignore#!#babel_function_string-before-15-5-or-14-10-vu-verify # These tests are meant for upgrade scenario where source version is 13_X ignore#!#sys_database_principals_dep_for_13_x-vu-cleanup diff --git a/test/JDBC/upgrade/13_4/schedule b/test/JDBC/upgrade/13_4/schedule index ed11eb933d..639d33751d 100644 --- a/test/JDBC/upgrade/13_4/schedule +++ b/test/JDBC/upgrade/13_4/schedule @@ -78,7 +78,7 @@ babelfish_cast_floor babelfish_migration_mode babelfish_namespace_ext babelfish_sysdatabases -babel_function_string +babel_function_string-before-15-5-or-14-10 BABEL_GRANT_CONNECT babel_isnumeric BABEL-LOGIN diff --git a/test/JDBC/upgrade/13_5/schedule b/test/JDBC/upgrade/13_5/schedule index 1e224e0b43..95e858c9a5 100644 --- a/test/JDBC/upgrade/13_5/schedule +++ b/test/JDBC/upgrade/13_5/schedule @@ -84,7 +84,7 @@ babelfish_cast_floor babelfish_migration_mode babelfish_namespace_ext babelfish_sysdatabases -babel_function_string +babel_function_string-before-15-5-or-14-10 BABEL_GRANT_CONNECT babel_isnumeric BABEL-LOGIN diff --git a/test/JDBC/upgrade/13_6/schedule b/test/JDBC/upgrade/13_6/schedule index 3943bbadbb..b32989ea35 100644 --- a/test/JDBC/upgrade/13_6/schedule +++ b/test/JDBC/upgrade/13_6/schedule @@ -106,7 +106,7 @@ babelfish_inconsistent_metadata babelfish_migration_mode babelfish_namespace_ext babelfish_sysdatabases -babel_function_string +babel_function_string-before-15-5-or-14-10 BABEL_GRANT_CONNECT babel_isnumeric BABEL-LOGIN diff --git a/test/JDBC/upgrade/13_7/schedule b/test/JDBC/upgrade/13_7/schedule index 474e6900d4..9a9d192198 100644 --- a/test/JDBC/upgrade/13_7/schedule +++ b/test/JDBC/upgrade/13_7/schedule @@ -104,7 +104,7 @@ babelfish_inconsistent_metadata babelfish_migration_mode babelfish_namespace_ext babelfish_sysdatabases -babel_function_string +babel_function_string-before-15-5-or-14-10 BABEL_GRANT_CONNECT babel_isnumeric BABEL-LOGIN diff --git a/test/JDBC/upgrade/13_8/schedule b/test/JDBC/upgrade/13_8/schedule index 474e6900d4..9a9d192198 100644 --- a/test/JDBC/upgrade/13_8/schedule +++ b/test/JDBC/upgrade/13_8/schedule @@ -104,7 +104,7 @@ babelfish_inconsistent_metadata babelfish_migration_mode babelfish_namespace_ext babelfish_sysdatabases -babel_function_string +babel_function_string-before-15-5-or-14-10 BABEL_GRANT_CONNECT babel_isnumeric BABEL-LOGIN diff --git a/test/JDBC/upgrade/13_9/schedule b/test/JDBC/upgrade/13_9/schedule index c606176c9a..21eeb1ddbc 100644 --- a/test/JDBC/upgrade/13_9/schedule +++ b/test/JDBC/upgrade/13_9/schedule @@ -103,7 +103,7 @@ babelfish_inconsistent_metadata babelfish_migration_mode babelfish_namespace_ext babelfish_sysdatabases -babel_function_string +babel_function_string-before-15-5-or-14-10 BABEL_GRANT_CONNECT babel_isnumeric BABEL-LOGIN diff --git a/test/JDBC/upgrade/14_10/schedule b/test/JDBC/upgrade/14_10/schedule index a9897f774c..3c3ca60159 100644 --- a/test/JDBC/upgrade/14_10/schedule +++ b/test/JDBC/upgrade/14_10/schedule @@ -185,7 +185,7 @@ babel_datetime babel_char BABEL-SQUARE BABEL-728 -babel_function_string +babel_function_string-before-15-5-or-14-10 BABEL-1566 BABEL-3360 BABEL-3380 diff --git a/test/JDBC/upgrade/14_3/schedule b/test/JDBC/upgrade/14_3/schedule index f68c4e41fe..8cae86caf8 100644 --- a/test/JDBC/upgrade/14_3/schedule +++ b/test/JDBC/upgrade/14_3/schedule @@ -108,7 +108,7 @@ babelfish_inconsistent_metadata babelfish_migration_mode babelfish_namespace_ext babelfish_sysdatabases -babel_function_string +babel_function_string-before-15-5-or-14-10 BABEL_GRANT_CONNECT babel_isnumeric BABEL-LOGIN diff --git a/test/JDBC/upgrade/14_5/schedule b/test/JDBC/upgrade/14_5/schedule index c78b90e882..24eb3372d8 100644 --- a/test/JDBC/upgrade/14_5/schedule +++ b/test/JDBC/upgrade/14_5/schedule @@ -107,7 +107,7 @@ babelfish_inconsistent_metadata babelfish_migration_mode babelfish_namespace_ext babelfish_sysdatabases -babel_function_string +babel_function_string-before-15-5-or-14-10 BABEL_GRANT_CONNECT babel_isnumeric BABEL-LOGIN diff --git a/test/JDBC/upgrade/14_6/schedule b/test/JDBC/upgrade/14_6/schedule index 61edb201b9..0e07709df5 100644 --- a/test/JDBC/upgrade/14_6/schedule +++ b/test/JDBC/upgrade/14_6/schedule @@ -122,7 +122,7 @@ babelfish_integrity_checker babelfish_migration_mode babelfish_namespace_ext babelfish_sysdatabases -babel_function_string +babel_function_string-before-15-5-or-14-10 BABEL_GRANT_CONNECT babel_isnumeric BABEL-LOGIN diff --git a/test/JDBC/upgrade/14_7/schedule b/test/JDBC/upgrade/14_7/schedule index b0fc3c87b8..c7bd69115e 100644 --- a/test/JDBC/upgrade/14_7/schedule +++ b/test/JDBC/upgrade/14_7/schedule @@ -133,7 +133,7 @@ babelfish_integrity_checker babelfish_migration_mode babelfish_namespace_ext babelfish_sysdatabases -babel_function_string +babel_function_string-before-15-5-or-14-10 BABEL_GRANT_CONNECT babel_isnumeric BABEL-LOGIN diff --git a/test/JDBC/upgrade/14_8/schedule b/test/JDBC/upgrade/14_8/schedule index b5aa90477e..0b372b4d07 100644 --- a/test/JDBC/upgrade/14_8/schedule +++ b/test/JDBC/upgrade/14_8/schedule @@ -131,7 +131,7 @@ babelfish_integrity_checker babelfish_migration_mode babelfish_namespace_ext babelfish_sysdatabases -babel_function_string +babel_function_string-before-15-5-or-14-10 BABEL_GRANT_CONNECT babel_isnumeric BABEL-LOGIN diff --git a/test/JDBC/upgrade/14_9/schedule b/test/JDBC/upgrade/14_9/schedule index 356a58ce8a..6ebd35e4ca 100644 --- a/test/JDBC/upgrade/14_9/schedule +++ b/test/JDBC/upgrade/14_9/schedule @@ -185,7 +185,7 @@ babel_datetime babel_char BABEL-SQUARE BABEL-728 -babel_function_string +babel_function_string-before-15-5-or-14-10 BABEL-1566 BABEL-3360 BABEL-3380 diff --git a/test/JDBC/upgrade/15_1/schedule b/test/JDBC/upgrade/15_1/schedule index abd07f86fb..08b63a3f60 100644 --- a/test/JDBC/upgrade/15_1/schedule +++ b/test/JDBC/upgrade/15_1/schedule @@ -122,7 +122,7 @@ babelfish_integrity_checker babelfish_migration_mode babelfish_namespace_ext babelfish_sysdatabases -babel_function_string +babel_function_string-before-15-5-or-14-10 BABEL_GRANT_CONNECT babel_isnumeric BABEL-LOGIN diff --git a/test/JDBC/upgrade/15_2/schedule b/test/JDBC/upgrade/15_2/schedule index ff7db62fd1..e93fd5b194 100644 --- a/test/JDBC/upgrade/15_2/schedule +++ b/test/JDBC/upgrade/15_2/schedule @@ -133,7 +133,7 @@ babelfish_integrity_checker babelfish_migration_mode babelfish_namespace_ext babelfish_sysdatabases -babel_function_string +babel_function_string-before-15-5-or-14-10 BABEL_GRANT_CONNECT babel_isnumeric BABEL-LOGIN diff --git a/test/JDBC/upgrade/15_3/schedule b/test/JDBC/upgrade/15_3/schedule index e8733a81e6..af790d7fc3 100644 --- a/test/JDBC/upgrade/15_3/schedule +++ b/test/JDBC/upgrade/15_3/schedule @@ -142,7 +142,7 @@ babelfish_integrity_checker babelfish_migration_mode babelfish_namespace_ext babelfish_sysdatabases -babel_function_string +babel_function_string-before-15-5-or-14-10 BABEL_GRANT_CONNECT babel_isnumeric BABEL-LOGIN diff --git a/test/JDBC/upgrade/15_4/schedule b/test/JDBC/upgrade/15_4/schedule index e7f12532e6..6659831f0a 100644 --- a/test/JDBC/upgrade/15_4/schedule +++ b/test/JDBC/upgrade/15_4/schedule @@ -144,7 +144,7 @@ babelfish_integrity_checker babelfish_migration_mode babelfish_namespace_ext babelfish_sysdatabases -babel_function_string +babel_function_string-before-15-5-or-14-10 BABEL_GRANT_CONNECT babel_isnumeric BABEL-LOGIN From 3011a283631c6aef6fca6b120df0f12012dffef7 Mon Sep 17 00:00:00 2001 From: Tanzeel Khan Date: Tue, 31 Oct 2023 08:29:52 +0000 Subject: [PATCH 06/28] update tests Signed-off-by: Tanzeel Khan --- test/JDBC/expected/BABEL_1940.out | 97 ++++++++++++++----------------- test/JDBC/input/BABEL_1940.sql | 25 +++++--- 2 files changed, 62 insertions(+), 60 deletions(-) diff --git a/test/JDBC/expected/BABEL_1940.out b/test/JDBC/expected/BABEL_1940.out index 6e3b515b88..aefe36d534 100644 --- a/test/JDBC/expected/BABEL_1940.out +++ b/test/JDBC/expected/BABEL_1940.out @@ -1,3 +1,5 @@ + +-- Test is only valid when default server encoding is WIN1252 SELECT CONVERT(VARCHAR(10), 0x123456789) GO ~~START~~ @@ -6,6 +8,38 @@ varchar ~~END~~ +SELECT CONVERT(VARBINARY(10), '#Eg‰') +GO +~~START~~ +varbinary +0123456789 +~~END~~ + + +SELECT CONVERT(VARCHAR(1), 0x99) +GO +~~START~~ +varchar +™ +~~END~~ + + +SELECT CONVERT(VARCHAR(2), 0x999999) +GO +~~START~~ +varchar +™™ +~~END~~ + + +SELECT CONVERT(VARBINARY(1), '™') +GO +~~START~~ +varbinary +99 +~~END~~ + + SELECT CONVERT(VARCHAR(10), 0x80) GO ~~START~~ @@ -14,12 +48,14 @@ varchar ~~END~~ -SELECT CONVERT(VARCHAR(10), 0xaaa) +-- 0x81 does not exist is empty in some encodings +SELECT CONVERT(VARCHAR(10), 0x81) GO ~~START~~ varchar -ª -~~END~~ +~~ERROR (Code: 33557097)~~ + +~~ERROR (Message: character with byte sequence 0x81 in encoding "WIN1252" has no equivalent in encoding "UTF8")~~ SELECT CONVERT(VARCHAR(10), 0x330033) @@ -147,69 +183,24 @@ varchar CREATE TABLE babel_1940_t2(a varchar(10) collate japanese_cs_as); GO -insert into babel_1940_t2 values ('ｳ'), ('C'), ('ﾊﾟ'), ('３'), ('c'), ('ｲ'), ('Ｃ'),('ﾊ'),('1'), -('ｱ'),('パ'), ('b'), ('2'), ('B'),('１'), ('Ａ'),('ア'),('A'), ('a'),('AbC'), ('aBc'); +INSERT INTO babel_1940_t2 VALUES ('a'), ('b'), ('™'), ('ƀ'), ('ä'); GO -~~ROW COUNT: 21~~ +~~ROW COUNT: 5~~ +-- Characters with no mapping transform to Ox3F or ? SELECT CONVERT(VARBINARY(10), a) FROM babel_1940_t2 GO ~~START~~ varbinary -3F -43 -3F3F -3F -63 -3F -3F -3F -31 -3F -3F -62 -32 -42 -3F -3F -3F -41 61 -416243 -614263 -~~END~~ - - -SELECT CONVERT(VARBINARY(10), CONVERT(VARCHAR(10), CONVERT(VARCHAR(10), a))) FROM babel_1940_t2 -GO -~~START~~ -varbinary -3F -43 -3F3F -3F -63 -3F -3F -3F -31 -3F -3F 62 -32 -42 -3F -3F +99 3F -41 -61 -416243 -614263 +E4 ~~END~~ - DROP TABLE babel_1940_t2 GO diff --git a/test/JDBC/input/BABEL_1940.sql b/test/JDBC/input/BABEL_1940.sql index 13bab80027..4714820233 100644 --- a/test/JDBC/input/BABEL_1940.sql +++ b/test/JDBC/input/BABEL_1940.sql @@ -1,10 +1,25 @@ +-- Test is only valid when default server encoding is WIN1252 + SELECT CONVERT(VARCHAR(10), 0x123456789) GO +SELECT CONVERT(VARBINARY(10), '#Eg‰') +GO + +SELECT CONVERT(VARCHAR(1), 0x99) +GO + +SELECT CONVERT(VARCHAR(2), 0x999999) +GO + +SELECT CONVERT(VARBINARY(1), '™') +GO + SELECT CONVERT(VARCHAR(10), 0x80) GO -SELECT CONVERT(VARCHAR(10), 0xaaa) +-- 0x81 does not exist is empty in some encodings +SELECT CONVERT(VARCHAR(10), 0x81) GO SELECT CONVERT(VARCHAR(10), 0x330033) @@ -58,17 +73,13 @@ GO CREATE TABLE babel_1940_t2(a varchar(10) collate japanese_cs_as); GO -insert into babel_1940_t2 values ('ｳ'), ('C'), ('ﾊﾟ'), ('３'), ('c'), ('ｲ'), ('Ｃ'),('ﾊ'),('1'), -('ｱ'),('パ'), ('b'), ('2'), ('B'),('１'), ('Ａ'),('ア'),('A'), ('a'),('AbC'), ('aBc'); +INSERT INTO babel_1940_t2 VALUES ('a'), ('b'), ('™'), ('ƀ'), ('ä'); GO +-- Characters with no mapping transform to Ox3F or ? SELECT CONVERT(VARBINARY(10), a) FROM babel_1940_t2 GO -SELECT CONVERT(VARBINARY(10), CONVERT(VARCHAR(10), CONVERT(VARCHAR(10), a))) FROM babel_1940_t2 -GO - - DROP TABLE babel_1940_t2 GO From c05919def1e4f7e7e15d3fd93714598174945a54 Mon Sep 17 00:00:00 2001 From: Tanzeel Khan Date: Thu, 2 Nov 2023 14:39:50 +0000 Subject: [PATCH 07/28] use correct func to build typename Signed-off-by: Tanzeel Khan --- contrib/babelfishpg_common/src/encoding/mb/conv.c | 9 +++++++++ contrib/babelfishpg_tsql/src/pltsql_coerce.c | 5 +++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/contrib/babelfishpg_common/src/encoding/mb/conv.c b/contrib/babelfishpg_common/src/encoding/mb/conv.c index 7a4561fcc2..bd382fd76f 100644 --- a/contrib/babelfishpg_common/src/encoding/mb/conv.c +++ b/contrib/babelfishpg_common/src/encoding/mb/conv.c @@ -401,6 +401,15 @@ TsqlLocalToUtf(const unsigned char *iso, int len, unsigned char b3 = 0; unsigned char b4 = 0; + if (*iso == '\0') + { + /* 0x00 byte */ + *utf++ = *iso++; + encodedByteLen += 1; + l = 1; + continue; + } + if (!IS_HIGHBIT_SET(*iso)) { /* ASCII case is easy, assume it's one-to-one conversion */ diff --git a/contrib/babelfishpg_tsql/src/pltsql_coerce.c b/contrib/babelfishpg_tsql/src/pltsql_coerce.c index 97776a000b..e09b9fded3 100644 --- a/contrib/babelfishpg_tsql/src/pltsql_coerce.c +++ b/contrib/babelfishpg_tsql/src/pltsql_coerce.c @@ -1124,7 +1124,8 @@ tsql_coerce_string_literal_hook(ParseCallbackState *pcbstate, Oid targetTypeId, * when explicit cast is called */ - TypeName *varcharTypeName = makeTypeName("varchar"); + TypeName *varcharTypeName = makeTypeNameFromNameList(list_make2(makeString("sys"), + makeString("varchar"))); Node *result; Const *tempcon; @@ -1132,7 +1133,7 @@ tsql_coerce_string_literal_hook(ParseCallbackState *pcbstate, Oid targetTypeId, baseType = typeidType(baseTypeId); pfree(varcharTypeName); - tempcon = makeConst(TEXTOID, -1, + tempcon = makeConst(varcharTypeName->typeOid, -1, tsql_get_server_collation_oid_internal(false), -1, PointerGetDatum(cstring_to_text(value)), false, false); From 7f9da366077460ca67de2ec7c8f26dc35736ca1f Mon Sep 17 00:00:00 2001 From: Tanzeel Khan Date: Thu, 2 Nov 2023 15:22:00 +0000 Subject: [PATCH 08/28] empty commit rerun actions Signed-off-by: Tanzeel Khan From e033ba447e8e9e3f2fcf3b0f3622b1187e91057d Mon Sep 17 00:00:00 2001 From: Tanzeel Khan Date: Fri, 3 Nov 2023 08:55:15 +0000 Subject: [PATCH 09/28] release syscache tuples Signed-off-by: Tanzeel Khan --- contrib/babelfishpg_tsql/src/pltsql_coerce.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/contrib/babelfishpg_tsql/src/pltsql_coerce.c b/contrib/babelfishpg_tsql/src/pltsql_coerce.c index e09b9fded3..5b945cf993 100644 --- a/contrib/babelfishpg_tsql/src/pltsql_coerce.c +++ b/contrib/babelfishpg_tsql/src/pltsql_coerce.c @@ -1130,8 +1130,6 @@ tsql_coerce_string_literal_hook(ParseCallbackState *pcbstate, Oid targetTypeId, Const *tempcon; typenameTypeIdAndMod(NULL, (const TypeName *)varcharTypeName, &baseTypeId, &baseTypeMod); - baseType = typeidType(baseTypeId); - pfree(varcharTypeName); tempcon = makeConst(varcharTypeName->typeOid, -1, tsql_get_server_collation_oid_internal(false), @@ -1143,6 +1141,10 @@ tsql_coerce_string_literal_hook(ParseCallbackState *pcbstate, Oid targetTypeId, COERCION_EXPLICIT, COERCE_EXPLICIT_CAST, location); + + pfree(varcharTypeName); + ReleaseSysCache(baseType); + return result; } else From e948fd08eaf8559901a49e1d6105dad5c7d1f8f4 Mon Sep 17 00:00:00 2001 From: Tanzeel Khan Date: Fri, 3 Nov 2023 09:30:34 +0000 Subject: [PATCH 10/28] empty commit rerun actions post engine changes Signed-off-by: Tanzeel Khan From cdd4e3576d95d83ac6d687f12d807ce4cdf3d8ae Mon Sep 17 00:00:00 2001 From: Tanzeel Khan Date: Fri, 3 Nov 2023 10:03:10 +0000 Subject: [PATCH 11/28] rerun actions with empty commit Signed-off-by: Tanzeel Khan From 62729cf9f0bcb373c11f95d0e3e0009979fe9939 Mon Sep 17 00:00:00 2001 From: Tanzeel Khan Date: Fri, 3 Nov 2023 11:05:45 +0000 Subject: [PATCH 12/28] add few more tests Signed-off-by: Tanzeel Khan --- test/JDBC/expected/BABEL_1940.out | 16 ++++++++++++++++ test/JDBC/input/BABEL_1940.sql | 6 ++++++ 2 files changed, 22 insertions(+) diff --git a/test/JDBC/expected/BABEL_1940.out b/test/JDBC/expected/BABEL_1940.out index aefe36d534..6dc2c3d0b2 100644 --- a/test/JDBC/expected/BABEL_1940.out +++ b/test/JDBC/expected/BABEL_1940.out @@ -40,6 +40,22 @@ varbinary ~~END~~ +SELECT CAST('™' AS VARBINARY) +GO +~~START~~ +varbinary +99 +~~END~~ + + +SELECT CAST('™™™' AS VARBINARY(2)) +GO +~~START~~ +varbinary +9999 +~~END~~ + + SELECT CONVERT(VARCHAR(10), 0x80) GO ~~START~~ diff --git a/test/JDBC/input/BABEL_1940.sql b/test/JDBC/input/BABEL_1940.sql index 4714820233..dd559ea6d6 100644 --- a/test/JDBC/input/BABEL_1940.sql +++ b/test/JDBC/input/BABEL_1940.sql @@ -15,6 +15,12 @@ GO SELECT CONVERT(VARBINARY(1), '™') GO +SELECT CAST('™' AS VARBINARY) +GO + +SELECT CAST('™™™' AS VARBINARY(2)) +GO + SELECT CONVERT(VARCHAR(10), 0x80) GO From a193211437bb292ee981ac8dd1d3e618be89f7ac Mon Sep 17 00:00:00 2001 From: Tanzeel Khan Date: Tue, 7 Nov 2023 09:41:52 +0000 Subject: [PATCH 13/28] rerun actions with empty commit Signed-off-by: Tanzeel Khan From d16c1264908a08d8a61c97859e85ab34df435daa Mon Sep 17 00:00:00 2001 From: Tanzeel Khan Date: Tue, 7 Nov 2023 09:48:55 +0000 Subject: [PATCH 14/28] empty commit Signed-off-by: Tanzeel Khan From 5f4507cd84af29a8172398f1f8f67d67a114a33e Mon Sep 17 00:00:00 2001 From: Tanzeel Khan Date: Wed, 8 Nov 2023 15:07:03 +0000 Subject: [PATCH 15/28] reject null byte Signed-off-by: Tanzeel Khan --- .../babelfishpg_common/src/encoding/mb/conv.c | 9 ++--- contrib/babelfishpg_common/src/varbinary.c | 34 +++++++++++++++---- test/JDBC/expected/BABEL_1940.out | 7 ++-- 3 files changed, 33 insertions(+), 17 deletions(-) diff --git a/contrib/babelfishpg_common/src/encoding/mb/conv.c b/contrib/babelfishpg_common/src/encoding/mb/conv.c index bd382fd76f..087a408b0f 100644 --- a/contrib/babelfishpg_common/src/encoding/mb/conv.c +++ b/contrib/babelfishpg_common/src/encoding/mb/conv.c @@ -401,14 +401,9 @@ TsqlLocalToUtf(const unsigned char *iso, int len, unsigned char b3 = 0; unsigned char b4 = 0; + /* "break" cases all represent errors */ if (*iso == '\0') - { - /* 0x00 byte */ - *utf++ = *iso++; - encodedByteLen += 1; - l = 1; - continue; - } + break; if (!IS_HIGHBIT_SET(*iso)) { diff --git a/contrib/babelfishpg_common/src/varbinary.c b/contrib/babelfishpg_common/src/varbinary.c index d4c8972061..50ea0ab7ca 100644 --- a/contrib/babelfishpg_common/src/varbinary.c +++ b/contrib/babelfishpg_common/src/varbinary.c @@ -641,8 +641,18 @@ varcharvarbinary(PG_FUNCTION_ARGS) "varbinary is not allowed. Use the CONVERT function " "to run this query."))); - collInfo = lookup_collation_table(get_server_collation_oid_internal(false)); - encoded_data = encoding_conv_util(data, len, PG_UTF8, collInfo.enc, &encodedByteLen); + PG_TRY(); + { + collInfo = lookup_collation_table(get_server_collation_oid_internal(false)); + encoded_data = encoding_conv_util(data, len, PG_UTF8, collInfo.enc, &encodedByteLen); + } + PG_CATCH(); + { + ereport(ERROR, + (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("Failed to convert from data type varchar to varbinary"))); + } + PG_END_TRY(); /* * If typmod is -1 (or invalid), use the actual length @@ -723,11 +733,21 @@ varbinaryvarchar(PG_FUNCTION_ARGS) * Cast the entire input binary data if maxlen is invalid or supplied data * fits it */ - if (maxlen < 0 || len <= maxlen) - encoded_result = encoding_conv_util(data, len, collInfo.enc, PG_UTF8, &encodedByteLen); - /* Else truncate it */ - else - encoded_result = encoding_conv_util(data, maxlen, collInfo.enc, PG_UTF8, &encodedByteLen); + PG_TRY(); + { + if (maxlen < 0 || len <= maxlen) + encoded_result = encoding_conv_util(data, len, collInfo.enc, PG_UTF8, &encodedByteLen); + /* Else truncate it */ + else + encoded_result = encoding_conv_util(data, maxlen, collInfo.enc, PG_UTF8, &encodedByteLen); + } + PG_CATCH(); + { + ereport(ERROR, + (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("Failed to convert from data type varbinary to varchar"))); + } + PG_END_TRY(); result = (VarChar *) cstring_to_text_with_len(encoded_result, encodedByteLen); diff --git a/test/JDBC/expected/BABEL_1940.out b/test/JDBC/expected/BABEL_1940.out index 6dc2c3d0b2..080bc9c819 100644 --- a/test/JDBC/expected/BABEL_1940.out +++ b/test/JDBC/expected/BABEL_1940.out @@ -71,15 +71,16 @@ GO varchar ~~ERROR (Code: 33557097)~~ -~~ERROR (Message: character with byte sequence 0x81 in encoding "WIN1252" has no equivalent in encoding "UTF8")~~ +~~ERROR (Message: Failed to convert from data type varbinary to varchar)~~ SELECT CONVERT(VARCHAR(10), 0x330033) GO ~~START~~ varchar -3 -~~END~~ +~~ERROR (Code: 33557097)~~ + +~~ERROR (Message: Failed to convert from data type varbinary to varchar)~~ SELECT CONVERT(VARBINARY(10), 'ｳ') From b798d509d1e35226f070fa42d29a510fd6a92d20 Mon Sep 17 00:00:00 2001 From: Tanzeel Khan Date: Wed, 8 Nov 2023 17:28:17 +0000 Subject: [PATCH 16/28] empty commit to rerun tests Signed-off-by: Tanzeel Khan From c0b7cf5b0ddea8f4c1356f1f07a43527740f78a2 Mon Sep 17 00:00:00 2001 From: Tanzeel Khan Date: Wed, 8 Nov 2023 20:53:01 +0000 Subject: [PATCH 17/28] allow trailing null byte in cast to varchar Signed-off-by: Tanzeel Khan --- contrib/babelfishpg_common/src/varbinary.c | 14 ++++++++++---- test/JDBC/expected/BABEL_1940.out | 17 +++++++++++++++++ test/JDBC/expected/babel_datatype.out | 7 ++++--- test/JDBC/input/BABEL_1940.sql | 8 ++++++++ 4 files changed, 39 insertions(+), 7 deletions(-) diff --git a/contrib/babelfishpg_common/src/varbinary.c b/contrib/babelfishpg_common/src/varbinary.c index 50ea0ab7ca..ebe59fdbe5 100644 --- a/contrib/babelfishpg_common/src/varbinary.c +++ b/contrib/babelfishpg_common/src/varbinary.c @@ -727,17 +727,23 @@ varbinaryvarchar(PG_FUNCTION_ARGS) coll_info collInfo; int encodedByteLen; + /* + * Allow trailing null bytes + * Its safe since multi byte UTF-8 does not contain 0x00 + */ + while(len>0 && data[len-1] == '\0') + len -= 1; + collInfo = lookup_collation_table(get_server_collation_oid_internal(false)); - /* - * Cast the entire input binary data if maxlen is invalid or supplied data - * fits it + * Cast the entire input binary data if maxlen is + * invalid or supplied data fits it + * Else truncate it */ PG_TRY(); { if (maxlen < 0 || len <= maxlen) encoded_result = encoding_conv_util(data, len, collInfo.enc, PG_UTF8, &encodedByteLen); - /* Else truncate it */ else encoded_result = encoding_conv_util(data, maxlen, collInfo.enc, PG_UTF8, &encodedByteLen); } diff --git a/test/JDBC/expected/BABEL_1940.out b/test/JDBC/expected/BABEL_1940.out index 080bc9c819..35479fd7af 100644 --- a/test/JDBC/expected/BABEL_1940.out +++ b/test/JDBC/expected/BABEL_1940.out @@ -218,6 +218,23 @@ E4 ~~END~~ +-- Truncate trailing null bytes +SELECT CAST(CAST(0x616263 as BINARY(128)) as VARCHAR) +GO +~~START~~ +varchar +abc +~~END~~ + + +-- Block intermidiate null byte +SELECT CAST(CAST(0x610063 as BINARY(128)) as VARCHAR) +GO +~~ERROR (Code: 33557097)~~ + +~~ERROR (Message: Failed to convert from data type varbinary to varchar)~~ + + DROP TABLE babel_1940_t2 GO diff --git a/test/JDBC/expected/babel_datatype.out b/test/JDBC/expected/babel_datatype.out index 2d9bf5db06..cce059dcac 100644 --- a/test/JDBC/expected/babel_datatype.out +++ b/test/JDBC/expected/babel_datatype.out @@ -2202,9 +2202,10 @@ binary -- BABEL-1030 select cast(cast(cast('a' AS nvarchar(10)) as binary(2)) as nvarchar(2)); GO -~~ERROR (Code: 33557097)~~ - -~~ERROR (Message: invalid Unicode code point 0x0)~~ +~~START~~ +nvarchar +a +~~END~~ select cast(cast('ab' AS nvarchar(10)) as binary(2)); GO diff --git a/test/JDBC/input/BABEL_1940.sql b/test/JDBC/input/BABEL_1940.sql index dd559ea6d6..96a636e778 100644 --- a/test/JDBC/input/BABEL_1940.sql +++ b/test/JDBC/input/BABEL_1940.sql @@ -86,6 +86,14 @@ GO SELECT CONVERT(VARBINARY(10), a) FROM babel_1940_t2 GO +-- Truncate trailing null bytes +SELECT CAST(CAST(0x616263 as BINARY(128)) as VARCHAR) +GO + +-- Block intermidiate null byte +SELECT CAST(CAST(0x610063 as BINARY(128)) as VARCHAR) +GO + DROP TABLE babel_1940_t2 GO From 66d3e7f5b16739447a5bcfc2dbeeac6b4a6400b2 Mon Sep 17 00:00:00 2001 From: Tanzeel Khan Date: Thu, 9 Nov 2023 05:18:33 +0000 Subject: [PATCH 18/28] remove varchar varbinary cast from geography/geometry cast Signed-off-by: Tanzeel Khan --- contrib/babelfishpg_common/sql/geography.sql | 2 +- contrib/babelfishpg_common/sql/geometry.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/babelfishpg_common/sql/geography.sql b/contrib/babelfishpg_common/sql/geography.sql index 1da875688e..eed4d51767 100644 --- a/contrib/babelfishpg_common/sql/geography.sql +++ b/contrib/babelfishpg_common/sql/geography.sql @@ -198,7 +198,7 @@ CREATE OR REPLACE FUNCTION sys.GEOGRAPHY(sys.bbf_binary) DECLARE varBin sys.bbf_varbinary; BEGIN - varBin := (SELECT CAST (CAST ($1 AS sys.VARCHAR) AS sys.bbf_varbinary)); + varBin := (SELECT CAST (CAST ($1 AS sys.bbf_varbinary))); -- Call the underlying function after preprocessing RETURN (SELECT sys.GEOGRAPHY(varBin)); END; diff --git a/contrib/babelfishpg_common/sql/geometry.sql b/contrib/babelfishpg_common/sql/geometry.sql index f87fe16744..62ebb9da42 100644 --- a/contrib/babelfishpg_common/sql/geometry.sql +++ b/contrib/babelfishpg_common/sql/geometry.sql @@ -219,7 +219,7 @@ CREATE OR REPLACE FUNCTION sys.GEOMETRY(sys.bbf_binary) DECLARE varBin sys.bbf_varbinary; BEGIN - varBin := (SELECT CAST (CAST ($1 AS sys.VARCHAR) AS sys.bbf_varbinary)); + varBin := (SELECT CAST (CAST ($1 AS sys.bbf_varbinary))); -- Call the underlying function after preprocessing RETURN (SELECT sys.GEOMETRY(varBin)); END; From cdf2397bf1c17f88e6f2cc049e0bbdd64723f779 Mon Sep 17 00:00:00 2001 From: Tanzeel Khan Date: Thu, 9 Nov 2023 06:12:57 +0000 Subject: [PATCH 19/28] push fix Signed-off-by: Tanzeel Khan --- contrib/babelfishpg_common/sql/geography.sql | 2 +- contrib/babelfishpg_common/sql/geometry.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/babelfishpg_common/sql/geography.sql b/contrib/babelfishpg_common/sql/geography.sql index eed4d51767..c986aa04b9 100644 --- a/contrib/babelfishpg_common/sql/geography.sql +++ b/contrib/babelfishpg_common/sql/geography.sql @@ -198,7 +198,7 @@ CREATE OR REPLACE FUNCTION sys.GEOGRAPHY(sys.bbf_binary) DECLARE varBin sys.bbf_varbinary; BEGIN - varBin := (SELECT CAST (CAST ($1 AS sys.bbf_varbinary))); + varBin := (SELECT CAST ($1 AS sys.bbf_varbinary)); -- Call the underlying function after preprocessing RETURN (SELECT sys.GEOGRAPHY(varBin)); END; diff --git a/contrib/babelfishpg_common/sql/geometry.sql b/contrib/babelfishpg_common/sql/geometry.sql index 62ebb9da42..c88d40efe9 100644 --- a/contrib/babelfishpg_common/sql/geometry.sql +++ b/contrib/babelfishpg_common/sql/geometry.sql @@ -219,7 +219,7 @@ CREATE OR REPLACE FUNCTION sys.GEOMETRY(sys.bbf_binary) DECLARE varBin sys.bbf_varbinary; BEGIN - varBin := (SELECT CAST (CAST ($1 AS sys.bbf_varbinary))); + varBin := (SELECT CAST ($1 AS sys.bbf_varbinary)); -- Call the underlying function after preprocessing RETURN (SELECT sys.GEOMETRY(varBin)); END; From 78b5e53cb22f2163421746e07329e4b040479b1e Mon Sep 17 00:00:00 2001 From: Tanzeel Khan Date: Thu, 9 Nov 2023 07:30:43 +0000 Subject: [PATCH 20/28] try fix for geometry test failures Signed-off-by: Tanzeel Khan --- contrib/babelfishpg_common/sql/geography.sql | 2 +- contrib/babelfishpg_common/sql/geometry.sql | 2 +- .../sql/upgrades/spatial_types--3.2.0--3.3.0.sql | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/contrib/babelfishpg_common/sql/geography.sql b/contrib/babelfishpg_common/sql/geography.sql index c986aa04b9..1ae7ec0345 100644 --- a/contrib/babelfishpg_common/sql/geography.sql +++ b/contrib/babelfishpg_common/sql/geography.sql @@ -198,7 +198,7 @@ CREATE OR REPLACE FUNCTION sys.GEOGRAPHY(sys.bbf_binary) DECLARE varBin sys.bbf_varbinary; BEGIN - varBin := (SELECT CAST ($1 AS sys.bbf_varbinary)); + varBin := $1; -- Call the underlying function after preprocessing RETURN (SELECT sys.GEOGRAPHY(varBin)); END; diff --git a/contrib/babelfishpg_common/sql/geometry.sql b/contrib/babelfishpg_common/sql/geometry.sql index c88d40efe9..9a3d7dee4e 100644 --- a/contrib/babelfishpg_common/sql/geometry.sql +++ b/contrib/babelfishpg_common/sql/geometry.sql @@ -219,7 +219,7 @@ CREATE OR REPLACE FUNCTION sys.GEOMETRY(sys.bbf_binary) DECLARE varBin sys.bbf_varbinary; BEGIN - varBin := (SELECT CAST ($1 AS sys.bbf_varbinary)); + varBin := $1; -- Call the underlying function after preprocessing RETURN (SELECT sys.GEOMETRY(varBin)); END; diff --git a/contrib/babelfishpg_common/sql/upgrades/spatial_types--3.2.0--3.3.0.sql b/contrib/babelfishpg_common/sql/upgrades/spatial_types--3.2.0--3.3.0.sql index f4b1fdcfec..76a2885572 100644 --- a/contrib/babelfishpg_common/sql/upgrades/spatial_types--3.2.0--3.3.0.sql +++ b/contrib/babelfishpg_common/sql/upgrades/spatial_types--3.2.0--3.3.0.sql @@ -223,7 +223,7 @@ CREATE OR REPLACE FUNCTION sys.GEOMETRY(sys.bbf_binary) DECLARE varBin sys.bbf_varbinary; BEGIN - varBin := (SELECT CAST (CAST ($1 AS sys.VARCHAR) AS sys.bbf_varbinary)); + varBin := $1; -- Call the underlying function after preprocessing RETURN (SELECT sys.GEOMETRY(varBin)); END; @@ -591,7 +591,7 @@ CREATE OR REPLACE FUNCTION sys.GEOGRAPHY(sys.bbf_binary) DECLARE varBin sys.bbf_varbinary; BEGIN - varBin := (SELECT CAST (CAST ($1 AS sys.VARCHAR) AS sys.bbf_varbinary)); + varBin := $1; -- Call the underlying function after preprocessing RETURN (SELECT sys.GEOGRAPHY(varBin)); END; From c3a3b991a93aab2bb31e7dd003b4472ed09f9426 Mon Sep 17 00:00:00 2001 From: Tanzeel Khan Date: Thu, 9 Nov 2023 12:35:53 +0000 Subject: [PATCH 21/28] add cast for binary to var binary Signed-off-by: Tanzeel Khan --- contrib/babelfishpg_common/sql/binary.sql | 3 +++ contrib/babelfishpg_common/sql/geography.sql | 2 +- contrib/babelfishpg_common/sql/geometry.sql | 2 +- .../sql/upgrades/babelfish_common_helper--3.2.0--3.3.0.sql | 4 ++++ .../sql/upgrades/spatial_types--3.2.0--3.3.0.sql | 4 ++-- 5 files changed, 11 insertions(+), 4 deletions(-) diff --git a/contrib/babelfishpg_common/sql/binary.sql b/contrib/babelfishpg_common/sql/binary.sql index 58f9037dfb..627b821c8b 100644 --- a/contrib/babelfishpg_common/sql/binary.sql +++ b/contrib/babelfishpg_common/sql/binary.sql @@ -317,3 +317,6 @@ CREATE OPERATOR sys.= ( alter OPERATOR family bbf_varbinary_ops USING btree add OPERATOR 3 sys.= (sys.bbf_varbinary, sys.bbf_binary), FUNCTION 1 sys.bbf_varbinary_binary_cmp(sys.bbf_varbinary, sys.bbf_binary); + +CREATE CAST (sys.BBF_BINARY AS sys.BBF_VARBINARY) + WITHOUT FUNCTION AS IMPLICIT; diff --git a/contrib/babelfishpg_common/sql/geography.sql b/contrib/babelfishpg_common/sql/geography.sql index 1ae7ec0345..c986aa04b9 100644 --- a/contrib/babelfishpg_common/sql/geography.sql +++ b/contrib/babelfishpg_common/sql/geography.sql @@ -198,7 +198,7 @@ CREATE OR REPLACE FUNCTION sys.GEOGRAPHY(sys.bbf_binary) DECLARE varBin sys.bbf_varbinary; BEGIN - varBin := $1; + varBin := (SELECT CAST ($1 AS sys.bbf_varbinary)); -- Call the underlying function after preprocessing RETURN (SELECT sys.GEOGRAPHY(varBin)); END; diff --git a/contrib/babelfishpg_common/sql/geometry.sql b/contrib/babelfishpg_common/sql/geometry.sql index 9a3d7dee4e..c88d40efe9 100644 --- a/contrib/babelfishpg_common/sql/geometry.sql +++ b/contrib/babelfishpg_common/sql/geometry.sql @@ -219,7 +219,7 @@ CREATE OR REPLACE FUNCTION sys.GEOMETRY(sys.bbf_binary) DECLARE varBin sys.bbf_varbinary; BEGIN - varBin := $1; + varBin := (SELECT CAST ($1 AS sys.bbf_varbinary)); -- Call the underlying function after preprocessing RETURN (SELECT sys.GEOMETRY(varBin)); END; diff --git a/contrib/babelfishpg_common/sql/upgrades/babelfish_common_helper--3.2.0--3.3.0.sql b/contrib/babelfishpg_common/sql/upgrades/babelfish_common_helper--3.2.0--3.3.0.sql index 8ade088b86..f9ef128db7 100644 --- a/contrib/babelfishpg_common/sql/upgrades/babelfish_common_helper--3.2.0--3.3.0.sql +++ b/contrib/babelfishpg_common/sql/upgrades/babelfish_common_helper--3.2.0--3.3.0.sql @@ -122,5 +122,9 @@ CREATE OR REPLACE AGGREGATE sys.min(sys.SMALLDATETIME) parallel = safe ); +-- binary +CREATE CAST (sys.BBF_BINARY AS sys.BBF_VARBINARY) + WITHOUT FUNCTION AS IMPLICIT; + -- Reset search_path to not affect any subsequent scripts SELECT set_config('search_path', trim(leading 'sys, ' from current_setting('search_path')), false); diff --git a/contrib/babelfishpg_common/sql/upgrades/spatial_types--3.2.0--3.3.0.sql b/contrib/babelfishpg_common/sql/upgrades/spatial_types--3.2.0--3.3.0.sql index 76a2885572..2227034d7f 100644 --- a/contrib/babelfishpg_common/sql/upgrades/spatial_types--3.2.0--3.3.0.sql +++ b/contrib/babelfishpg_common/sql/upgrades/spatial_types--3.2.0--3.3.0.sql @@ -223,7 +223,7 @@ CREATE OR REPLACE FUNCTION sys.GEOMETRY(sys.bbf_binary) DECLARE varBin sys.bbf_varbinary; BEGIN - varBin := $1; + varBin := (SELECT CAST ($1 AS sys.bbf_varbinary)); -- Call the underlying function after preprocessing RETURN (SELECT sys.GEOMETRY(varBin)); END; @@ -591,7 +591,7 @@ CREATE OR REPLACE FUNCTION sys.GEOGRAPHY(sys.bbf_binary) DECLARE varBin sys.bbf_varbinary; BEGIN - varBin := $1; + varBin := (SELECT CAST ($1 AS sys.bbf_varbinary)); -- Call the underlying function after preprocessing RETURN (SELECT sys.GEOGRAPHY(varBin)); END; From 0d8bbf69954a827e9038d069358ccf8db3316e42 Mon Sep 17 00:00:00 2001 From: Tanzeel Khan Date: Thu, 9 Nov 2023 13:16:17 +0000 Subject: [PATCH 22/28] empty commit Signed-off-by: Tanzeel Khan From 25fda371226e6873483ee1b635a883d48b3716f0 Mon Sep 17 00:00:00 2001 From: Tanzeel Khan Date: Thu, 9 Nov 2023 20:08:58 +0000 Subject: [PATCH 23/28] fix typmod for binary data type Signed-off-by: Tanzeel Khan --- contrib/babelfishpg_common/sql/binary.sql | 17 ++++- .../babelfish_common_helper--3.2.0--3.3.0.sql | 16 ++++- contrib/babelfishpg_common/src/varbinary.c | 25 ++++++- test/JDBC/expected/BABEL_1940.out | 68 ++++++++++++++++++- test/JDBC/expected/babel_datatype.out | 6 +- test/JDBC/input/BABEL_1940.sql | 32 ++++++++- 6 files changed, 152 insertions(+), 12 deletions(-) diff --git a/contrib/babelfishpg_common/sql/binary.sql b/contrib/babelfishpg_common/sql/binary.sql index 627b821c8b..b0472300d7 100644 --- a/contrib/babelfishpg_common/sql/binary.sql +++ b/contrib/babelfishpg_common/sql/binary.sql @@ -46,6 +46,21 @@ CREATE TYPE sys.BBF_BINARY ( COLLATABLE = false ); +CREATE OR REPLACE FUNCTION sys.bbfbinary(sys.BBF_BINARY, integer, boolean) +RETURNS sys.BBF_BINARY +AS 'babelfishpg_common', 'binary' +LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; + +-- typmod cast for sys.BBF_VARBINARY +CREATE CAST (sys.BBF_BINARY AS sys.BBF_BINARY) +WITH FUNCTION sys.bbfbinary(sys.BBF_BINARY, integer, BOOLEAN) AS ASSIGNMENT; + +CREATE CAST (sys.BBF_BINARY AS sys.BBF_VARBINARY) +WITHOUT FUNCTION AS IMPLICIT; + +CREATE CAST (sys.BBF_VARBINARY AS sys.BBF_BINARY) +WITHOUT FUNCTION AS IMPLICIT; + -- casting functions for sys.BINARY CREATE OR REPLACE FUNCTION sys.varcharbinary(sys.VARCHAR, integer, boolean) RETURNS sys.BBF_BINARY @@ -318,5 +333,3 @@ alter OPERATOR family bbf_varbinary_ops USING btree add OPERATOR 3 sys.= (sys.bbf_varbinary, sys.bbf_binary), FUNCTION 1 sys.bbf_varbinary_binary_cmp(sys.bbf_varbinary, sys.bbf_binary); -CREATE CAST (sys.BBF_BINARY AS sys.BBF_VARBINARY) - WITHOUT FUNCTION AS IMPLICIT; diff --git a/contrib/babelfishpg_common/sql/upgrades/babelfish_common_helper--3.2.0--3.3.0.sql b/contrib/babelfishpg_common/sql/upgrades/babelfish_common_helper--3.2.0--3.3.0.sql index f9ef128db7..7f485ea43b 100644 --- a/contrib/babelfishpg_common/sql/upgrades/babelfish_common_helper--3.2.0--3.3.0.sql +++ b/contrib/babelfishpg_common/sql/upgrades/babelfish_common_helper--3.2.0--3.3.0.sql @@ -122,9 +122,21 @@ CREATE OR REPLACE AGGREGATE sys.min(sys.SMALLDATETIME) parallel = safe ); --- binary +CREATE OR REPLACE FUNCTION sys.bbfbinary(sys.BBF_BINARY, integer, boolean) +RETURNS sys.BBF_BINARY +AS 'babelfishpg_common', 'binary' +LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; + +-- typmod cast for sys.BBF_VARBINARY +CREATE CAST (sys.BBF_BINARY AS sys.BBF_BINARY) +WITH FUNCTION sys.bbfbinary(sys.BBF_BINARY, integer, BOOLEAN) AS ASSIGNMENT; + +-- binary varbinary cast CREATE CAST (sys.BBF_BINARY AS sys.BBF_VARBINARY) - WITHOUT FUNCTION AS IMPLICIT; +WITHOUT FUNCTION AS IMPLICIT; + +CREATE CAST (sys.BBF_VARBINARY AS sys.BBF_BINARY) +WITHOUT FUNCTION AS IMPLICIT; -- Reset search_path to not affect any subsequent scripts SELECT set_config('search_path', trim(leading 'sys, ' from current_setting('search_path')), false); diff --git a/contrib/babelfishpg_common/src/varbinary.c b/contrib/babelfishpg_common/src/varbinary.c index ebe59fdbe5..7766c4ad80 100644 --- a/contrib/babelfishpg_common/src/varbinary.c +++ b/contrib/babelfishpg_common/src/varbinary.c @@ -633,6 +633,7 @@ varcharvarbinary(PG_FUNCTION_ARGS) bytea *result; coll_info collInfo; int encodedByteLen; + MemoryContext ccxt = CurrentMemoryContext; if (!isExplicit) ereport(ERROR, @@ -648,9 +649,18 @@ varcharvarbinary(PG_FUNCTION_ARGS) } PG_CATCH(); { + MemoryContext ectx; + ErrorData *errorData; + + ectx = MemoryContextSwitchTo(ccxt); + errorData = CopyErrorData(); + FlushErrorState(); + MemoryContextSwitchTo(ectx); + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), - errmsg("Failed to convert from data type varchar to varbinary"))); + errmsg("Failed to convert from data type varchar to varbinary, %s", + errorData->message))); } PG_END_TRY(); @@ -726,6 +736,7 @@ varbinaryvarchar(PG_FUNCTION_ARGS) int32 maxlen = typmod - VARHDRSZ; coll_info collInfo; int encodedByteLen; + MemoryContext ccxt; /* * Allow trailing null bytes @@ -735,6 +746,7 @@ varbinaryvarchar(PG_FUNCTION_ARGS) len -= 1; collInfo = lookup_collation_table(get_server_collation_oid_internal(false)); + ccxt = CurrentMemoryContext; /* * Cast the entire input binary data if maxlen is * invalid or supplied data fits it @@ -749,9 +761,18 @@ varbinaryvarchar(PG_FUNCTION_ARGS) } PG_CATCH(); { + MemoryContext ectx; + ErrorData *errorData; + + ectx = MemoryContextSwitchTo(ccxt); + errorData = CopyErrorData(); + FlushErrorState(); + MemoryContextSwitchTo(ectx); + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), - errmsg("Failed to convert from data type varbinary to varchar"))); + errmsg("Failed to convert from data type varbinary to varchar, %s", + errorData->message))); } PG_END_TRY(); diff --git a/test/JDBC/expected/BABEL_1940.out b/test/JDBC/expected/BABEL_1940.out index 35479fd7af..4b3744c6f9 100644 --- a/test/JDBC/expected/BABEL_1940.out +++ b/test/JDBC/expected/BABEL_1940.out @@ -1,5 +1,13 @@ -- Test is only valid when default server encoding is WIN1252 +SELECT CONVERT(VARCHAR(MAX), 0x123456789) +GO +~~START~~ +varchar +#Eg‰ +~~END~~ + + SELECT CONVERT(VARCHAR(10), 0x123456789) GO ~~START~~ @@ -71,7 +79,7 @@ GO varchar ~~ERROR (Code: 33557097)~~ -~~ERROR (Message: Failed to convert from data type varbinary to varchar)~~ +~~ERROR (Message: Failed to convert from data type varbinary to varchar, character with byte sequence 0x81 in encoding "WIN1252" has no equivalent in encoding "UTF8")~~ SELECT CONVERT(VARCHAR(10), 0x330033) @@ -80,7 +88,7 @@ GO varchar ~~ERROR (Code: 33557097)~~ -~~ERROR (Message: Failed to convert from data type varbinary to varchar)~~ +~~ERROR (Message: Failed to convert from data type varbinary to varchar, invalid byte sequence for encoding "WIN1252": 0x00)~~ SELECT CONVERT(VARBINARY(10), 'ｳ') @@ -232,7 +240,7 @@ SELECT CAST(CAST(0x610063 as BINARY(128)) as VARCHAR) GO ~~ERROR (Code: 33557097)~~ -~~ERROR (Message: Failed to convert from data type varbinary to varchar)~~ +~~ERROR (Message: Failed to convert from data type varbinary to varchar, invalid byte sequence for encoding "WIN1252": 0x00)~~ DROP TABLE babel_1940_t2 @@ -240,3 +248,57 @@ GO DROP TABLE babel_1940_t1 GO + +-- Tests for typmod binary data type +SELECT CAST(CAST(0x1234567891234567891234567891234567891234567891234567891234567 AS BINARY) AS VARBINARY) +GO +~~START~~ +varbinary +012345678912345678912345678912345678912345678912345678912345 +~~END~~ + + +SELECT DATALENGTH(CAST(0X6162636465 AS BINARY(3))) +GO +~~START~~ +int +3 +~~END~~ + + +SELECT DATALENGTH(CAST(0X61 AS BINARY(3))) +GO +~~START~~ +int +3 +~~END~~ + + +CREATE TABLE babel_1940_t3 (id BINARY(2)) +GO + +-- Implicit truncation should fail +INSERT INTO babel_1940_t3 VALUES (CAST(0x61 AS BINARY(3))) +GO +~~ERROR (Code: 33557097)~~ + +~~ERROR (Message: String or binary data would be truncated. +The statement has been terminated.)~~ + + +-- Implicit appending should work +INSERT INTO babel_1940_t3 VALUES (CAST(0x61 AS BINARY(1))) +GO +~~ROW COUNT: 1~~ + + +SELECT DATALENGTH(id) FROM babel_1940_t3 +GO +~~START~~ +int +2 +~~END~~ + + +DROP TABLE babel_1940_t3 +GO diff --git a/test/JDBC/expected/babel_datatype.out b/test/JDBC/expected/babel_datatype.out index cce059dcac..d571a2b4c4 100644 --- a/test/JDBC/expected/babel_datatype.out +++ b/test/JDBC/expected/babel_datatype.out @@ -2279,14 +2279,16 @@ GO insert into testing6 values (cast('ab' as varbinary)); GO -~~ROW COUNT: 1~~ +~~ERROR (Code: 33557097)~~ + +~~ERROR (Message: String or binary data would be truncated. +The statement has been terminated.)~~ select * from testing6; GO ~~START~~ binary 61 -61 ~~END~~ diff --git a/test/JDBC/input/BABEL_1940.sql b/test/JDBC/input/BABEL_1940.sql index 96a636e778..65c77ca83e 100644 --- a/test/JDBC/input/BABEL_1940.sql +++ b/test/JDBC/input/BABEL_1940.sql @@ -1,5 +1,8 @@ -- Test is only valid when default server encoding is WIN1252 +SELECT CONVERT(VARCHAR(MAX), 0x123456789) +GO + SELECT CONVERT(VARCHAR(10), 0x123456789) GO @@ -98,4 +101,31 @@ DROP TABLE babel_1940_t2 GO DROP TABLE babel_1940_t1 -GO \ No newline at end of file +GO + +-- Tests for typmod binary data type +SELECT CAST(CAST(0x1234567891234567891234567891234567891234567891234567891234567 AS BINARY) AS VARBINARY) +GO + +SELECT DATALENGTH(CAST(0X6162636465 AS BINARY(3))) +GO + +SELECT DATALENGTH(CAST(0X61 AS BINARY(3))) +GO + +CREATE TABLE babel_1940_t3 (id BINARY(2)) +GO + +-- Implicit truncation should fail +INSERT INTO babel_1940_t3 VALUES (CAST(0x61 AS BINARY(3))) +GO + +-- Implicit appending should work +INSERT INTO babel_1940_t3 VALUES (CAST(0x61 AS BINARY(1))) +GO + +SELECT DATALENGTH(id) FROM babel_1940_t3 +GO + +DROP TABLE babel_1940_t3 +GO From b768e8289334203d01c7f692beb960979227f10f Mon Sep 17 00:00:00 2001 From: Tanzeel Khan Date: Thu, 9 Nov 2023 20:59:54 +0000 Subject: [PATCH 24/28] remove typmod fix for now Signed-off-by: Tanzeel Khan --- contrib/babelfishpg_common/sql/binary.sql | 10 --- .../babelfish_common_helper--3.2.0--3.3.0.sql | 9 --- test/JDBC/expected/BABEL_1940.out | 68 ++++--------------- test/JDBC/input/BABEL_1940.sql | 33 ++------- 4 files changed, 22 insertions(+), 98 deletions(-) diff --git a/contrib/babelfishpg_common/sql/binary.sql b/contrib/babelfishpg_common/sql/binary.sql index b0472300d7..98e0b425e4 100644 --- a/contrib/babelfishpg_common/sql/binary.sql +++ b/contrib/babelfishpg_common/sql/binary.sql @@ -46,15 +46,6 @@ CREATE TYPE sys.BBF_BINARY ( COLLATABLE = false ); -CREATE OR REPLACE FUNCTION sys.bbfbinary(sys.BBF_BINARY, integer, boolean) -RETURNS sys.BBF_BINARY -AS 'babelfishpg_common', 'binary' -LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; - --- typmod cast for sys.BBF_VARBINARY -CREATE CAST (sys.BBF_BINARY AS sys.BBF_BINARY) -WITH FUNCTION sys.bbfbinary(sys.BBF_BINARY, integer, BOOLEAN) AS ASSIGNMENT; - CREATE CAST (sys.BBF_BINARY AS sys.BBF_VARBINARY) WITHOUT FUNCTION AS IMPLICIT; @@ -332,4 +323,3 @@ CREATE OPERATOR sys.= ( alter OPERATOR family bbf_varbinary_ops USING btree add OPERATOR 3 sys.= (sys.bbf_varbinary, sys.bbf_binary), FUNCTION 1 sys.bbf_varbinary_binary_cmp(sys.bbf_varbinary, sys.bbf_binary); - diff --git a/contrib/babelfishpg_common/sql/upgrades/babelfish_common_helper--3.2.0--3.3.0.sql b/contrib/babelfishpg_common/sql/upgrades/babelfish_common_helper--3.2.0--3.3.0.sql index 7f485ea43b..e60fc4a681 100644 --- a/contrib/babelfishpg_common/sql/upgrades/babelfish_common_helper--3.2.0--3.3.0.sql +++ b/contrib/babelfishpg_common/sql/upgrades/babelfish_common_helper--3.2.0--3.3.0.sql @@ -122,15 +122,6 @@ CREATE OR REPLACE AGGREGATE sys.min(sys.SMALLDATETIME) parallel = safe ); -CREATE OR REPLACE FUNCTION sys.bbfbinary(sys.BBF_BINARY, integer, boolean) -RETURNS sys.BBF_BINARY -AS 'babelfishpg_common', 'binary' -LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; - --- typmod cast for sys.BBF_VARBINARY -CREATE CAST (sys.BBF_BINARY AS sys.BBF_BINARY) -WITH FUNCTION sys.bbfbinary(sys.BBF_BINARY, integer, BOOLEAN) AS ASSIGNMENT; - -- binary varbinary cast CREATE CAST (sys.BBF_BINARY AS sys.BBF_VARBINARY) WITHOUT FUNCTION AS IMPLICIT; diff --git a/test/JDBC/expected/BABEL_1940.out b/test/JDBC/expected/BABEL_1940.out index 4b3744c6f9..54e701cb15 100644 --- a/test/JDBC/expected/BABEL_1940.out +++ b/test/JDBC/expected/BABEL_1940.out @@ -208,6 +208,20 @@ varchar CREATE TABLE babel_1940_t2(a varchar(10) collate japanese_cs_as); GO +-- only null bytes becomes empty string since we remove trailing nulls +INSERT INTO babel_1940_t2 VALUES (CAST (0x00 AS VARCHAR)) +GO +~~ROW COUNT: 1~~ + + +SELECT * FROM babel_1940_t2 WHERE a = ''; +GO +~~START~~ +varchar + +~~END~~ + + INSERT INTO babel_1940_t2 VALUES ('a'), ('b'), ('™'), ('ƀ'), ('ä'); GO ~~ROW COUNT: 5~~ @@ -218,6 +232,7 @@ SELECT CONVERT(VARBINARY(10), a) FROM babel_1940_t2 GO ~~START~~ varbinary + 61 62 99 @@ -249,56 +264,3 @@ GO DROP TABLE babel_1940_t1 GO --- Tests for typmod binary data type -SELECT CAST(CAST(0x1234567891234567891234567891234567891234567891234567891234567 AS BINARY) AS VARBINARY) -GO -~~START~~ -varbinary -012345678912345678912345678912345678912345678912345678912345 -~~END~~ - - -SELECT DATALENGTH(CAST(0X6162636465 AS BINARY(3))) -GO -~~START~~ -int -3 -~~END~~ - - -SELECT DATALENGTH(CAST(0X61 AS BINARY(3))) -GO -~~START~~ -int -3 -~~END~~ - - -CREATE TABLE babel_1940_t3 (id BINARY(2)) -GO - --- Implicit truncation should fail -INSERT INTO babel_1940_t3 VALUES (CAST(0x61 AS BINARY(3))) -GO -~~ERROR (Code: 33557097)~~ - -~~ERROR (Message: String or binary data would be truncated. -The statement has been terminated.)~~ - - --- Implicit appending should work -INSERT INTO babel_1940_t3 VALUES (CAST(0x61 AS BINARY(1))) -GO -~~ROW COUNT: 1~~ - - -SELECT DATALENGTH(id) FROM babel_1940_t3 -GO -~~START~~ -int -2 -~~END~~ - - -DROP TABLE babel_1940_t3 -GO diff --git a/test/JDBC/input/BABEL_1940.sql b/test/JDBC/input/BABEL_1940.sql index 65c77ca83e..d547a7b236 100644 --- a/test/JDBC/input/BABEL_1940.sql +++ b/test/JDBC/input/BABEL_1940.sql @@ -82,6 +82,13 @@ GO CREATE TABLE babel_1940_t2(a varchar(10) collate japanese_cs_as); GO +-- only null bytes becomes empty string since we remove trailing nulls +INSERT INTO babel_1940_t2 VALUES (CAST (0x00 AS VARCHAR)) +GO + +SELECT * FROM babel_1940_t2 WHERE a = ''; +GO + INSERT INTO babel_1940_t2 VALUES ('a'), ('b'), ('™'), ('ƀ'), ('ä'); GO @@ -103,29 +110,3 @@ GO DROP TABLE babel_1940_t1 GO --- Tests for typmod binary data type -SELECT CAST(CAST(0x1234567891234567891234567891234567891234567891234567891234567 AS BINARY) AS VARBINARY) -GO - -SELECT DATALENGTH(CAST(0X6162636465 AS BINARY(3))) -GO - -SELECT DATALENGTH(CAST(0X61 AS BINARY(3))) -GO - -CREATE TABLE babel_1940_t3 (id BINARY(2)) -GO - --- Implicit truncation should fail -INSERT INTO babel_1940_t3 VALUES (CAST(0x61 AS BINARY(3))) -GO - --- Implicit appending should work -INSERT INTO babel_1940_t3 VALUES (CAST(0x61 AS BINARY(1))) -GO - -SELECT DATALENGTH(id) FROM babel_1940_t3 -GO - -DROP TABLE babel_1940_t3 -GO From d944eb046996d5f4b6fc9ebd4a599d699b734a7f Mon Sep 17 00:00:00 2001 From: Tanzeel Khan Date: Thu, 9 Nov 2023 21:27:24 +0000 Subject: [PATCH 25/28] push correct test out file Signed-off-by: Tanzeel Khan --- test/JDBC/expected/babel_datatype.out | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/test/JDBC/expected/babel_datatype.out b/test/JDBC/expected/babel_datatype.out index d571a2b4c4..cce059dcac 100644 --- a/test/JDBC/expected/babel_datatype.out +++ b/test/JDBC/expected/babel_datatype.out @@ -2279,16 +2279,14 @@ GO insert into testing6 values (cast('ab' as varbinary)); GO -~~ERROR (Code: 33557097)~~ - -~~ERROR (Message: String or binary data would be truncated. -The statement has been terminated.)~~ +~~ROW COUNT: 1~~ select * from testing6; GO ~~START~~ binary 61 +61 ~~END~~ From f2e7f81ae15aebcbdc71dad643359cb5c4ba14d9 Mon Sep 17 00:00:00 2001 From: Tanzeel Khan Date: Fri, 10 Nov 2023 07:05:26 +0000 Subject: [PATCH 26/28] just comments Signed-off-by: Tanzeel Khan --- contrib/babelfishpg_common/src/varbinary.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/contrib/babelfishpg_common/src/varbinary.c b/contrib/babelfishpg_common/src/varbinary.c index 7766c4ad80..a05aa2605b 100644 --- a/contrib/babelfishpg_common/src/varbinary.c +++ b/contrib/babelfishpg_common/src/varbinary.c @@ -741,6 +741,10 @@ varbinaryvarchar(PG_FUNCTION_ARGS) /* * Allow trailing null bytes * Its safe since multi byte UTF-8 does not contain 0x00 + * This is needed since we implicity add trailing zeroes to + * binary type if input is less than binary(n) + * ex: CAST(CAST('a' AS BINARY(10)) AS VARCHAR) should work + * and not fail because of null byte */ while(len>0 && data[len-1] == '\0') len -= 1; From 278c0d5c5a061436d11ad7b09d09275eb0d04777 Mon Sep 17 00:00:00 2001 From: Tanzeel Khan Date: Fri, 10 Nov 2023 07:36:18 +0000 Subject: [PATCH 27/28] add some varbinary binary test case Signed-off-by: Tanzeel Khan --- test/JDBC/expected/BABEL_1940.out | 31 +++++++++++++++++++++++++++++++ test/JDBC/input/BABEL_1940.sql | 11 +++++++++++ 2 files changed, 42 insertions(+) diff --git a/test/JDBC/expected/BABEL_1940.out b/test/JDBC/expected/BABEL_1940.out index 54e701cb15..799aa22457 100644 --- a/test/JDBC/expected/BABEL_1940.out +++ b/test/JDBC/expected/BABEL_1940.out @@ -264,3 +264,34 @@ GO DROP TABLE babel_1940_t1 GO +SELECT CAST(CAST(0x61 AS VARBINARY) AS BINARY(3)) +GO +~~START~~ +binary +610000 +~~END~~ + + +SELECT CAST(CAST(0x61 AS BINARY(3)) AS VARBINARY(2)) +GO +~~START~~ +varbinary +61 +~~END~~ + + +SELECT CAST(CAST(0x6161616161 AS BINARY(3)) AS VARBINARY(2)) +GO +~~START~~ +varbinary +6161 +~~END~~ + + +SELECT CAST(CAST(0x616263646566 AS VARBINARY(5)) AS BINARY(3)) +GO +~~START~~ +binary +616263 +~~END~~ + diff --git a/test/JDBC/input/BABEL_1940.sql b/test/JDBC/input/BABEL_1940.sql index d547a7b236..246491eba7 100644 --- a/test/JDBC/input/BABEL_1940.sql +++ b/test/JDBC/input/BABEL_1940.sql @@ -110,3 +110,14 @@ GO DROP TABLE babel_1940_t1 GO +SELECT CAST(CAST(0x61 AS VARBINARY) AS BINARY(3)) +GO + +SELECT CAST(CAST(0x61 AS BINARY(3)) AS VARBINARY(2)) +GO + +SELECT CAST(CAST(0x6161616161 AS BINARY(3)) AS VARBINARY(2)) +GO + +SELECT CAST(CAST(0x616263646566 AS VARBINARY(5)) AS BINARY(3)) +GO From 0bbf3c851c3a7b3213f2bd18a5bce385d127b2ee Mon Sep 17 00:00:00 2001 From: Tanzeel Khan Date: Fri, 10 Nov 2023 07:43:54 +0000 Subject: [PATCH 28/28] some minor refactoring Signed-off-by: Tanzeel Khan --- contrib/babelfishpg_common/src/varbinary.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/contrib/babelfishpg_common/src/varbinary.c b/contrib/babelfishpg_common/src/varbinary.c index a05aa2605b..7fade4c93d 100644 --- a/contrib/babelfishpg_common/src/varbinary.c +++ b/contrib/babelfishpg_common/src/varbinary.c @@ -666,7 +666,7 @@ varcharvarbinary(PG_FUNCTION_ARGS) /* * If typmod is -1 (or invalid), use the actual length - * Length should be checked after encoding into server enc + * Length should be checked after encoding into server encoding */ if (typmod < (int32) VARHDRSZ) maxlen = encodedByteLen; @@ -736,7 +736,7 @@ varbinaryvarchar(PG_FUNCTION_ARGS) int32 maxlen = typmod - VARHDRSZ; coll_info collInfo; int encodedByteLen; - MemoryContext ccxt; + MemoryContext ccxt = CurrentMemoryContext; /* * Allow trailing null bytes @@ -749,8 +749,6 @@ varbinaryvarchar(PG_FUNCTION_ARGS) while(len>0 && data[len-1] == '\0') len -= 1; - collInfo = lookup_collation_table(get_server_collation_oid_internal(false)); - ccxt = CurrentMemoryContext; /* * Cast the entire input binary data if maxlen is * invalid or supplied data fits it @@ -758,6 +756,7 @@ varbinaryvarchar(PG_FUNCTION_ARGS) */ PG_TRY(); { + collInfo = lookup_collation_table(get_server_collation_oid_internal(false)); if (maxlen < 0 || len <= maxlen) encoded_result = encoding_conv_util(data, len, collInfo.enc, PG_UTF8, &encodedByteLen); else