fix conversion of varchar to binary varbinary and vice versa

contrib/babelfishpg_common/src/encoding/mb/conv.c

@@ -401,10 +401,6 @@ TsqlLocalToUtf(const unsigned char *iso, int len,
 		unsigned char b3 = 0;
 		unsigned char b4 = 0;
 
-		/* "break" cases all represent errors */
-		if (*iso == '\0')
-			break;
-
 		if (!IS_HIGHBIT_SET(*iso))
 		{
 			/* ASCII case is easy, assume it's one-to-one conversion */

contrib/babelfishpg_common/src/varbinary.c

@@ -16,7 +16,9 @@
 #include "access/hash.h"
 #include "catalog/pg_collation.h"
 #include "catalog/pg_type.h"
+#include "collation.h"
 #include "common/int.h"
+#include "encoding/encoding.h"
 #include "lib/hyperloglog.h"
 #include "libpq/pqformat.h"
 #include "miscadmin.h"
@@ -621,13 +623,16 @@ Datum
 varcharvarbinary(PG_FUNCTION_ARGS)
 {
 	VarChar    *source = PG_GETARG_VARCHAR_PP(0);
-	char	   *data = VARDATA_ANY(source);
+	char	   *data = VARDATA_ANY(source);		/* Source string is UTF-8 */
+	char	   *encoded_data;
 	char	   *rp;
 	size_t		len = VARSIZE_ANY_EXHDR(source);
 	int32		typmod = PG_GETARG_INT32(1);
 	bool		isExplicit = PG_GETARG_BOOL(2);
 	int32		maxlen;
 	bytea	   *result;
+	coll_info	collInfo;
+	int			encodedByteLen;
 
 	if (!isExplicit)
 		ereport(ERROR,
@@ -636,20 +641,26 @@ varcharvarbinary(PG_FUNCTION_ARGS)
 						"varbinary is not allowed. Use the CONVERT function "
 						"to run this query.")));
 
-	/* If typmod is -1 (or invalid), use the actual length */
+	collInfo = lookup_collation_table(get_server_collation_oid_internal(false));
+	encoded_data = encoding_conv_util(data, len, PG_UTF8, collInfo.enc, &encodedByteLen);
+
+	/* 
+	 * If typmod is -1 (or invalid), use the actual length
+	 * Length should be checked after encoding into server enc
+	 */
 	if (typmod < (int32) VARHDRSZ)
-		maxlen = len;
+		maxlen = encodedByteLen;
 	else
 		maxlen = typmod - VARHDRSZ;
 
-	if (len > maxlen)
-		len = maxlen;
+	if (encodedByteLen > maxlen)
+		encodedByteLen = maxlen;
 
-	result = (bytea *) palloc(len + VARHDRSZ);
-	SET_VARSIZE(result, len + VARHDRSZ);
+	result = (bytea *) palloc(encodedByteLen + VARHDRSZ);
+	SET_VARSIZE(result, encodedByteLen + VARHDRSZ);
 
 	rp = VARDATA(result);
-	memcpy(rp, data, len);
+	memcpy(rp, encoded_data, encodedByteLen);
 
 	PG_RETURN_BYTEA_P(result);
 }
@@ -697,21 +708,29 @@ Datum
 varbinaryvarchar(PG_FUNCTION_ARGS)
 {
 	bytea	   *source = PG_GETARG_BYTEA_PP(0);
-	char	   *data = VARDATA_ANY(source);
+	char	   *data = VARDATA_ANY(source);		/* Source data is server encoded */
+	VarChar    *result;
+	char 	   *encoded_result;
 	size_t		len = VARSIZE_ANY_EXHDR(source);
 	int32		typmod = PG_GETARG_INT32(1);
 	int32		maxlen = typmod - VARHDRSZ;
-	VarChar    *result;
+	coll_info	collInfo;
+	int			encodedByteLen;
+
+	collInfo = lookup_collation_table(get_server_collation_oid_internal(false));
 
 	/*
 	 * Cast the entire input binary data if maxlen is invalid or supplied data
 	 * fits it
 	 */
 	if (maxlen < 0 || len <= maxlen)
-		result = (VarChar *) cstring_to_text_with_len(data, len);
+		encoded_result = encoding_conv_util(data, len, collInfo.enc, PG_UTF8, &encodedByteLen);
 	/* Else truncate it */
 	else
-		result = (VarChar *) cstring_to_text_with_len(data, maxlen);
+		encoded_result = encoding_conv_util(data, maxlen, collInfo.enc, PG_UTF8, &encodedByteLen);
+
+	result = (VarChar *) cstring_to_text_with_len(encoded_result, encodedByteLen);
+
 	PG_RETURN_VARCHAR_P(result);
 }
 

contrib/babelfishpg_tsql/src/pltsql_coerce.c

@@ -16,6 +16,7 @@
 #include "catalog/pg_type.h"
 #include "catalog/pg_proc.h"
 #include "catalog/pg_namespace.h"
+#include "collation.h"
 #include "executor/spi.h"
 #include "mb/pg_wchar.h"
 #include "nodes/makefuncs.h"
@@ -32,6 +33,7 @@
 #include "utils/lsyscache.h"
 #include "utils/syscache.h"
 #include "pltsql_instr.h"
+#include "parser/parse_target.h"
 
 
 #include <math.h>
@@ -972,7 +974,7 @@ tsql_coerce_string_literal_hook(ParseCallbackState *pcbstate, Oid targetTypeId,
 		if (ccontext != COERCION_EXPLICIT)
 		{
 			/*
-			 * T-SQL may forbid casting from string literal to certain
+			 * T-SQL forbids implicit casting from string literal to certain
 			 * datatypes (i.e. binary, varbinary)
 			 */
 			if ((*common_utility_plugin_ptr->is_tsql_binary_datatype) (baseTypeId))
@@ -1100,6 +1102,34 @@ tsql_coerce_string_literal_hook(ParseCallbackState *pcbstate, Oid targetTypeId,
 					newcon->constvalue = stringTypeDatum(baseType, value, inputTypeMod);
 			}
 		}
+		else if ((*common_utility_plugin_ptr->is_tsql_binary_datatype) (baseTypeId) ||
+				 (*common_utility_plugin_ptr->is_tsql_varbinary_datatype) (baseTypeId))
+		{
+			/*
+			 * binary datatype should be passed in client encoding
+			 * when explicit cast is called
+			 */
+
+			TypeName 	*varcharTypeName = makeTypeName("varchar");
+			Node 		*result;
+			Const 		*tempcon;
+
+			typenameTypeIdAndMod(NULL, (const TypeName *)varcharTypeName, &baseTypeId, &baseTypeMod);
+			baseType = typeidType(baseTypeId);
+			pfree(varcharTypeName);
+
+			tempcon = makeConst(TEXTOID, -1,
+								tsql_get_server_collation_oid_internal(false),
+								-1, PointerGetDatum(cstring_to_text(value)),
+								false, false);
+
+			result = coerce_to_target_type(NULL, (Node *) tempcon, baseTypeId,
+										   targetTypeId, targetTypeMod,
+										   COERCION_EXPLICIT,
+										   COERCE_EXPLICIT_CAST,
+										   location);
+			return result;
+		}
 		else
 		{
 			newcon->constvalue = stringTypeDatum(baseType, value, inputTypeMod);

test/JDBC/expected/BABEL_1940.out

@@ -0,0 +1,217 @@
+SELECT CONVERT(VARCHAR(10), 0x123456789)
+GO
+~~START~~
+varchar
+
#Eg‰
+~~END~~
+
+
+SELECT CONVERT(VARCHAR(10), 0x80)
+GO
+~~START~~
+varchar
+€
+~~END~~
+
+
+SELECT CONVERT(VARCHAR(10), 0xaaa)
+GO
+~~START~~
+varchar
+<newline>ª
+~~END~~
+
+
+SELECT CONVERT(VARCHAR(10), 0x330033)
+GO
+~~START~~
+varchar
+3
+~~END~~
+
+
+SELECT CONVERT(VARBINARY(10), 'ｳ')
+GO
+~~START~~
+varbinary
+3F
+~~END~~
+
+
+SELECT CONVERT(VARBINARY(10), 'ﾊﾟ')
+GO
+~~START~~
+varbinary
+3F3F
+~~END~~
+
+
+SELECT CONVERT(VARBINARY(10), 'A')
+GO
+~~START~~
+varbinary
+41
+~~END~~
+
+
+SELECT CONVERT(VARBINARY(10), 'ア')
+GO
+~~START~~
+varbinary
+3F
+~~END~~
+
+
+SELECT CONVERT(VARBINARY(10), 0x81)
+GO
+~~START~~
+varbinary
+81
+~~END~~
+
+
+SELECT CONVERT(VARBINARY(10), 0x330033)
+GO
+~~START~~
+varbinary
+330033
+~~END~~
+
+
+DECLARE @key varchar(20) = 'part1'
+DECLARE @email varchar(20) = 'part2'
+SELECT CONVERT(VARCHAR(10), HASHBYTES('SHA1', @key + LOWER(@email)))
+GO
+~~START~~
+varchar
+æ/fact¢+Ó
+~~END~~
+
+
+
+CREATE TABLE babel_1940_t1 (a VARBINARY(9))
+GO
+
+INSERT INTO babel_1940_t1 VALUES(0x80)
+INSERT INTO babel_1940_t1 VALUES(0xaaa)
+INSERT INTO babel_1940_t1 VALUES(0x123456789)
+GO
+~~ROW COUNT: 1~~
+
+~~ROW COUNT: 1~~
+
+~~ROW COUNT: 1~~
+
+
+SELECT * FROM babel_1940_t1
+GO
+~~START~~
+varbinary
+80
+0AAA
+0123456789
+~~END~~
+
+
+SELECT CONVERT(VARCHAR(9), a) FROM babel_1940_t1
+GO
+~~START~~
+varchar
+€
+<newline>ª
+
#Eg‰
+~~END~~
+
+
+SELECT CAST(a as VARCHAR(9)) FROM babel_1940_t1
+GO
+~~START~~
+varchar
+€
+<newline>ª
+
#Eg‰
+~~END~~
+
+
+SELECT CAST(a as VARCHAR(10)) FROM babel_1940_t1
+GO
+~~START~~
+varchar
+€
+<newline>ª
+
#Eg‰
+~~END~~
+
+
+
+CREATE TABLE babel_1940_t2(a varchar(10) collate japanese_cs_as);
+GO
+
+insert into babel_1940_t2 values ('ｳ'), ('C'), ('ﾊﾟ'), ('３'), ('c'), ('ｲ'), ('Ｃ'),('ﾊ'),('1'), 
+('ｱ'),('パ'), ('b'), ('2'), ('B'),('１'), ('Ａ'),('ア'),('A'), ('a'),('AbC'), ('aBc');
+GO
+~~ROW COUNT: 21~~
+
+
+SELECT CONVERT(VARBINARY(10), a) FROM babel_1940_t2
+GO
+~~START~~
+varbinary
+3F
+43
+3F3F
+3F
+63
+3F
+3F
+3F
+31
+3F
+3F
+62
+32
+42
+3F
+3F
+3F
+41
+61
+416243
+614263
+~~END~~
+
+
+SELECT CONVERT(VARBINARY(10), CONVERT(VARCHAR(10), CONVERT(VARCHAR(10), a))) FROM babel_1940_t2
+GO
+~~START~~
+varbinary
+3F
+43
+3F3F
+3F
+63
+3F
+3F
+3F
+31
+3F
+3F
+62
+32
+42
+3F
+3F
+3F
+41
+61
+416243
+614263
+~~END~~
+
+
+
+DROP TABLE babel_1940_t2
+GO
+
+DROP TABLE babel_1940_t1
+GO