fix conversion of varchar to binary varbinary and vice versa

contrib/babelfishpg_common/sql/binary.sql

@@ -46,6 +46,12 @@ CREATE TYPE sys.BBF_BINARY (
     COLLATABLE     = false
 );
 
+CREATE CAST (sys.BBF_BINARY AS sys.BBF_VARBINARY)
+WITHOUT FUNCTION AS IMPLICIT;
+
+CREATE CAST (sys.BBF_VARBINARY AS sys.BBF_BINARY)
+WITHOUT FUNCTION AS IMPLICIT;
+
 -- casting functions for sys.BINARY
 CREATE OR REPLACE FUNCTION sys.varcharbinary(sys.VARCHAR, integer, boolean)
 RETURNS sys.BBF_BINARY

contrib/babelfishpg_common/sql/upgrades/babelfishpg_common--2.4.0--2.5.0.sql

@@ -37,5 +37,12 @@ CREATE OPERATOR sys./ (
 END IF;
 END $$;
 
+-- binary varbinary cast
+CREATE CAST (sys.BBF_BINARY AS sys.BBF_VARBINARY)
+WITHOUT FUNCTION AS IMPLICIT;
+
+CREATE CAST (sys.BBF_VARBINARY AS sys.BBF_BINARY)
+WITHOUT FUNCTION AS IMPLICIT;
+
 -- Reset search_path to not affect any subsequent scripts
 SELECT set_config('search_path', trim(leading 'sys, ' from current_setting('search_path')), false);

contrib/babelfishpg_common/src/varbinary.c

@@ -16,7 +16,9 @@
 #include "access/hash.h"
 #include "catalog/pg_collation.h"
 #include "catalog/pg_type.h"
+#include "collation.h"
 #include "common/int.h"
+#include "encoding/encoding.h"
 #include "lib/hyperloglog.h"
 #include "libpq/pqformat.h"
 #include "miscadmin.h"
@@ -621,13 +623,17 @@ Datum
 varcharvarbinary(PG_FUNCTION_ARGS)
 {
 	VarChar    *source = PG_GETARG_VARCHAR_PP(0);
-	char	   *data = VARDATA_ANY(source);
+	char	   *data = VARDATA_ANY(source);		/* Source string is UTF-8 */
+	char	   *encoded_data;
 	char	   *rp;
 	size_t		len = VARSIZE_ANY_EXHDR(source);
 	int32		typmod = PG_GETARG_INT32(1);
 	bool		isExplicit = PG_GETARG_BOOL(2);
 	int32		maxlen;
 	bytea	   *result;
+	coll_info	collInfo;
+	int			encodedByteLen;
+	MemoryContext ccxt = CurrentMemoryContext;
 
 	if (!isExplicit)
 		ereport(ERROR,
@@ -636,20 +642,45 @@ varcharvarbinary(PG_FUNCTION_ARGS)
 						"varbinary is not allowed. Use the CONVERT function "
 						"to run this query.")));
 
-	/* If typmod is -1 (or invalid), use the actual length */
+	PG_TRY();
+	{
+		collInfo = lookup_collation_table(get_server_collation_oid_internal(false));
+		encoded_data = encoding_conv_util(data, len, PG_UTF8, collInfo.enc, &encodedByteLen);
+	}
+	PG_CATCH();
+	{
+		MemoryContext ectx;
+		ErrorData    *errorData;
+
+		ectx = MemoryContextSwitchTo(ccxt);
+		errorData = CopyErrorData();
+		FlushErrorState();
+		MemoryContextSwitchTo(ectx);
+
+		ereport(ERROR,
+			   (errcode(ERRCODE_INTERNAL_ERROR),
+				errmsg("Failed to convert from data type varchar to varbinary, %s",
+				errorData->message)));
+	}
+	PG_END_TRY();
+
+	/* 
+	 * If typmod is -1 (or invalid), use the actual length
+	 * Length should be checked after encoding into server encoding
+	 */
 	if (typmod < (int32) VARHDRSZ)
-		maxlen = len;
+		maxlen = encodedByteLen;
 	else
 		maxlen = typmod - VARHDRSZ;
 
-	if (len > maxlen)
-		len = maxlen;
+	if (encodedByteLen > maxlen)
+		encodedByteLen = maxlen;
 
-	result = (bytea *) palloc(len + VARHDRSZ);
-	SET_VARSIZE(result, len + VARHDRSZ);
+	result = (bytea *) palloc(encodedByteLen + VARHDRSZ);
+	SET_VARSIZE(result, encodedByteLen + VARHDRSZ);
 
 	rp = VARDATA(result);
-	memcpy(rp, data, len);
+	memcpy(rp, encoded_data, encodedByteLen);
 
 	PG_RETURN_BYTEA_P(result);
 }
@@ -697,21 +728,59 @@ Datum
 varbinaryvarchar(PG_FUNCTION_ARGS)
 {
 	bytea	   *source = PG_GETARG_BYTEA_PP(0);
-	char	   *data = VARDATA_ANY(source);
+	char	   *data = VARDATA_ANY(source);		/* Source data is server encoded */
+	VarChar    *result;
+	char 	   *encoded_result;
 	size_t		len = VARSIZE_ANY_EXHDR(source);
 	int32		typmod = PG_GETARG_INT32(1);
 	int32		maxlen = typmod - VARHDRSZ;
-	VarChar    *result;
+	coll_info	collInfo;
+	int			encodedByteLen;
+	MemoryContext ccxt = CurrentMemoryContext;
 
 	/*
-	 * Cast the entire input binary data if maxlen is invalid or supplied data
-	 * fits it
+	 * Allow trailing null bytes 
+	 * Its safe since multi byte UTF-8 does not contain 0x00 
+	 * This is needed since we implicity add trailing zeroes to 
+	 * binary type if input is less than binary(n)
+	 * ex: CAST(CAST('a' AS BINARY(10)) AS VARCHAR) should work
+	 * and not fail because of null byte
 	 */
-	if (maxlen < 0 || len <= maxlen)
-		result = (VarChar *) cstring_to_text_with_len(data, len);
-	/* Else truncate it */
-	else
-		result = (VarChar *) cstring_to_text_with_len(data, maxlen);
+	while(len>0 && data[len-1] == '\0')
+		len -= 1;
+
+	/*
+	 * Cast the entire input binary data if maxlen is 
+	 * invalid or supplied data fits it
+	 * Else truncate it
+	 */
+	PG_TRY();
+	{
+		collInfo = lookup_collation_table(get_server_collation_oid_internal(false));
+		if (maxlen < 0 || len <= maxlen)
+			encoded_result = encoding_conv_util(data, len, collInfo.enc, PG_UTF8, &encodedByteLen);
+		else
+			encoded_result = encoding_conv_util(data, maxlen, collInfo.enc, PG_UTF8, &encodedByteLen);
+	}
+	PG_CATCH();
+	{
+		MemoryContext ectx;
+		ErrorData    *errorData;
+
+		ectx = MemoryContextSwitchTo(ccxt);
+		errorData = CopyErrorData();
+		FlushErrorState();
+		MemoryContextSwitchTo(ectx);
+
+		ereport(ERROR,
+			   (errcode(ERRCODE_INTERNAL_ERROR),
+				errmsg("Failed to convert from data type varbinary to varchar, %s",
+				errorData->message)));
+	}
+	PG_END_TRY();
+
+	result = (VarChar *) cstring_to_text_with_len(encoded_result, encodedByteLen);
+
 	PG_RETURN_VARCHAR_P(result);
 }
 

contrib/babelfishpg_tsql/src/pltsql_coerce.c

@@ -16,6 +16,7 @@
 #include "catalog/pg_type.h"
 #include "catalog/pg_proc.h"
 #include "catalog/pg_namespace.h"
+#include "collation.h"
 #include "executor/spi.h"
 #include "mb/pg_wchar.h"
 #include "nodes/makefuncs.h"
@@ -33,6 +34,7 @@
 #include "utils/lsyscache.h"
 #include "utils/syscache.h"
 #include "pltsql_instr.h"
+#include "parser/parse_target.h"
 
 
 #include <math.h>
@@ -941,7 +943,7 @@ tsql_coerce_string_literal_hook(ParseCallbackState *pcbstate, Oid targetTypeId,
 		if (ccontext != COERCION_EXPLICIT)
 		{
 			/*
-			 * T-SQL may forbid casting from string literal to certain
+			 * T-SQL forbids implicit casting from string literal to certain
 			 * datatypes (i.e. binary, varbinary)
 			 */
 			if ((*common_utility_plugin_ptr->is_tsql_binary_datatype) (baseTypeId))
@@ -1069,6 +1071,37 @@ tsql_coerce_string_literal_hook(ParseCallbackState *pcbstate, Oid targetTypeId,
 					newcon->constvalue = stringTypeDatum(baseType, value, inputTypeMod);
 			}
 		}
+		else if ((*common_utility_plugin_ptr->is_tsql_binary_datatype) (baseTypeId) ||
+				 (*common_utility_plugin_ptr->is_tsql_varbinary_datatype) (baseTypeId))
+		{
+			/*
+			 * binary datatype should be passed in client encoding
+			 * when explicit cast is called
+			 */
+
+			TypeName 	*varcharTypeName = makeTypeNameFromNameList(list_make2(makeString("sys"),
+																	makeString("varchar")));
+			Node 		*result;
+			Const 		*tempcon;
+
+			typenameTypeIdAndMod(NULL, (const TypeName *)varcharTypeName, &baseTypeId, &baseTypeMod);
+
+			tempcon = makeConst(varcharTypeName->typeOid, -1,
+								tsql_get_server_collation_oid_internal(false),
+								-1, PointerGetDatum(cstring_to_text(value)),
+								false, false);
+
+			result = coerce_to_target_type(NULL, (Node *) tempcon, baseTypeId,
+										   targetTypeId, targetTypeMod,
+										   COERCION_EXPLICIT,
+										   COERCE_EXPLICIT_CAST,
+										   location);
+
+			pfree(varcharTypeName);
+			ReleaseSysCache(baseType);
+
+			return result;
+		}
 		else
 		{
 			newcon->constvalue = stringTypeDatum(baseType, value, inputTypeMod);