Merge pull request #633 from amake/native-encoding-fix

Use platform native encoding by default, not UTF-8

I'll call this good enough for now.  If we think of some better/more appropriate tests, we can put into a future PR.

CHANGES.md

@@ -61,6 +61,7 @@ Bug Fixes
 * [#601](https://github.com/java-native-access/jna/pull/601): Remove COMThread and COM initialization from objects and require callers to initialize COM themselves. Asserts are added to guard correct usage. - [@matthiasblaesing](https://github.com/matthiasblaesing).
 * [#602](https://github.com/java-native-access/jna/pull/602): Make sure SID related memory is properly released once no longer required [@lgoldstein](https://github.com/lgoldstein).
 * [#610](https://github.com/java-native-access/jna/pull/610): Fixed issue #604: Kernel32#GetLastError() always returns ERROR_SUCCESS [@lgoldstein](https://github.com/lgoldstein).
+* [#633](https://github.com/java-native-access/jna/pull/633): Restore default usage of platform native encoding for Java strings passed to native functions (was hard-coded to UTF-8 in 4.0 and later) [@amake](https://github.com/amake)
 * [#634](https://github.com/java-native-access/jna/pull/634): Improve BSTR handling - [@matthiasblaesing](https://github.com/matthiasblaesing).
 
 Release 4.2.1

src/com/sun/jna/Native.java

@@ -36,6 +36,7 @@
 import java.net.URLClassLoader;
 import java.nio.Buffer;
 import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
 import java.security.AccessController;
 import java.security.PrivilegedAction;
 import java.util.ArrayList;
@@ -95,7 +96,7 @@
  */
 public final class Native implements Version {
 
-    public static final String DEFAULT_ENCODING = "utf8";
+    public static final String DEFAULT_ENCODING = Charset.defaultCharset().name();
     public static boolean DEBUG_LOAD = Boolean.getBoolean("jna.debug_load");
     public static boolean DEBUG_JNA_LOAD = Boolean.getBoolean("jna.debug_load.jna");
 

test/com/sun/jna/CallbacksTest.java

@@ -16,6 +16,7 @@
 import java.io.PrintStream;
 import java.lang.ref.Reference;
 import java.lang.ref.WeakReference;
+import java.nio.charset.Charset;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashSet;
@@ -652,8 +653,9 @@ public String callback(String arg, String arg2) {
                 return arg + arg2;
             }
         };
-        final String VALUE = "value" + UNICODE;
-        final String VALUE2 = getName() + UNICODE;
+        Charset charset = Charset.forName(Native.getDefaultStringEncoding());
+        final String VALUE = "value" + charset.decode(charset.encode(UNICODE));
+        final String VALUE2 = getName() + charset.decode(charset.encode(UNICODE));
         String value = lib.callStringCallback(cb, VALUE, VALUE2);
         assertTrue("Callback not called", called[0]);
         assertEquals("Wrong String callback argument 0", VALUE, cbargs[0]);
@@ -673,8 +675,9 @@ public String callback(String arg, String arg2) {
         Map<?, ?> m = CallbackReference.allocations;
         m.clear();
 
-        String arg = getName() + "1" + UNICODE;
-        String arg2 = getName() + "2" + UNICODE;
+        Charset charset = Charset.forName(Native.getDefaultStringEncoding());
+        String arg = getName() + "1" + charset.decode(charset.encode(UNICODE));
+        String arg2 = getName() + "2" + charset.decode(charset.encode(UNICODE));
         String value = lib.callStringCallback(cb, arg, arg2);
         WeakReference<Object> ref = new WeakReference<Object>(value);
 
@@ -723,7 +726,8 @@ public String[] callback(String[] arg) {
                 return arg;
             }
         };
-        final String VALUE = "value" + UNICODE;
+        Charset charset = Charset.forName(Native.getDefaultStringEncoding());
+        final String VALUE = "value" + charset.decode(charset.encode(UNICODE));
         final String[] VALUE_ARRAY = { VALUE, null };
         Pointer value = lib.callStringArrayCallback(cb, VALUE_ARRAY);
         assertTrue("Callback not called", called[0]);
@@ -795,7 +799,8 @@ public TestStructure.ByValue callback(TestStructure.ByValue s) {
     public void testUnionByValueCallbackArgument() throws Exception{
         TestLibrary.TestUnion arg = new TestLibrary.TestUnion();
         arg.setType(String.class);
-        final String VALUE = getName() + UNICODE;
+        Charset charset = Charset.forName(arg.getStringEncoding());
+        final String VALUE = getName() + charset.decode(charset.encode(UNICODE));
         arg.f1 = VALUE;
         final boolean[] called = { false };
         final TestLibrary.TestUnion[] cbvalue = { null };

test/com/sun/jna/NativeTest.java

@@ -14,6 +14,7 @@
 
 import java.io.File;
 import java.lang.reflect.Method;
+import java.nio.charset.Charset;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
@@ -131,26 +132,40 @@ public void testToStringList() {
     public void testDefaultStringEncoding() throws Exception {
         final String UNICODE = "\u0444\u043b\u0441\u0432\u0443";
         final String UNICODEZ = UNICODE + "\0more stuff";
-        byte[] utf8 = Native.getBytes(UNICODE);
+        byte[] nativeEnc = Native.getBytes(UNICODE);
         byte[] expected = UNICODE.getBytes(Native.DEFAULT_ENCODING);
-        for (int i=0;i < Math.min(utf8.length, expected.length);i++) {
+        for (int i=0;i < Math.min(nativeEnc.length, expected.length);i++) {
             assertEquals("Improperly encoded at " + i,
-                         expected[i], utf8[i]);
+                         expected[i], nativeEnc[i]);
         }
-        assertEquals("Wrong number of encoded characters", expected.length, utf8.length);
-        String result = Native.toString(utf8);
-        assertEquals("Improperly decoded", UNICODE, result);
-
+        assertEquals("Wrong number of encoded characters", expected.length, nativeEnc.length);
+        String result = Native.toString(nativeEnc);
+        // The native encoding might not support our test string; the result
+        // will then be all '?'
+        if (!result.matches("^\\?+$")) {
+            assertEquals("Improperly decoded", UNICODE, result);
+        }
+        // When the native encoding doesn't support our test string, we can only
+        // usefully compare the lengths.
         assertEquals("Should truncate bytes at NUL terminator",
-                     UNICODE, Native.toString(UNICODEZ.getBytes(Native.DEFAULT_ENCODING)));
+                UNICODE.length(), Native.toString(UNICODEZ.getBytes(Native.DEFAULT_ENCODING)).length());
     }
 
     public void testCustomizeDefaultStringEncoding() {
         Properties oldprops = (Properties)System.getProperties().clone();
-        final String ENCODING = System.getProperty("file.encoding");
+        String encoding = null;
+        // Choose a charset that is not the default encoding so we can actually
+        // tell we changed it.
+        for (String charset : Charset.availableCharsets().keySet()) {
+            if (!charset.equals(Native.DEFAULT_ENCODING)) {
+                encoding = charset;
+                break;
+            }
+        }
+        assertNotNull("No available encodings other than the default!?", encoding);
         try {
-            System.setProperty("jna.encoding", ENCODING);
-            assertEquals("Default encoding should match jna.encoding setting", ENCODING, Native.getDefaultStringEncoding());
+            System.setProperty("jna.encoding", encoding);
+            assertEquals("Default encoding should match jna.encoding setting", encoding, Native.getDefaultStringEncoding());
         }
         finally {
             System.setProperties(oldprops);

test/com/sun/jna/ReturnTypesTest.java

@@ -12,6 +12,7 @@
  */
 package com.sun.jna;
 
+import java.nio.charset.Charset;
 import java.util.Collections;
 import java.util.List;
 import junit.framework.TestCase;
@@ -314,7 +315,8 @@ public void testReturnPointerArray() {
     }
 
     public void testReturnStringArray() {
-        final String VALUE = getName() + UNICODE;
+        Charset charset = Charset.forName(Native.getDefaultStringEncoding());
+        final String VALUE = getName() + charset.decode(charset.encode(UNICODE));
         String[] input = {
             VALUE, null,
         };

test/com/sun/jna/StructureTest.java

@@ -12,6 +12,7 @@
  */
 package com.sun.jna;
 
+import java.nio.charset.Charset;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
@@ -1084,9 +1085,10 @@ protected List<String> getFieldOrder() {
                 return Arrays.asList("inner");
             }
         }
-        final String VALUE = getName() + UNICODE;
-        final WString WVALUE = new WString(VALUE);
         StructureFromPointer o = new StructureFromPointer();
+        Charset charset = Charset.forName(o.getStringEncoding());
+        final String VALUE = getName() + charset.decode(charset.encode(UNICODE));
+        final WString WVALUE = new WString(VALUE);
         o.s = VALUE;
         o.ws = WVALUE;
         o.write();