Fix macOS default locale to have UTF-8 encoding set

If LANG is unset in macOS (either manually or when launching GUI version
of Vim from the Dock where this environmental variable is not set by the
OS), Vim tries to query the system locale and set that. However, the
system API doesn't set the encoding part, and there are tools like new
versions of Python that seems to want to have an explicit encoding bit
set. Also, macOS defaults to UTF-8 anyway, so just make sure to append
".UTF-8" to the locale to explicitly set it.

Also, add regression test for both this change and for 509f803 (vim#7003)
which added logic to make sure we set LC_NUMERIC to "C" in the default
locale detection.

This is a port of macvim-dev/macvim#1036

src/os_mac_conv.c

@@ -570,12 +570,20 @@ mac_lang_init(void)
 {
     if (mch_getenv((char_u *)"LANG") == NULL)
     {
+	// If LANG is unset (either because it was manually unset or a GUI
+	// version is opened from the Dock), we want to just query the system
+	// locale.
 	char	buf[20];
 	if (LocaleRefGetPartString(NULL,
 		    kLocaleLanguageMask | kLocaleLanguageVariantMask |
 		    kLocaleRegionMask | kLocaleRegionVariantMask,
 		    sizeof buf, buf) == noErr && *buf)
 	{
+	    // The API does not provide an encoding component. Explicitly add
+	    // ".UTF-8" since that's native in macOS, and it helps tools like
+	    // Python parse the locale correctly.
+	    strlcat(buf, ".UTF-8", sizeof(buf)/sizeof(char));
+
 	    vim_setenv((char_u *)"LANG", (char_u *)buf);
 #   ifdef HAVE_LOCALE_H
 	    setlocale(LC_ALL, "");

src/testdir/Make_all.mak

@@ -175,6 +175,7 @@ NEW_TESTS = \
 	test_listener \
 	test_listlbr \
 	test_listlbr_utf8 \
+	test_locale \
 	test_lua \
 	test_makeencoding \
 	test_man \

src/testdir/test_alot.vim

@@ -19,6 +19,7 @@ source test_glob2regpat.vim
 source test_global.vim
 source test_jumps.vim
 source test_lispwords.vim
+source test_locale.vim
 source test_move.vim
 source test_put.vim
 source test_recover.vim

src/testdir/test_locale.vim

@@ -0,0 +1,26 @@
+" Test for locale behaviors
+
+source check.vim
+source term_util.vim
+
+" Test that on macOS, we correctly set the default locale to have UTF-8
+" encoding and LC_NUMERIC is set to "C", if $LANG env variable is not set.
+func Test_macos_default_locale()
+  if !has('osxdarwin')
+    return
+  endif
+
+  " Run Vim after unsetting all the locale environmental vars, and capture the
+  " output of :lang.
+  let lang_results = system("unset LANG; unset LC_MESSAGES; " ..
+                            \ v:progpath ..
+                            \ " --clean -esX -c 'redir @a' -c 'lang' -c 'put a' -c 'print' -c 'qa!' ")
+
+  " Check that:
+  " 1. The locale is the form of <locale>.UTF-8.
+  " 2. Check that fourth item (LC_NUMERIC) is properly set to "C".
+  " Example match: "en_US.UTF-8/en_US.UTF-8/en_US.UTF-8/C/en_US.UTF-8/en_US.UTF-8"
+  call assert_match('"\([a-zA-Z_]\+\.UTF-8/\)\{3}C\(/[a-zA-Z_]\+\.UTF-8\)\{2}"', lang_results, "Default locale doesn't have UTF-8 encoding set")
+endfunc
+
+" vim: shiftwidth=2 sts=2 expandtab