Merge pull request #1040 from felixbuenemann/fix-encoding-list

brianmario · Apr 27, 2019 · d7c91ff · d7c91ff
2 parents 624409d + d0cc2d3
commit d7c91ff
Show file tree

Hide file tree

Showing 10 changed files with 138 additions and 74 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -65,6 +65,7 @@ matrix:
           - mysql2gem.example.com
     - rvm: 2.4
       env: DB=mysql80
+      dist: xenial
       addons:
         hosts:
           - mysql2gem.example.com

diff --git a/.travis_Dockerfile_centos b/.travis_Dockerfile_centos
@@ -17,7 +17,7 @@ RUN yum -y install \
   gcc \
   gcc-c++ \
   make
-RUN gem update --system > /dev/null
-RUN gem install bundler
+RUN gem install --no-document "rubygems-update:~>2.7" && update_rubygems
+RUN gem install --no-document "bundler:~>1.17"
 
 CMD sh .travis_centos.sh
diff --git a/.travis_mysql57.sh b/.travis_mysql57.sh
@@ -5,8 +5,8 @@ set -eux
 apt-get purge -qq '^mysql*' '^libmysql*'
 rm -fr /etc/mysql
 rm -fr /var/lib/mysql
-apt-key add - < support/5072E1F5.asc
-add-apt-repository 'deb http://repo.mysql.com/apt/ubuntu/ trusty mysql-5.7'
+apt-key add support/5072E1F5.asc
+add-apt-repository 'http://repo.mysql.com/apt/ubuntu mysql-5.7'
 apt-get update -qq
 apt-get install -qq mysql-server libmysqlclient-dev
 

diff --git a/.travis_mysql80.sh b/.travis_mysql80.sh
@@ -5,8 +5,8 @@ set -eux
 apt-get purge -qq '^mysql*' '^libmysql*'
 rm -fr /etc/mysql
 rm -fr /var/lib/mysql
-apt-key add - < support/5072E1F5.asc
-add-apt-repository 'deb http://repo.mysql.com/apt/ubuntu/ trusty mysql-8.0'
+apt-key add support/5072E1F5.asc
+add-apt-repository 'http://repo.mysql.com/apt/ubuntu mysql-8.0'
 apt-get update -qq
 apt-get install -qq mysql-server libmysqlclient-dev
 

diff --git a/ext/mysql2/mysql_enc_name_to_ruby.h b/ext/mysql2/mysql_enc_name_to_ruby.h
@@ -30,7 +30,7 @@ error "gperf generated tables don't work with this execution character set. Plea
 #endif
 
 struct mysql2_mysql_enc_name_to_rb_map { const char *name; const char *rb_name; };
-/* maximum key range = 66, duplicates = 0 */
+/* maximum key range = 71, duplicates = 0 */
 
 #ifdef __GNUC__
 __inline
@@ -46,32 +46,32 @@ mysql2_mysql_enc_name_to_rb_hash (str, len)
 {
   static const unsigned char asso_values[] =
     {
-      69, 69, 69, 69, 69, 69, 69, 69, 69, 69,
-      69, 69, 69, 69, 69, 69, 69, 69, 69, 69,
-      69, 69, 69, 69, 69, 69, 69, 69, 69, 69,
-      69, 69, 69, 69, 69, 69, 69, 69, 69, 69,
-      69, 69, 69, 69, 69, 69, 69, 69, 40,  5,
-       0, 69,  0, 40, 25, 20, 10, 55, 69, 69,
-      69, 69, 69, 69, 69, 69, 69, 69, 69, 69,
-      69, 69, 69, 69, 69, 69, 69, 69, 69, 69,
-      69, 69, 69, 69, 69, 69, 69, 69, 69, 69,
-      69, 69, 69, 69, 69, 69, 69, 35,  5,  0,
-      10,  0, 20,  0,  5,  5, 69,  0, 10, 15,
-       0,  0, 69, 69, 25,  5,  5,  0, 69, 30,
-      69,  0, 69, 69, 69, 69, 69, 69, 69, 69,
-      69, 69, 69, 69, 69, 69, 69, 69, 69, 69,
-      69, 69, 69, 69, 69, 69, 69, 69, 69, 69,
-      69, 69, 69, 69, 69, 69, 69, 69, 69, 69,
-      69, 69, 69, 69, 69, 69, 69, 69, 69, 69,
-      69, 69, 69, 69, 69, 69, 69, 69, 69, 69,
-      69, 69, 69, 69, 69, 69, 69, 69, 69, 69,
-      69, 69, 69, 69, 69, 69, 69, 69, 69, 69,
-      69, 69, 69, 69, 69, 69, 69, 69, 69, 69,
-      69, 69, 69, 69, 69, 69, 69, 69, 69, 69,
-      69, 69, 69, 69, 69, 69, 69, 69, 69, 69,
-      69, 69, 69, 69, 69, 69, 69, 69, 69, 69,
-      69, 69, 69, 69, 69, 69, 69, 69, 69, 69,
-      69, 69, 69, 69, 69, 69
+      74, 74, 74, 74, 74, 74, 74, 74, 74, 74,
+      74, 74, 74, 74, 74, 74, 74, 74, 74, 74,
+      74, 74, 74, 74, 74, 74, 74, 74, 74, 74,
+      74, 74, 74, 74, 74, 74, 74, 74, 74, 74,
+      74, 74, 74, 74, 74, 74, 74, 74, 15,  5,
+       0, 74,  5, 25, 40, 10, 20, 50, 74, 74,
+      74, 74, 74, 74, 74, 74, 74, 74, 74, 74,
+      74, 74, 74, 74, 74, 74, 74, 74, 74, 74,
+      74, 74, 74, 74, 74, 74, 74, 74, 74, 74,
+      74, 74, 74, 74, 74, 74, 74, 40,  5,  0,
+      15, 10,  0,  0,  0,  5, 74,  0, 25,  5,
+       0,  5, 74, 74, 20,  5,  5,  0, 74, 45,
+      74,  0, 74, 74, 74, 74, 74, 74, 74, 74,
+      74, 74, 74, 74, 74, 74, 74, 74, 74, 74,
+      74, 74, 74, 74, 74, 74, 74, 74, 74, 74,
+      74, 74, 74, 74, 74, 74, 74, 74, 74, 74,
+      74, 74, 74, 74, 74, 74, 74, 74, 74, 74,
+      74, 74, 74, 74, 74, 74, 74, 74, 74, 74,
+      74, 74, 74, 74, 74, 74, 74, 74, 74, 74,
+      74, 74, 74, 74, 74, 74, 74, 74, 74, 74,
+      74, 74, 74, 74, 74, 74, 74, 74, 74, 74,
+      74, 74, 74, 74, 74, 74, 74, 74, 74, 74,
+      74, 74, 74, 74, 74, 74, 74, 74, 74, 74,
+      74, 74, 74, 74, 74, 74, 74, 74, 74, 74,
+      74, 74, 74, 74, 74, 74, 74, 74, 74, 74,
+      74, 74, 74, 74, 74, 74
     };
   return len + asso_values[(unsigned char)str[2]] + asso_values[(unsigned char)str[0]] + asso_values[(unsigned char)str[len - 1]];
 }
@@ -89,66 +89,70 @@ mysql2_mysql_enc_name_to_rb (str, len)
 {
   enum
     {
-      TOTAL_KEYWORDS = 39,
+      TOTAL_KEYWORDS = 41,
       MIN_WORD_LENGTH = 3,
       MAX_WORD_LENGTH = 8,
       MIN_HASH_VALUE = 3,
-      MAX_HASH_VALUE = 68
+      MAX_HASH_VALUE = 73
     };
 
   static const struct mysql2_mysql_enc_name_to_rb_map wordlist[] =
     {
       {""}, {""}, {""},
       {"gbk", "GBK"},
       {""},
-      {"greek", "ISO-8859-7"},
+      {"utf32", "UTF-32"},
       {"gb2312", "GB2312"},
       {"keybcs2", NULL},
       {""},
       {"ucs2", "UTF-16BE"},
       {"koi8u", "KOI8-R"},
       {"binary", "ASCII-8BIT"},
-      {"eucjpms", "eucJP-ms"},
-      {""},
+      {"utf8mb4", "UTF-8"},
+      {"macroman", "macRoman"},
       {"ujis", "eucJP-ms"},
-      {"cp852", "CP852"},
+      {"greek", "ISO-8859-7"},
       {"cp1251", "Windows-1251"},
-      {"geostd8", NULL},
+      {"utf16le", "UTF-16LE"},
       {""},
       {"sjis", "Shift_JIS"},
       {"macce", "macCentEuro"},
-      {"latin2", "ISO-8859-2"},
+      {"cp1257", "Windows-1257"},
+      {"eucjpms", "eucJP-ms"},
+      {""},
+      {"utf8", "UTF-8"},
+      {"cp852", "CP852"},
+      {"cp1250", "Windows-1250"},
+      {"gb18030", "GB18030"},
       {""},
-      {"macroman", "macRoman"},
-      {"dec8", NULL},
-      {"utf32", "UTF-32"},
-      {"latin1", "ISO-8859-1"},
-      {"utf8mb4", "UTF-8"},
-      {"hp8", NULL},
       {"swe7", NULL},
+      {"koi8r", "KOI8-R"},
+      {"tis620", "TIS-620"},
+      {"geostd8", NULL},
+      {""},
+      {"big5", "Big5"},
       {"euckr", "EUC-KR"},
-      {"cp1257", "Windows-1257"},
+      {"latin2", "ISO-8859-2"},
       {""}, {""},
-      {"utf8", "UTF-8"},
-      {"koi8r", "KOI8-R"},
-      {"cp1256", "Windows-1256"},
-      {""}, {""}, {""},
-      {"cp866", "IBM866"},
+      {"dec8", NULL},
+      {"cp850", "CP850"},
+      {"latin1", "ISO-8859-1"},
+      {""},
+      {"hp8", NULL},
+      {""},
+      {"utf16", "UTF-16"},
       {"latin7", "ISO-8859-13"},
       {""}, {""}, {""},
       {"ascii", "US-ASCII"},
-      {"hebrew", "ISO-8859-8"},
-      {""}, {""},
-      {"big5", "Big5"},
-      {"utf16", "UTF-16"},
-      {"cp1250", "Windows-1250"},
-      {""}, {""}, {""},
-      {"cp850", "CP850"},
-      {"tis620", "TIS-620"},
+      {"cp1256", "Windows-1256"},
       {""}, {""}, {""},
       {"cp932", "Windows-31J"},
+      {"hebrew", "ISO-8859-8"},
+      {""}, {""}, {""}, {""},
       {"latin5", "ISO-8859-9"},
-      {""}, {""}, {""}, {""}, {""}, {""},
+      {""}, {""}, {""},
+      {"cp866", "IBM866"},
+      {""}, {""}, {""}, {""}, {""}, {""}, {""},
       {"armscii8", NULL}
     };
 

diff --git a/ext/mysql2/mysql_enc_to_ruby.h b/ext/mysql2/mysql_enc_to_ruby.h
@@ -54,13 +54,13 @@ static const char *mysql2_mysql_enc_to_rb[] = {
   "macRoman",
   "UTF-16",
   "UTF-16",
-  "",
+  "UTF-16LE",
   "Windows-1256",
   "Windows-1257",
   "Windows-1257",
   "UTF-32",
   "UTF-32",
-  "",
+  "UTF-16LE",
   "ASCII-8BIT",
   NULL,
   "US-ASCII",
@@ -74,7 +74,7 @@ static const char *mysql2_mysql_enc_to_rb[] = {
   NULL,
   "KOI8-R",
   "KOI8-R",
-  NULL,
+  "UTF-8",
   "ISO-8859-2",
   "ISO-8859-9",
   "ISO-8859-13",
@@ -246,14 +246,65 @@ static const char *mysql2_mysql_enc_to_rb[] = {
   "UTF-8",
   "UTF-8",
   "UTF-8",
+  "GB18030",
+  "GB18030",
+  "GB18030",
+  NULL,
+  NULL,
   NULL,
   NULL,
+  "UTF-8",
+  "UTF-8",
+  "UTF-8",
+  "UTF-8",
+  "UTF-8",
+  "UTF-8",
+  "UTF-8",
+  "UTF-8",
+  "UTF-8",
+  "UTF-8",
+  "UTF-8",
+  "UTF-8",
+  "UTF-8",
+  "UTF-8",
+  "UTF-8",
+  "UTF-8",
+  "UTF-8",
+  NULL,
+  "UTF-8",
+  "UTF-8",
+  "UTF-8",
   NULL,
+  "UTF-8",
+  "UTF-8",
+  "UTF-8",
+  "UTF-8",
+  "UTF-8",
+  "UTF-8",
+  "UTF-8",
+  "UTF-8",
+  "UTF-8",
+  "UTF-8",
+  "UTF-8",
+  "UTF-8",
+  "UTF-8",
+  "UTF-8",
+  "UTF-8",
+  "UTF-8",
+  "UTF-8",
+  "UTF-8",
   NULL,
+  "UTF-8",
+  "UTF-8",
+  "UTF-8",
   NULL,
+  "UTF-8",
   NULL,
   NULL,
+  "UTF-8",
+  "UTF-8",
+  "UTF-8",
+  "UTF-8",
+  "UTF-8",
   "UTF-8"
 };
-
-#define CHARSETNR_SIZE (sizeof(mysql2_mysql_enc_to_rb)/sizeof(mysql2_mysql_enc_to_rb[0]))
diff --git a/ext/mysql2/result.c b/ext/mysql2/result.c
@@ -1,6 +1,7 @@
 #include <mysql2_ext.h>
 
 #include "mysql_enc_to_ruby.h"
+#define MYSQL2_CHARSETNR_SIZE (sizeof(mysql2_mysql_enc_to_rb)/sizeof(mysql2_mysql_enc_to_rb[0]))
 
 static rb_encoding *binaryEncoding;
 
@@ -179,7 +180,7 @@ static VALUE mysql2_set_field_string_encoding(VALUE val, MYSQL_FIELD field, rb_e
     const char *enc_name;
     int enc_index;
 
-    enc_name = (field.charsetnr-1 < CHARSETNR_SIZE) ? mysql2_mysql_enc_to_rb[field.charsetnr-1] : NULL;
+    enc_name = (field.charsetnr-1 < MYSQL2_CHARSETNR_SIZE) ? mysql2_mysql_enc_to_rb[field.charsetnr-1] : NULL;
 
     if (enc_name != NULL) {
       /* use the field encoding we were able to match */

diff --git a/support/5072E1F5.asc b/support/5072E1F5.asc
@@ -1,5 +1,5 @@
 -----BEGIN PGP PUBLIC KEY BLOCK-----
-Version: GnuPG v1.4.5 (GNU/Linux)
+Version: GnuPG v1
 
 mQGiBD4+owwRBAC14GIfUfCyEDSIePvEW3SAFUdJBtoQHH/nJKZyQT7h9bPlUWC3
 RODjQReyCITRrdwyrKUGku2FmeVGwn2u2WmDMNABLnpprWPkBdCk96+OmSLN9brZ
@@ -11,9 +11,9 @@ kYpXBACmWpP8NJTkamEnPCia2ZoOHODANwpUkP43I7jsDmgtobZX9qnrAXw+uNDI
 QJEXM6FSbi0LLtZciNlYsafwAPEOMDKpMqAK6IyisNtPvaLd8lH0bPAnWqcyefep
 rv0sxxqUEMcM3o7wwgfN83POkDasDbs3pjwPhxvhz6//62zQJ7Q2TXlTUUwgUmVs
 ZWFzZSBFbmdpbmVlcmluZyA8bXlzcWwtYnVpbGRAb3NzLm9yYWNsZS5jb20+iGwE
-ExECACwCGyMCHgECF4ACGQEGCwkIBwMCBhUKCQgCAwUWAgMBAAUCWKcFIAUJHirJ
-FAAKCRCMcY07UHLh9VcFAJ46pUyVd8BZ2r5CppMC1tmyQ3ceRgCfVPwuVsiS0VER
-5WUqtAQDt+DoetCIaQQTEQIAKQIbIwYLCQgHAwIEFQIIAwQWAgMBAh4BAheAAhkB
+ExECACwCGyMCHgECF4ACGQEGCwkIBwMCBhUKCQgCAwUWAgMBAAUCXEBY+wUJI87e
+5AAKCRCMcY07UHLh9RZPAJ9uvm0zlzfCN+DHxHVaoFLFjdVYTQCfborsC9tmEZYa
+whhogjeBkZkorbyIaQQTEQIAKQIbIwYLCQgHAwIEFQIIAwQWAgMBAh4BAheAAhkB
 BQJTAdRmBQkaZsvLAAoJEIxxjTtQcuH1X4MAoKNLWAbCBUj96637kv6Xa/fJuX5m
 AJwPtmgDfjUe2iuhXdTrFEPT19SB6ohmBBMRAgAmAhsjBgsJCAcDAgQVAggDBBYC
 AwECHgECF4AFAk53PioFCRP7AhUACgkQjHGNO1By4fUmzACeJdfqgc9gWTUhgmcM
@@ -428,5 +428,5 @@ GoaU9u41oyZTIiXPiFidJoIZCh7fdurP8pn3X+R5HUNXMr7M+ba8lSNxce/F3kmH
 0L7rsKqdh9d/aVxhJINJ+inVDnrXWVoXu9GBjT8Nco1iU9SIVAQYEQIADAUCTnc9
 7QUJE/sBuAASB2VHUEcAAQEJEIxxjTtQcuH1FJsAmwWK9vmwRJ/y9gTnJ8PWf0BV
 roUTAKClYAhZuX2nUNwH4vlEJQHDqYa5yQ==
-=HfUN
+=ghXk
 -----END PGP PUBLIC KEY BLOCK-----
diff --git a/support/mysql_enc_to_ruby.rb b/support/mysql_enc_to_ruby.rb
@@ -43,6 +43,8 @@
   "geostd8"  => "NULL",
   "cp932"    => "Windows-31J",
   "eucjpms"  => "eucJP-ms",
+  "utf16le"  => "UTF-16LE",
+  "gb18030"  => "GB18030",
 }
 
 client     = Mysql2::Client.new(username: user, password: pass, host: host, port: port.to_i)
@@ -52,7 +54,10 @@
 
 collations.each do |collation|
   mysql_col_idx = collation[2].to_i
-  rb_enc = mysql_to_rb[collation[1]]
+  rb_enc = mysql_to_rb.fetch(collation[1]) do |mysql_enc|
+    $stderr.puts "WARNING: Missing mapping for collation \"#{collation[0]}\" with encoding \"#{mysql_enc}\" and id #{mysql_col_idx}, assuming NULL"
+    "NULL"
+  end
   encodings[mysql_col_idx - 1] = [mysql_col_idx, rb_enc]
 end
 

diff --git a/support/ruby_enc_to_mysql.rb b/support/ruby_enc_to_mysql.rb
@@ -38,6 +38,8 @@
   "geostd8"  => nil,
   "cp932"    => "Windows-31J",
   "eucjpms"  => "eucJP-ms",
+  "utf16le"  => "UTF-16LE",
+  "gb18030"  => "GB18030",
 }
 
 puts <<-HEADER