Improve encoding support in CSV converter

Refers to railsadminteam#2191
diowa · Dec 13, 2015 · 84ed930 · 84ed930
1 parent e28cd18
commit 84ed930
Show file tree

Hide file tree

Showing 3 changed files with 59 additions and 5 deletions.
diff --git a/app/views/rails_admin/main/export.html.haml b/app/views/rails_admin/main/export.html.haml
@@ -60,7 +60,7 @@
       %label.col-sm-2.control-label{for: "csv_options_encoding_to"}= t('admin.export.csv.encoding_to')
       .col-sm-10.controls
         -# from http://books.google.com/support/partner/bin/answer.py?answer=30990 :
-        = select_tag 'csv_options[encoding_to]', options_for_select(RailsAdmin::CSVConverter::TARGET_ENCODINGS), include_blank: true, placeholder: t('admin.misc.search'), :'data-enumeration' => true
+        = select_tag 'csv_options[encoding_to]', options_for_select(Encoding.name_list.sort), include_blank: true, placeholder: t('admin.misc.search'), :'data-enumeration' => true
         %p.help-block= t('admin.export.csv.encoding_to_help', name: guessed_encoding)
 
     .form-group.control-group

diff --git a/lib/rails_admin/support/csv_converter.rb b/lib/rails_admin/support/csv_converter.rb
@@ -3,8 +3,49 @@
 
 module RailsAdmin
   class CSVConverter
-    UTF8_ENCODINGS = [nil, '', 'utf8', 'utf-8', 'unicode', 'UTF8', 'UTF-8', 'UNICODE', 'utf8mb4']
-    TARGET_ENCODINGS = %w(UTF-8 UTF-16LE UTF-16BE UTF-32LE UTF-32BE UTF-7 ISO-8859-1 ISO-8859-15 IBM850 MacRoman Windows-1252 ISO-8859-3 IBM852 ISO-8859-2 Windows-1250 IBM855 ISO-8859-5 KOI8-R MacCyrillic Windows-1251 IBM866 GB2312 GBK GB18030 Big5 Big5-HKSCS EUC-TW EUC-JP ISO-2022-JP Shift_JIS EUC-KR)
+    # Taken from here:
+    #   https://github.com/tmtm/ruby-mysql/blob/master/lib/mysql/charset.rb
+    # Author: TOMITA Masahiro <[email protected]>
+    ENCODINGS = {
+      'armscii8' => nil,
+      'ascii'    => Encoding::US_ASCII,
+      'big5'     => Encoding::Big5,
+      'binary'   => Encoding::ASCII_8BIT,
+      'cp1250'   => Encoding::Windows_1250,
+      'cp1251'   => Encoding::Windows_1251,
+      'cp1256'   => Encoding::Windows_1256,
+      'cp1257'   => Encoding::Windows_1257,
+      'cp850'    => Encoding::CP850,
+      'cp852'    => Encoding::CP852,
+      'cp866'    => Encoding::IBM866,
+      'cp932'    => Encoding::Windows_31J,
+      'dec8'     => nil,
+      'eucjpms'  => Encoding::EucJP_ms,
+      'euckr'    => Encoding::EUC_KR,
+      'gb2312'   => Encoding::EUC_CN,
+      'gbk'      => Encoding::GBK,
+      'geostd8'  => nil,
+      'greek'    => Encoding::ISO_8859_7,
+      'hebrew'   => Encoding::ISO_8859_8,
+      'hp8'      => nil,
+      'keybcs2'  => nil,
+      'koi8r'    => Encoding::KOI8_R,
+      'koi8u'    => Encoding::KOI8_U,
+      'latin1'   => Encoding::ISO_8859_1,
+      'latin2'   => Encoding::ISO_8859_2,
+      'latin5'   => Encoding::ISO_8859_9,
+      'latin7'   => Encoding::ISO_8859_13,
+      'macce'    => Encoding::MacCentEuro,
+      'macroman' => Encoding::MacRoman,
+      'sjis'     => Encoding::SHIFT_JIS,
+      'swe7'     => nil,
+      'tis620'   => Encoding::TIS_620,
+      'ucs2'     => Encoding::UTF_16BE,
+      'ujis'     => Encoding::EucJP_ms,
+      'utf8'     => Encoding::UTF_8,
+      'utf8mb4'  => Encoding::UTF_8,
+    }
+
     def initialize(objects = [], schema = {})
       return self if (@objects = objects).blank?
 
@@ -35,8 +76,8 @@ def initialize(objects = [], schema = {})
 
     def to_csv(options = {})
       # encoding shenanigans first
-      @encoding_from = Encoding.find(UTF8_ENCODINGS.include?(@abstract_model.encoding) ? 'UTF-8' : @abstract_model.encoding)
-      @encoding_to = Encoding.find(options[:encoding_to].presence || @encoding_from)
+      @encoding_from = ENCODINGS[@abstract_model.encoding] || Encoding::UTF_8
+      @encoding_to = options[:encoding_to].present? ? Encoding.find(options[:encoding_to]) : @encoding_from
 
       csv_string = generate_csv_string(options)
 

diff --git a/spec/rails_admin/support/csv_converter_spec.rb b/spec/rails_admin/support/csv_converter_spec.rb
@@ -45,6 +45,19 @@
       end
     end
 
+    context 'when encoding FROM MySQL latin1' do
+      let(:encoding) { '' }
+      let(:objects) { FactoryGirl.create_list :player, 1, number: 1, name: 'Josè'.encode('ISO-8859-1') }
+
+      it 'exports to ISO-8859-1', active_record: true do
+        expect(::ActiveRecord::Base.connection).to receive(:encoding) { 'latin1' }
+        expect(subject[1]).to eq 'ISO-8859-1'
+        expect(subject[2].encoding).to eq Encoding::ISO_8859_1
+        expect(subject[2].unpack('H*').first).
+          to eq '4e756d6265722c4e616d650a312c4a6f73e80a'
+      end
+    end
+
     context 'when encoding to UTF-8' do
       let(:encoding) { 'UTF-8' }