From 84ed930039f7a7ce51491813580558242314770c Mon Sep 17 00:00:00 2001 From: Geremia Taglialatela Date: Thu, 10 Dec 2015 16:26:44 +0100 Subject: [PATCH] Improve encoding support in CSV converter Refers to #2191 --- app/views/rails_admin/main/export.html.haml | 2 +- lib/rails_admin/support/csv_converter.rb | 49 +++++++++++++++++-- .../rails_admin/support/csv_converter_spec.rb | 13 +++++ 3 files changed, 59 insertions(+), 5 deletions(-) diff --git a/app/views/rails_admin/main/export.html.haml b/app/views/rails_admin/main/export.html.haml index 246aea3ca0..c9ec8fad3b 100644 --- a/app/views/rails_admin/main/export.html.haml +++ b/app/views/rails_admin/main/export.html.haml @@ -60,7 +60,7 @@ %label.col-sm-2.control-label{for: "csv_options_encoding_to"}= t('admin.export.csv.encoding_to') .col-sm-10.controls -# from http://books.google.com/support/partner/bin/answer.py?answer=30990 : - = select_tag 'csv_options[encoding_to]', options_for_select(RailsAdmin::CSVConverter::TARGET_ENCODINGS), include_blank: true, placeholder: t('admin.misc.search'), :'data-enumeration' => true + = select_tag 'csv_options[encoding_to]', options_for_select(Encoding.name_list.sort), include_blank: true, placeholder: t('admin.misc.search'), :'data-enumeration' => true %p.help-block= t('admin.export.csv.encoding_to_help', name: guessed_encoding) .form-group.control-group diff --git a/lib/rails_admin/support/csv_converter.rb b/lib/rails_admin/support/csv_converter.rb index c54dd98dbc..8f96694601 100644 --- a/lib/rails_admin/support/csv_converter.rb +++ b/lib/rails_admin/support/csv_converter.rb @@ -3,8 +3,49 @@ module RailsAdmin class CSVConverter - UTF8_ENCODINGS = [nil, '', 'utf8', 'utf-8', 'unicode', 'UTF8', 'UTF-8', 'UNICODE', 'utf8mb4'] - TARGET_ENCODINGS = %w(UTF-8 UTF-16LE UTF-16BE UTF-32LE UTF-32BE UTF-7 ISO-8859-1 ISO-8859-15 IBM850 MacRoman Windows-1252 ISO-8859-3 IBM852 ISO-8859-2 Windows-1250 IBM855 ISO-8859-5 KOI8-R MacCyrillic Windows-1251 IBM866 GB2312 GBK GB18030 Big5 Big5-HKSCS EUC-TW EUC-JP ISO-2022-JP Shift_JIS EUC-KR) + # Taken from here: + # https://github.com/tmtm/ruby-mysql/blob/master/lib/mysql/charset.rb + # Author: TOMITA Masahiro + ENCODINGS = { + 'armscii8' => nil, + 'ascii' => Encoding::US_ASCII, + 'big5' => Encoding::Big5, + 'binary' => Encoding::ASCII_8BIT, + 'cp1250' => Encoding::Windows_1250, + 'cp1251' => Encoding::Windows_1251, + 'cp1256' => Encoding::Windows_1256, + 'cp1257' => Encoding::Windows_1257, + 'cp850' => Encoding::CP850, + 'cp852' => Encoding::CP852, + 'cp866' => Encoding::IBM866, + 'cp932' => Encoding::Windows_31J, + 'dec8' => nil, + 'eucjpms' => Encoding::EucJP_ms, + 'euckr' => Encoding::EUC_KR, + 'gb2312' => Encoding::EUC_CN, + 'gbk' => Encoding::GBK, + 'geostd8' => nil, + 'greek' => Encoding::ISO_8859_7, + 'hebrew' => Encoding::ISO_8859_8, + 'hp8' => nil, + 'keybcs2' => nil, + 'koi8r' => Encoding::KOI8_R, + 'koi8u' => Encoding::KOI8_U, + 'latin1' => Encoding::ISO_8859_1, + 'latin2' => Encoding::ISO_8859_2, + 'latin5' => Encoding::ISO_8859_9, + 'latin7' => Encoding::ISO_8859_13, + 'macce' => Encoding::MacCentEuro, + 'macroman' => Encoding::MacRoman, + 'sjis' => Encoding::SHIFT_JIS, + 'swe7' => nil, + 'tis620' => Encoding::TIS_620, + 'ucs2' => Encoding::UTF_16BE, + 'ujis' => Encoding::EucJP_ms, + 'utf8' => Encoding::UTF_8, + 'utf8mb4' => Encoding::UTF_8, + } + def initialize(objects = [], schema = {}) return self if (@objects = objects).blank? @@ -35,8 +76,8 @@ def initialize(objects = [], schema = {}) def to_csv(options = {}) # encoding shenanigans first - @encoding_from = Encoding.find(UTF8_ENCODINGS.include?(@abstract_model.encoding) ? 'UTF-8' : @abstract_model.encoding) - @encoding_to = Encoding.find(options[:encoding_to].presence || @encoding_from) + @encoding_from = ENCODINGS[@abstract_model.encoding] || Encoding::UTF_8 + @encoding_to = options[:encoding_to].present? ? Encoding.find(options[:encoding_to]) : @encoding_from csv_string = generate_csv_string(options) diff --git a/spec/rails_admin/support/csv_converter_spec.rb b/spec/rails_admin/support/csv_converter_spec.rb index 59e684e1fc..f086057187 100644 --- a/spec/rails_admin/support/csv_converter_spec.rb +++ b/spec/rails_admin/support/csv_converter_spec.rb @@ -45,6 +45,19 @@ end end + context 'when encoding FROM MySQL latin1' do + let(:encoding) { '' } + let(:objects) { FactoryGirl.create_list :player, 1, number: 1, name: 'Josè'.encode('ISO-8859-1') } + + it 'exports to ISO-8859-1', active_record: true do + expect(::ActiveRecord::Base.connection).to receive(:encoding) { 'latin1' } + expect(subject[1]).to eq 'ISO-8859-1' + expect(subject[2].encoding).to eq Encoding::ISO_8859_1 + expect(subject[2].unpack('H*').first). + to eq '4e756d6265722c4e616d650a312c4a6f73e80a' + end + end + context 'when encoding to UTF-8' do let(:encoding) { 'UTF-8' }