Merge pull request #1592 from okkez/fix-1590

Set UTF-8 encoding to string from configuration file
fluent · Jun 12, 2017 · 6d66b50 · 6d66b50
2 parents 6fb570f + 831f4af
commit 6d66b50
Show file tree

Hide file tree

Showing 2 changed files with 82 additions and 2 deletions.
diff --git a/lib/fluent/supervisor.rb b/lib/fluent/supervisor.rb
@@ -218,7 +218,8 @@ def self.load_config(path, params = {})
 
       config_fname = File.basename(path)
       config_basedir = File.dirname(path)
-      config_data = File.read(path)
+      # Assume fluent.conf encoding is UTF-8
+      config_data = File.open(path, "r:utf-8:utf-8") {|f| f.read }
       inline_config = params['inline_config']
       if inline_config == '-'
         config_data << "\n" << STDIN.read
@@ -717,7 +718,7 @@ def read_config
       $log.info :supervisor, "reading config file", path: @config_path
       @config_fname = File.basename(@config_path)
       @config_basedir = File.dirname(@config_path)
-      @config_data = File.read(@config_path)
+      @config_data = File.open(@config_path, "r:utf-8:utf-8") {|f| f.read }
       if @inline_config == '-'
         @config_data << "\n" << STDIN.read
       elsif @inline_config

diff --git a/test/test_supervisor.rb b/test/test_supervisor.rb
@@ -77,6 +77,46 @@ def test_read_config
     $log.out.reset
   end
 
+  def test_read_config_with_multibyte_string
+    tmp_path = "#{TMP_DIR}/dir/test_multibyte_config.conf"
+    conf_str = %[
+<source>
+  @type forward
+  @id forward_input
+  @label @INPUT
+</source>
+<label @INPUT>
+  <filter>
+    @type record_transformer
+    <record>
+      message こんにちは. ${record["name"]} has made a order of ${record["item"]} just now.
+    </record>
+  </filter>
+  <match>
+    @type stdout
+  </match>
+</label>
+]
+    FileUtils.mkdir_p(File.dirname(tmp_path))
+    File.open(tmp_path, "w:utf-8") {|file| file.write(conf_str) }
+
+    opts = Fluent::Supervisor.default_options
+    sv = Fluent::Supervisor.new(opts)
+
+    use_v1_config = {}
+    use_v1_config['use_v1_config'] = true
+
+    sv.instance_variable_set(:@config_path, tmp_path)
+    sv.instance_variable_set(:@use_v1_config, use_v1_config)
+    sv.send(:read_config)
+
+    conf = sv.instance_variable_get(:@conf)
+    label = conf.elements.detect {|e| e.name == "label" }
+    filter = label.elements.detect {|e| e.name == "filter" }
+    record_transformer = filter.elements.detect {|e| e.name = "record_transformer" }
+    assert_equal(Encoding::UTF_8, record_transformer["message"].encoding)
+  end
+
   def test_system_config
     opts = Fluent::Supervisor.default_options
     sv = Fluent::Supervisor.new(opts)
@@ -319,6 +359,45 @@ def test_load_config_for_daemonize
     assert_equal Fluent::Log::LEVEL_INFO, se_config[:log_level]
   end
 
+  def test_load_config_with_multibyte_string
+    tmp_path = "#{TMP_DIR}/dir/test_multibyte_config.conf"
+    conf_str = %[
+<source>
+  @type forward
+  @id forward_input
+  @label @INPUT
+</source>
+<label @INPUT>
+  <filter>
+    @type record_transformer
+    <record>
+      message こんにちは. ${record["name"]} has made a order of ${record["item"]} just now.
+    </record>
+  </filter>
+  <match>
+    @type stdout
+  </match>
+</label>
+]
+    FileUtils.mkdir_p(File.dirname(tmp_path))
+    File.open(tmp_path, "w:utf-8") {|file| file.write(conf_str) }
+
+    params = {}
+    params['workers'] = 1
+    params['use_v1_config'] = true
+    params['log_path'] = 'test/tmp/supervisor/log'
+    params['suppress_repeated_stacktrace'] = true
+    params['log_level'] = Fluent::Log::LEVEL_INFO
+    load_config_proc =  Proc.new { Fluent::Supervisor.load_config(tmp_path, params) }
+
+    se_config = load_config_proc.call
+    conf = se_config[:fluentd_conf]
+    label = conf.elements.detect {|e| e.name == "label" }
+    filter = label.elements.detect {|e| e.name == "filter" }
+    record_transformer = filter.elements.detect {|e| e.name = "record_transformer" }
+    assert_equal(Encoding::UTF_8, record_transformer["message"].encoding)
+  end
+
   def test_logger
     opts = Fluent::Supervisor.default_options
     sv = Fluent::Supervisor.new(opts)