Correct handling of UTF-8 and invalid sequences. Mark all ruby files …

…as UTF-8 encoded. Fixes #22
driskell · Jul 21, 2014 · e38c682 · e38c682
1 parent ef6384a
commit e38c682
Show file tree

Hide file tree

Showing 16 changed files with 41 additions and 2 deletions.
diff --git a/lib/log-courier/client.rb b/lib/log-courier/client.rb
@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 # Copyright 2014 Jason Woods.
 #
 # This file is a modification of code from Logstash Forwarder.

diff --git a/lib/log-courier/client_tls.rb b/lib/log-courier/client_tls.rb
@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 # Copyright 2014 Jason Woods.
 #
 # This file is a modification of code from Logstash Forwarder.

diff --git a/lib/log-courier/server.rb b/lib/log-courier/server.rb
@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 # Copyright 2014 Jason Woods.
 #
 # This file is a modification of code from Logstash Forwarder.
@@ -144,8 +146,15 @@ def process_jdat(message, comm, event_queue)
           raise ProtocolError
         end
 
-        # Extract message, and force UTF-8 to ensure we don't break anything, replacing invalid sequences
-        data = message[p...p + length].encode('utf-8', 'binary', :invalid => :replace, :undef => :replace, :replace => '?').force_encoding('UTF-8')
+        # Extract message
+        data = message[p...p + length].force_encoding('utf-8')
+
+        # Ensure valid encoding
+        unless data.valid_encoding?
+          data.chars.map do |c|
+            c.valid_encoding? ? c : "\xEF\xBF\xBD"
+          end
+        end
         p += length
 
         # Decode the JSON

diff --git a/lib/log-courier/server_tcp.rb b/lib/log-courier/server_tcp.rb
@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 # Copyright 2014 Jason Woods.
 #
 # This file is a modification of code from Logstash Forwarder.

diff --git a/lib/log-courier/server_zmq.rb b/lib/log-courier/server_zmq.rb
@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 # Copyright 2014 Jason Woods.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

diff --git a/spec/courier_spec.rb b/spec/courier_spec.rb
@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 # Copyright 2014 Jason Woods.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

diff --git a/spec/gem_spec.rb b/spec/gem_spec.rb
@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 # Copyright 2014 Jason Woods.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

diff --git a/spec/lib/common.rb b/spec/lib/common.rb
@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 # Copyright 2014 Jason Woods.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

diff --git a/spec/lib/helpers/common.rb b/spec/lib/helpers/common.rb
@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 # Copyright 2014 Jason Woods.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

diff --git a/spec/lib/helpers/log-courier.rb b/spec/lib/helpers/log-courier.rb
@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 # Copyright 2014 Jason Woods.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

diff --git a/spec/lib/logfile.rb b/spec/lib/logfile.rb
@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 # Copyright 2014 Jason Woods.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

diff --git a/spec/lib/logfile/multiline.rb b/spec/lib/logfile/multiline.rb
@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 # Copyright 2014 Jason Woods.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

diff --git a/spec/multiline_spec.rb b/spec/multiline_spec.rb
@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 # Copyright 2014 Jason Woods.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

diff --git a/spec/plainzmq_spec.rb b/spec/plainzmq_spec.rb
@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 # Copyright 2014 Jason Woods.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

diff --git a/spec/tcp_spec.rb b/spec/tcp_spec.rb
@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 # Copyright 2014 Jason Woods.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

diff --git a/spec/zmq_spec.rb b/spec/zmq_spec.rb
@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 # Copyright 2014 Jason Woods.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");