From 990a2579c745b4297250e3d88e7cc35852d3e903 Mon Sep 17 00:00:00 2001 From: Jason Woods Date: Tue, 26 Oct 2021 08:55:11 +0100 Subject: [PATCH] Restore output plugin and further code quality improvements to ruby --- CHANGELOG.md | 13 +- docs/LogstashIntegration.md | 61 +++-- lc-lib/core/version.go | 4 +- ruby/log-courier/.gitignore | 3 +- ruby/log-courier/.rubocop.yml | 6 +- ruby/log-courier/Gemfile.lock | 31 +-- ruby/log-courier/LICENSE | 192 --------------- ruby/log-courier/lib/log-courier/client.rb | 8 +- .../log-courier/lib/log-courier/client_tcp.rb | 28 +-- .../lib => lib/log-courier/rspec}/openssl.cnf | 0 .../log-courier/rspec/spec_helper.rb} | 121 ++++++---- ruby/log-courier/lib/log-courier/server.rb | 6 +- .../log-courier/lib/log-courier/server_tcp.rb | 12 +- ruby/log-courier/lib/log-courier/version.rb | 5 +- ruby/log-courier/log-courier.gemspec | 21 +- ruby/log-courier/spec/gem_spec.rb | 122 ++++------ ruby/log-courier/spec/lib/common.rb | 46 ---- ruby/log-courier/spec/lib/logfile.rb | 142 ----------- ruby/logstash-input-courier/.gitignore | 2 - ruby/logstash-input-courier/.rubocop.yml | 6 +- ruby/logstash-input-courier/.travis.yml | 6 - ruby/logstash-input-courier/Gemfile.lock | 31 +-- ruby/logstash-input-courier/LICENSE | 192 --------------- .../lib/logstash/inputs/courier.rb | 15 +- .../logstash-input-courier.gemspec | 13 +- .../spec/courier_spec.rb | 31 --- ruby/logstash-input-courier/spec/gem_spec.rb | 61 +++++ .../spec/lib/helpers/common.rb | 193 --------------- .../spec/lib/openssl.cnf | 41 ---- .../spec/lib/rspec_configure.rb | 43 ---- ruby/logstash-input-courier/version.txt | 1 - ruby/logstash-output-courier/.gitignore | 6 +- ruby/logstash-output-courier/.rspec | 2 + ruby/logstash-output-courier/.rubocop.yml | 30 +++ ruby/logstash-output-courier/.travis.yml | 6 - ruby/logstash-output-courier/Gemfile | 20 +- ruby/logstash-output-courier/Gemfile.lock | 224 +++++++++++++----- ruby/logstash-output-courier/LICENSE | 192 --------------- ruby/logstash-output-courier/Rakefile | 18 +- .../download_logstash.sh | 18 ++ .../lib/logstash/outputs/courier.rb | 35 +-- .../logstash-output-courier.gemspec | 31 +-- ruby/logstash-output-courier/spec/gem_spec.rb | 59 +++++ .../spec/lib/common.rb | 46 ---- .../spec/lib/helpers/common.rb | 209 ---------------- .../spec/lib/logfile.rb | 145 ------------ .../spec/lib/openssl.cnf | 41 ---- ruby/logstash-output-courier/version.txt | 1 - 48 files changed, 631 insertions(+), 1908 deletions(-) delete mode 100644 ruby/log-courier/LICENSE rename ruby/log-courier/{spec/lib => lib/log-courier/rspec}/openssl.cnf (100%) rename ruby/log-courier/{spec/lib/helpers/common.rb => lib/log-courier/rspec/spec_helper.rb} (65%) delete mode 100644 ruby/log-courier/spec/lib/common.rb delete mode 100644 ruby/log-courier/spec/lib/logfile.rb delete mode 100644 ruby/logstash-input-courier/.travis.yml delete mode 100644 ruby/logstash-input-courier/LICENSE delete mode 100644 ruby/logstash-input-courier/spec/courier_spec.rb create mode 100644 ruby/logstash-input-courier/spec/gem_spec.rb delete mode 100644 ruby/logstash-input-courier/spec/lib/helpers/common.rb delete mode 100644 ruby/logstash-input-courier/spec/lib/openssl.cnf delete mode 100644 ruby/logstash-input-courier/spec/lib/rspec_configure.rb delete mode 100644 ruby/logstash-input-courier/version.txt create mode 100644 ruby/logstash-output-courier/.rspec create mode 100644 ruby/logstash-output-courier/.rubocop.yml delete mode 100644 ruby/logstash-output-courier/.travis.yml delete mode 100644 ruby/logstash-output-courier/LICENSE create mode 100755 ruby/logstash-output-courier/download_logstash.sh create mode 100644 ruby/logstash-output-courier/spec/gem_spec.rb delete mode 100644 ruby/logstash-output-courier/spec/lib/common.rb delete mode 100644 ruby/logstash-output-courier/spec/lib/helpers/common.rb delete mode 100644 ruby/logstash-output-courier/spec/lib/logfile.rb delete mode 100644 ruby/logstash-output-courier/spec/lib/openssl.cnf delete mode 100644 ruby/logstash-output-courier/version.txt diff --git a/CHANGELOG.md b/CHANGELOG.md index 0017b4ca0..587bc9cd4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,14 +1,23 @@ # Change Log -## 2.7.3 (Future) +## 2.7.3 -Future +26th October 2021 Log Courier - Add new [`add timezone name field`](docs/log-courier/Configuration.md#add-timezone-name-field) configuration that adds the timezone name such as `UTC` or `Europe/London` for use with the Logstash Date Filter. The existing `timezone` / `event.timezone` (ECS) fields were in a format the filter could not use (#345) - Fix race that might cause a file to use a configuration other than the first configuration it matches (#367) +Logstash Input Plugin + +- Removed `peer_recv_queue` configuration as it is unused. Only a single payload is received and processed at any one time by the plugin. + +Logstash Output Plugin + +- Now maintained again and updated to use latest log-courier ruby implementation which includes protocol handshake support +- Added support for `tcp` only output + ## 2.7.2 21st October 2021 diff --git a/docs/LogstashIntegration.md b/docs/LogstashIntegration.md index 3f088cdcd..89bdaf285 100644 --- a/docs/LogstashIntegration.md +++ b/docs/LogstashIntegration.md @@ -3,29 +3,30 @@ - [Logstash Integration](#logstash-integration) - [Overview](#overview) - [Installation](#installation) - - [Configuration](#configuration) + - [Input Configuration](#input-configuration) + - [Output Configuration](#output-configuration) ## Overview -Log Courier is built to work seamlessly with [Logstash](http://logstash.net). It -communicates via an input plugin called "courier". +Log Courier can be used to send events to [Logstash](http://logstash.net) by installing an input plugin. -(NOTE: An output plugin exists for Logstash to Logstash transmission but is archived -and no longer maintained and its use is not advised.) +Additionally, Logstash can send events to Log Carver by installing an output plugin. ## Installation -Simply run the following commands as the user Logstash was installed with to -install the latest stable version of the Log Courier plugin. +Simply run the following commands as the user Logstash was installed with to install the latest stable version of the Log Courier input plugin. cd /path/to/logstash ./bin/logstash-plugin install logstash-input-courier -Once the installation is complete, you can start using the plugin! +To install the output plugin, run the following -## Configuration + cd /path/to/logstash + ./bin/logstash-plugin install logstash-output-courier + +## Input Configuration -The 'courier' input plugin will now be available. An example configuration follows. +An example configuration for the `courier` input plugin is below: input { courier { @@ -37,20 +38,40 @@ The 'courier' input plugin will now be available. An example configuration follo The following options are available: -- transport - "tcp", "tls", "plainzmq" or "zmq" (default: "tls") +- transport - "tcp" or "tls" (default: "tls") - address - Interface address to listen on (defaults to all interfaces) - port - The port number to listen on - ssl_certificate - Path to server SSL certificate (tls) - ssl_key - Path to server SSL private key (tls) - ssl_key_passphrase - Password for ssl_key (tls, optional) - ssl_verify - If true, verifies client certificates (tls, default false) -- ssl_verify_default_ca - Accept client certificates signed by systems root CAs -(tls) -- ssl_verify_ca - Path to an SSL CA certificate to use for client certificate -verification (tls) +- ssl_verify_default_ca - Accept client certificates signed by systems root CAs (tls) +- ssl_verify_ca - Path to an SSL CA certificate to use for client certificate verification (tls) +- min_tls_version - Sets the minimum TLS version when transport is "tls", defaults to 1.2, minimum is 1.0 and maximum 1.3 +- max_packet_size - The maximum packet size to accept (default 10485760, corresponds to Log Courier's `spool max bytes`) +- add_peer_fields - Add "peer" field to events that identifies source host, and "peer_ssl_dn" for TLS peers with client certificates + +## Output Configuration + +An example configuration for the `courier` output plugin is below: + + output { + courier { + addresses => ['127.0.0.1'] + port => 12345 + ssl_certificate => "/opt/logstash/ssl/logstash.cer" + } + } + +The following options are available: + +- transport - "tcp" or "tls" (default: "tls") +- addresses - Address to connect to in array format (only one value is supported at the moment) +- port - Port to connect to +- ssl_ca - Path to SSL certificate to verify server certificate +- ssl_certificate - Path to client SSL certificate (optional) +- ssl_key - Path to client SSL private key (optional) +- ssl_key_passphrase - Password for ssl_key (optional) - min_tls_version - Sets the minimum TLS version when transport is "tls", defaults to 1.2, minimum is 1.0 and maximum 1.3 -- max_packet_size - The maximum packet size to accept (default 10485760, -corresponds to Log Courier's `"spool max bytes"`) -- peer_recv_queue - The size of the internal queue for each peer -- add_peer_fields - Add "peer" field to events that identifies source host, and -"peer_ssl_dn" for TLS peers with client certificates +- spool_size - Maximum number of events to spool before a flush is forced (default 1024) +- idle_timeout - Maximum time in seconds to wait for a full spool before flushing anyway (default 5) diff --git a/lc-lib/core/version.go b/lc-lib/core/version.go index 97714fed1..2c249f555 100644 --- a/lc-lib/core/version.go +++ b/lc-lib/core/version.go @@ -19,5 +19,5 @@ package core // LogCourierVersion is the library version number const LogCourierMajorVersion uint32 = 2 const LogCourierMinorVersion uint32 = 7 -const LogCourierPatchVersion uint32 = 0 -const LogCourierVersion string = "2.7.0" +const LogCourierPatchVersion uint32 = 3 +const LogCourierVersion string = "2.7.3" diff --git a/ruby/log-courier/.gitignore b/ruby/log-courier/.gitignore index ab7e1e759..b84bbaa8f 100644 --- a/ruby/log-courier/.gitignore +++ b/ruby/log-courier/.gitignore @@ -1,3 +1,2 @@ +/lib/log-courier/rspec/tmp /pkg -/spec/tmp -/vendor diff --git a/ruby/log-courier/.rubocop.yml b/ruby/log-courier/.rubocop.yml index 8abc6afad..8c1447f43 100644 --- a/ruby/log-courier/.rubocop.yml +++ b/ruby/log-courier/.rubocop.yml @@ -1,5 +1,7 @@ AllCops: NewCops: enable +Layout/LineLength: + Max: 140 Metrics/AbcSize: Enabled: false Metrics/BlockLength: @@ -10,13 +12,11 @@ Metrics/ClassLength: Enabled: false Metrics/CyclomaticComplexity: Enabled: false -Metrics/LineLength: - Max: 140 Metrics/PerceivedComplexity: Enabled: false Metrics/MethodLength: Enabled: false -Naming/Filename: +Naming/FileName: Enabled: false Style/FrozenStringLiteralComment: Enabled: false diff --git a/ruby/log-courier/Gemfile.lock b/ruby/log-courier/Gemfile.lock index d40f5ee04..ca4a02e14 100644 --- a/ruby/log-courier/Gemfile.lock +++ b/ruby/log-courier/Gemfile.lock @@ -1,24 +1,24 @@ PATH remote: . specs: - log-courier (2.7.2) + log-courier (2.7.3) cabin (~> 0.6) multi_json (~> 1.10) GEM remote: https://rubygems.org/ specs: - ast (2.4.0) + ast (2.4.2) cabin (0.9.0) diff-lcs (1.4.4) - jaro_winkler (1.5.4-java) - jrjackson (0.4.13-java) + jrjackson (0.4.11-java) multi_json (1.15.0) - parallel (1.19.1) - parser (2.7.0.4) - ast (~> 2.4.0) + parallel (1.21.0) + parser (3.0.2.0) + ast (~> 2.4.1) rainbow (3.0.0) - rake (13.0.1) + rake (13.0.6) + regexp_parser (2.1.1) rexml (3.2.5) rspec (3.10.0) rspec-core (~> 3.10.0) @@ -33,16 +33,19 @@ GEM diff-lcs (>= 1.2.0, < 2.0) rspec-support (~> 3.10.0) rspec-support (3.10.2) - rubocop (0.80.1) - jaro_winkler (~> 1.5.1) + rubocop (1.22.2) parallel (~> 1.10) - parser (>= 2.7.0.1) + parser (>= 3.0.0.0) rainbow (>= 2.2.2, < 4.0) + regexp_parser (>= 1.8, < 3.0) rexml + rubocop-ast (>= 1.12.0, < 2.0) ruby-progressbar (~> 1.7) - unicode-display_width (>= 1.4.0, < 1.7) - ruby-progressbar (1.10.1) - unicode-display_width (1.6.1) + unicode-display_width (>= 1.4.0, < 3.0) + rubocop-ast (1.12.0) + parser (>= 3.0.1.1) + ruby-progressbar (1.11.0) + unicode-display_width (2.1.0) PLATFORMS java diff --git a/ruby/log-courier/LICENSE b/ruby/log-courier/LICENSE deleted file mode 100644 index a67adf1f4..000000000 --- a/ruby/log-courier/LICENSE +++ /dev/null @@ -1,192 +0,0 @@ -Log Courier project licence: - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - Copyright 2014 Jason Woods. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/ruby/log-courier/lib/log-courier/client.rb b/ruby/log-courier/lib/log-courier/client.rb index 09108361e..f23a7845a 100644 --- a/ruby/log-courier/lib/log-courier/client.rb +++ b/ruby/log-courier/lib/log-courier/client.rb @@ -17,14 +17,10 @@ require 'log-courier/event_queue' require 'log-courier/protocol' +require 'log-courier/version' require 'multi_json' require 'zlib' -# Dummy until its no longer needed -class NativeException - def dummy; end -end - module LogCourier class TimeoutError < StandardError; end @@ -349,7 +345,7 @@ def run_io_loop @logger&.warn 'Timeout occurred' rescue ShutdownSignal raise - rescue StandardError, NativeException => e + rescue StandardError => e # Unknown error occurred @logger&.warn e, hint: 'Unknown error' end diff --git a/ruby/log-courier/lib/log-courier/client_tcp.rb b/ruby/log-courier/lib/log-courier/client_tcp.rb index c1547d19a..ffb03f7ae 100644 --- a/ruby/log-courier/lib/log-courier/client_tcp.rb +++ b/ruby/log-courier/lib/log-courier/client_tcp.rb @@ -35,12 +35,12 @@ def initialize(options = {}) @logger = @options[:logger] - [:port, :ssl_ca].each do |k| - raise "output/courier: '#{k}' is required" if @options[k].nil? - end + raise "output/courier: 'port' is required" if @options[:port].nil? return unless @options[:transport] == 'tls' + raise "output/courier: 'ssl_ca' is required if 'transport' is 'tls'" if @options[:ssl_ca].nil? + c = 0 [:ssl_certificate, :ssl_key].each do c += 1 @@ -71,7 +71,7 @@ def connect(io_control) run_send io_control rescue ShutdownSignal # Shutdown - rescue StandardError, NativeException => e # Can remove NativeException after 9.2.14.0 JRuby + rescue StandardError => e @logger&.warn e, hint: 'Unknown write error' io_control << ['F'] end @@ -79,7 +79,7 @@ def connect(io_control) run_recv io_control rescue ShutdownSignal # Shutdown - rescue StandardError, NativeException => e # Can remove NativeException after 9.2.14.0 JRuby + rescue StandardError => e @logger&.warn e, hint: 'Unknown read error' io_control << ['F'] end @@ -125,7 +125,7 @@ def resume_send def handshake(io_control) return true if @options[:disable_handshake] - @socket.write ['HELO', 20, 0, 2, 7, 2, 'RYLC'].pack('A4NNNNNA4') + @socket.write ['HELO', 20, 0, MAJOR_VERSION, MINOR_VERSION, PATCH_VERSION, 'RYLC'].pack('A4NNNNNA4') signature, data = receive if signature != 'VERS' @@ -140,7 +140,7 @@ def handshake(io_control) @logger&.info 'Remote identified', server_version: @vers[:client_version] true - rescue StandardError, NativeException => e # Can remove NativeException after 9.2.14.0 JRuby + rescue StandardError => e @logger&.warn e, hint: 'Unknown write error' io_control << ['F'] false @@ -177,7 +177,7 @@ def run_send(io_control) rescue OpenSSL::SSL::SSLError => e @logger&.warn 'SSL write error', error: e.message io_control << ['F'] - rescue IOError, Errno::ECONNRESET => e + rescue IOError, SystemCallError => e @logger&.warn 'Write error', error: e.message io_control << ['F'] end @@ -195,7 +195,7 @@ def run_recv(io_control) rescue EOFError @logger&.warn 'Connection closed by server' io_control << ['F'] - rescue IOError, Errno::ECONNRESET => e + rescue IOError, SystemCallError => e @logger&.warn 'Read error', error: e.message io_control << ['F'] end @@ -271,15 +271,11 @@ def tls_connect @logger&.info 'Connected successfully', address: address, port: port end - # Add extra logging data now we're connected - @logger['address'] = address - @logger['port'] = port - return true - rescue OpenSSL::SSL::SSLError, IOError, Errno::ECONNRESET => e + rescue OpenSSL::SSL::SSLError, IOError, SystemCallError => e @logger&.warn 'Connection failed', error: e.message, address: address, port: port - rescue StandardError, NativeException => e # Can remove NativeException after 9.2.14.0 JRuby - @logger&.warn e, hint: 'Unknown connection failure', address: address, port: port + rescue StandardError => e + @logger&.warn 'Unknown connection failure', hint: e.message, address: address, port: port end false diff --git a/ruby/log-courier/spec/lib/openssl.cnf b/ruby/log-courier/lib/log-courier/rspec/openssl.cnf similarity index 100% rename from ruby/log-courier/spec/lib/openssl.cnf rename to ruby/log-courier/lib/log-courier/rspec/openssl.cnf diff --git a/ruby/log-courier/spec/lib/helpers/common.rb b/ruby/log-courier/lib/log-courier/rspec/spec_helper.rb similarity index 65% rename from ruby/log-courier/spec/lib/helpers/common.rb rename to ruby/log-courier/lib/log-courier/rspec/spec_helper.rb index c487bfde0..7fa91892e 100644 --- a/ruby/log-courier/spec/lib/helpers/common.rb +++ b/ruby/log-courier/lib/log-courier/rspec/spec_helper.rb @@ -13,69 +13,117 @@ # limitations under the License. require 'cabin' +require 'log-courier/client' require 'log-courier/server' +TEMP_PATH = File.join(File.dirname(__FILE__), 'tmp') +EVENT_WAIT_COUNT = 50 +EVENT_WAIT_TIME = 0.5 + # Common helpers for testing both ruby client and the courier -shared_context 'Helpers' do +shared_context 'LogCourier' do before :all do Thread.abort_on_exception = true + FileUtils.rm_r(TEMP_PATH) if File.directory?(TEMP_PATH) + Dir.mkdir(TEMP_PATH) + @ssl_cert = File.open(File.join(TEMP_PATH, 'ssl_cert'), 'w') @ssl_key = File.open(File.join(TEMP_PATH, 'ssl_key'), 'w') @ssl_csr = File.open(File.join(TEMP_PATH, 'ssl_csr'), 'w') # Generate the ssl key - system("openssl req -config spec/lib/openssl.cnf -new -batch -keyout #{@ssl_key.path} -out #{@ssl_csr.path}") + cnf_path = "#{File.dirname(__FILE__)}/openssl.cnf" + system("openssl req -config #{cnf_path} -new -batch -keyout #{@ssl_key.path} -out #{@ssl_csr.path}") system( - "openssl x509 -extfile spec/lib/openssl.cnf -extensions extensions_section -req -days 365 -in #{@ssl_csr.path}" \ + "openssl x509 -extfile #{cnf_path} -extensions extensions_section -req -days 365 -in #{@ssl_csr.path}" \ " -signkey #{@ssl_key.path} -out #{@ssl_cert.path}", ) end after :all do - [@ssl_cert, @ssl_key, @ssl_csr].each do |f| - File.unlink(f.path) if File.file?(f.path) - end + FileUtils.rm_r(TEMP_PATH) if File.directory?(TEMP_PATH) end before :each do - # When we add a file we log it here, so after we can remove them - @files = [] - @event_queue = SizedQueue.new 10_000 + @clients = {} @servers = {} @server_counts = {} @server_threads = {} end after :each do - # Remove any files we created for the test - @files.each(&:close) + unless @servers.length.zero? + id, = @servers.first + raise "Server was not shutdown: #{id}" + end + unless @clients.length.zero? + id, = @clients.first + raise "Client was not shutdown: #{id}" + end + end + + def start_client(**args) + args = { + id: '__default__', + transport: 'tls', + addresses: ['127.0.0.1'], + }.merge!(**args) + + args[:ssl_ca] = @ssl_cert.path if args[:transport] == 'tls' + + id = args[:id] + args[:port] = server_port(id) unless args.key?(:port) + + logger = Cabin::Channel.new + logger.subscribe $stdout + logger['instance'] = "Client #{id}" + logger.level = :debug + + # Reset server for each test + @clients[id] = LogCourier::Client.new( + logger: logger, + **args, + ) + end - @files = [] + def shutdown_client(which = nil) + which = if which.nil? + @clients.keys + else + [which] + end + which.each do |id| + @clients[id].shutdown + @clients.delete id + end + nil end - # A helper that starts a Log Courier server def start_server(**args) args = { id: '__default__', transport: 'tls', }.merge!(**args) + if args[:transport] == 'tls' + args[:ssl_certificate] = @ssl_cert.path + args[:ssl_key] = @ssl_key.path + end + id = args[:id] logger = Cabin::Channel.new logger.subscribe $stdout - logger['instance'] = id + logger['instance'] = "Server #{id}" logger.level = :debug raise 'Server already initialised' if @servers.key?(id) # Reset server for each test @servers[id] = LogCourier::Server.new( - ssl_certificate: @ssl_cert.path, - ssl_key: @ssl_key.path, logger: logger, **args, ) @@ -89,6 +137,7 @@ def start_server(**args) rescue LogCourier::ShutdownSignal 0 end + @servers[id] end # A helper to shutdown a Log Courier server @@ -105,6 +154,7 @@ def shutdown_server(which = nil) @server_counts.delete id @servers.delete id end + nil end # A helper to get the port a server is bound to @@ -117,32 +167,7 @@ def server_count(id = '__default__') @server_counts[id] end - # A helper that creates a new log file - def create_log(type = LogFile, path = nil) - path ||= File.join(TEMP_PATH, 'logs', "log-#{@files.length}") - - # Return a new file for testing, and log it for cleanup afterwards - f = type.new(path) - @files.push(f) - f - end - - # Rename a log file and create a new one in its place - def rotate(logfile, prefix = '') - old_name = logfile.path - - new_name = if prefix == '' - "#{logfile.path}r" - else - File.join(File.dirname(f.path), "#{prefix}#{File.basename(f.path)}r") - end - - logfile.rename new_name - - create_log(logfile.class, old_name) - end - - def receive_and_check(args = {}, &block) + def receive_and_check(args = {}) args = { total: nil, check: true, @@ -181,20 +206,12 @@ def receive_and_check(args = {}, &block) total -= 1 next unless check - if block.nil? - found = @files.find do |f| - next unless f.pending? - - f.logged?({ event: e }.merge!(args)) - end - expect(found).to_not be_nil, "Event received not recognised: #{e}" - else - block.call e - end + yield e end end # Fancy calculation to give a nice "expected" output of expected num of events expect(orig_total - total).to eq orig_total + nil end end diff --git a/ruby/log-courier/lib/log-courier/server.rb b/ruby/log-courier/lib/log-courier/server.rb index 055f8d71e..453fd5794 100644 --- a/ruby/log-courier/lib/log-courier/server.rb +++ b/ruby/log-courier/lib/log-courier/server.rb @@ -17,14 +17,10 @@ require 'log-courier/event_queue' require 'log-courier/protocol' +require 'log-courier/version' require 'multi_json' require 'zlib' -# NativeException in case it is missing -class NativeException - def dummy; end -end - module LogCourier class TimeoutError < StandardError; end diff --git a/ruby/log-courier/lib/log-courier/server_tcp.rb b/ruby/log-courier/lib/log-courier/server_tcp.rb index 1674a743d..a849971cc 100644 --- a/ruby/log-courier/lib/log-courier/server_tcp.rb +++ b/ruby/log-courier/lib/log-courier/server_tcp.rb @@ -137,7 +137,7 @@ def initialize(options = {}) end @logger&.warn 'Ephemeral port allocated', transport: @options[:transport], port: @port if @options[:port].zero? - rescue StandardError, NativeException => e # Until Jruby updated we need NativeException + rescue StandardError => e raise "input/courier: Failed to initialise: #{e}" end end @@ -178,7 +178,7 @@ def run(&block) nil rescue ShutdownSignal nil - rescue StandardError, NativeException => e # Can remove NativeException after 9.2.14.0 JRuby + rescue StandardError => e # Some other unknown problem @logger&.warn e.message, hint: 'Unknown error, shutting down' nil @@ -275,7 +275,7 @@ def run(&block) # Read errors, only action is to shutdown which we'll do in ensure @logger&.warn 'SSL error, connection aborted', error: e.message, peer: @peer nil - rescue IOError, Errno::ECONNRESET => e + rescue IOError, SystemCallError => e # Read errors, only action is to shutdown which we'll do in ensure @logger&.warn 'Connection aborted', error: e.message, peer: @peer nil @@ -287,14 +287,14 @@ def run(&block) # Shutting down @logger&.info 'Server shutting down, closing connection', peer: @peer nil - rescue StandardError, NativeException => e # Can remove NativeException after 9.2.14.0 JRuby + rescue StandardError => e # Some other unknown problem @logger&.warn e.message, hint: 'Unknown error, connection aborted', peer: @peer nil ensure begin @fd.close - rescue OpenSSL::SSL::SSLError, IOError, NativeException + rescue OpenSSL::SSL::SSLError, IOError # Ignore during close end end @@ -376,7 +376,7 @@ def handshake # Minor Version 4 bytes # Patch Version 4 bytes # Client String 4 bytes - data = [0, 2, 7, 2, 'RYLC'].pack('NNNNA4') + data = [0, MAJOR_VERSION, MINOR_VERSION, PATCH_VERSION, 'RYLC'].pack('NNNNA4') send 'VERS', data end diff --git a/ruby/log-courier/lib/log-courier/version.rb b/ruby/log-courier/lib/log-courier/version.rb index 1e9603900..316d7e405 100644 --- a/ruby/log-courier/lib/log-courier/version.rb +++ b/ruby/log-courier/lib/log-courier/version.rb @@ -1,4 +1,7 @@ # Version of LogCourier module LogCourier - VERSION = '2.7.0'.freeze + MAJOR_VERSION = 2 + MINOR_VERSION = 7 + PATCH_VERSION = 3 + VERSION = '2.7.3'.freeze end diff --git a/ruby/log-courier/log-courier.gemspec b/ruby/log-courier/log-courier.gemspec index cbd8ad554..25242ef36 100644 --- a/ruby/log-courier/log-courier.gemspec +++ b/ruby/log-courier/log-courier.gemspec @@ -1,22 +1,23 @@ Gem::Specification.new do |gem| gem.name = 'log-courier' - gem.version = '2.7.2' + gem.version = '2.7.3' gem.description = 'Log Courier library' gem.summary = 'Ruby implementation of the Courier protocol' - gem.homepage = 'https://github.com/driskell/ruby-log-courier' + gem.homepage = 'https://github.com/driskell/log-courier' gem.authors = ['Jason Woods'] gem.email = ['devel@jasonwoods.me.uk'] gem.licenses = ['Apache-2.0'] gem.rubyforge_project = 'nowarning' gem.require_paths = ['lib'] - gem.files = %w( - lib/log-courier/client.rb - lib/log-courier/client_tcp.rb - lib/log-courier/event_queue.rb - lib/log-courier/protocol.rb - lib/log-courier/server.rb - lib/log-courier/server_tcp.rb - ) + gem.files = [ + 'lib/log-courier/client.rb', + 'lib/log-courier/client_tcp.rb', + 'lib/log-courier/event_queue.rb', + 'lib/log-courier/protocol.rb', + 'lib/log-courier/server.rb', + 'lib/log-courier/server_tcp.rb', + 'lib/log-courier/rspec/spec_helper.rb', + ] gem.add_runtime_dependency 'cabin', '~> 0.6' gem.add_runtime_dependency 'multi_json', '~> 1.10' diff --git a/ruby/log-courier/spec/gem_spec.rb b/ruby/log-courier/spec/gem_spec.rb index 2b75c5c76..6ed6b2468 100644 --- a/ruby/log-courier/spec/gem_spec.rb +++ b/ruby/log-courier/spec/gem_spec.rb @@ -12,119 +12,90 @@ # See the License for the specific language governing permissions and # limitations under the License. -require 'cabin' require 'timeout' -require 'lib/common' - -require 'log-courier/client' +require 'log-courier/rspec/spec_helper' describe 'log-courier gem' do - include_context 'Helpers' - - before :all do - @host = Socket.gethostname - end - - def startup(**kwargs) - logger = Cabin::Channel.new - logger.subscribe $stdout - logger.level = :debug - - # Reset server for each test - @client = LogCourier::Client.new( - ssl_ca: @ssl_cert.path, - addresses: ['127.0.0.1'], - port: server_port, - logger: logger, - **kwargs, - ) - end - - def shutdown - @client.shutdown - end + include_context 'LogCourier' it 'should send and receive events with tcp client' do start_server( transport: 'tcp', ) - startup( + client = start_client( transport: 'tcp', ) # Allow 60 seconds Timeout.timeout(60) do 5_000.times do |i| - @client.publish 'message' => "gem line test #{i}", 'host' => @host, 'path' => 'gemfile.log' + client.publish 'message' => "gem line test #{i}", 'host' => 'testing.example.com', 'path' => 'gemfile.log' end - end - # Receive and check - i = 0 - receive_and_check(total: 5_000) do |e| - expect(e['message']).to eq "gem line test #{i}" - expect(e['host']).to eq @host - expect(e['path']).to eq 'gemfile.log' - i += 1 + # Receive and check + i = 0 + receive_and_check(total: 5_000) do |e| + expect(e['message']).to eq "gem line test #{i}" + expect(e['host']).to eq 'testing.example.com' + expect(e['path']).to eq 'gemfile.log' + i += 1 + end end - expect(shutdown).to eq true - + shutdown_client shutdown_server end it 'should send and receive events with tls client' do start_server - startup + client = start_client # Allow 60 seconds Timeout.timeout(60) do 5_000.times do |i| - @client.publish 'message' => "gem line test #{i}", 'host' => @host, 'path' => 'gemfile.log' + client.publish 'message' => "gem line test #{i}", 'host' => 'testing.example.com', 'path' => 'gemfile.log' end - end - # Receive and check - i = 0 - receive_and_check(total: 5_000) do |e| - expect(e['message']).to eq "gem line test #{i}" - expect(e['host']).to eq @host - expect(e['path']).to eq 'gemfile.log' - i += 1 + # Receive and check + i = 0 + receive_and_check(total: 5_000) do |e| + expect(e['message']).to eq "gem line test #{i}" + expect(e['host']).to eq 'testing.example.com' + expect(e['path']).to eq 'gemfile.log' + i += 1 + end end - expect(shutdown).to eq true - + shutdown_client shutdown_server end it 'should send and receive events with tls client that does not handshake' do start_server - startup( + client = start_client( disable_handshake: true, ) # Allow 60 seconds Timeout.timeout(60) do 5_000.times do |i| - @client.publish 'message' => "gem line test #{i}", 'host' => @host, 'path' => 'gemfile.log' + client.publish 'message' => "gem line test #{i}", 'host' => 'testing.example.com', 'path' => 'gemfile.log' end - end - # Receive and check - i = 0 - receive_and_check(total: 5_000) do |e| - expect(e['message']).to eq "gem line test #{i}" - expect(e['host']).to eq @host - expect(e['path']).to eq 'gemfile.log' - i += 1 + # Receive and check + i = 0 + receive_and_check(total: 5_000) do |e| + expect(e['message']).to eq "gem line test #{i}" + expect(e['host']).to eq 'testing.example.com' + expect(e['path']).to eq 'gemfile.log' + i += 1 + end end - expect(shutdown).to eq true - + shutdown_client shutdown_server end @@ -133,26 +104,25 @@ def shutdown disable_handshake: true, ) - startup + client = start_client # Allow 60 seconds Timeout.timeout(60) do 5_000.times do |i| - @client.publish 'message' => "gem line test #{i}", 'host' => @host, 'path' => 'gemfile.log' + client.publish 'message' => "gem line test #{i}", 'host' => 'testing.example.com', 'path' => 'gemfile.log' end - end - # Receive and check - i = 0 - receive_and_check(total: 5_000) do |e| - expect(e['message']).to eq "gem line test #{i}" - expect(e['host']).to eq @host - expect(e['path']).to eq 'gemfile.log' - i += 1 + # Receive and check + i = 0 + receive_and_check(total: 5_000) do |e| + expect(e['message']).to eq "gem line test #{i}" + expect(e['host']).to eq 'testing.example.com' + expect(e['path']).to eq 'gemfile.log' + i += 1 + end end - expect(shutdown).to eq true - + shutdown_client shutdown_server end end diff --git a/ruby/log-courier/spec/lib/common.rb b/ruby/log-courier/spec/lib/common.rb deleted file mode 100644 index 03b23e113..000000000 --- a/ruby/log-courier/spec/lib/common.rb +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright 2014-2021 Jason Woods. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -$LOAD_PATH << File.join(File.dirname(File.dirname(File.dirname(__FILE__))), 'lib') - -require 'fileutils' -require 'lib/helpers/common' -require 'lib/logfile' -require 'socket' - -TEMP_PATH = File.join(File.dirname(File.dirname(__FILE__)), 'tmp') -STARTUP_WAIT_TIME = 2 -EVENT_WAIT_COUNT = 50 -EVENT_WAIT_TIME = 0.5 - -RSpec.configure do |config| - config.before :all do - FileUtils.rm_r(TEMP_PATH) if File.directory?(TEMP_PATH) - Dir.mkdir(TEMP_PATH) - Dir.mkdir(File.join(TEMP_PATH, 'logs')) - puts "\n\n" - end - - config.after :all do - FileUtils.rm_r(TEMP_PATH) if File.directory?(TEMP_PATH) - end - - config.before :each do - puts "\n\n" - end - - config.after :each do - puts "\n\n" - end -end diff --git a/ruby/log-courier/spec/lib/logfile.rb b/ruby/log-courier/spec/lib/logfile.rb deleted file mode 100644 index 695e3a315..000000000 --- a/ruby/log-courier/spec/lib/logfile.rb +++ /dev/null @@ -1,142 +0,0 @@ -# Copyright 2014-2021 Jason Woods. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Helper class that will log to a file and also validate the received entries -class LogFile - attr_reader :count, :path - - def initialize(path) - @file = File.open(path, 'a+') - 2021 - @path = path - @orig_path = path - @count = 0 - - @host = Socket.gethostname - @next = 1 - @gaps = {} - end - - def close - @file.close unless @file.closed? - File.unlink(@path) if File.file?(@path) - end - - def rename(dst) - # Close first and then rename, then reopen, so we work on Windows - @file.close - File.rename @path, dst - @file.reopen dst, 'a+' - @path = dst - end - - def log(num = 1) - num.times do |i| - i += @count + @next - @file.puts @orig_path + " test event #{i}" - @file.flush - end - @count += num - self - end - - def log_partial_start - @file.write @orig_path - @file.flush - self - end - - def log_partial_end - i = @count + @next - @file.puts " test event #{i}" - @file.flush - @count += 1 - self - end - - def skip(num = @count) - @count -= num - @next += num - self - end - - def pending? - @count != 0 || !@gaps.empty? - end - - def logged?(args = {}) - args = { - event: { 'host' => nil }, - check_file: true, - check_order: true, - host: @host, - }.merge!(args) - - event = args[:event] - - return false if event['host'] != args[:host] - return false if args[:check_file] && event['path'] != @orig_path - - if args[:check_order] - # Regular simple test that follows the event number - return false if event['message'] != @orig_path + " test event #{@next}" - elsif event['message'] != @orig_path + " test event #{@next}" - # For when the numbers might not be in order - match = /\A#{Regexp.escape(@orig_path)} test event (?\d+)\z/.match(event['message']) - return false if match.nil? - - number = match['number'].to_i - return false if number >= @next + count - - if @gaps.key?(number) - @gaps[number + 1] = @gaps[number] if @gaps[number] != number - @gaps.delete number - return true - end - fs = nil - fe = nil - @gaps.each do |s, e| - next if number < s || number > e - - fs = s - fe = e - break - end - unless fs.nil? - if number == fs && number == fe - @gaps.delete number - elsif number == fs - @gaps[fs + 1] = fe - @gaps.delete fs - elsif number == fe - @gaps[fs] = fe - 1 - else - @gaps[fs] = number - 1 - @gaps[number + 1] = fe - end - return true - end - return false if number < @next - - @gaps[@next] = number - 1 - @count -= (number + 1) - @next - @next = number + 1 - return true - end - - # Count and return - @count -= 1 - @next += 1 - true - end -end diff --git a/ruby/logstash-input-courier/.gitignore b/ruby/logstash-input-courier/.gitignore index e637ec8c0..af5f2f795 100644 --- a/ruby/logstash-input-courier/.gitignore +++ b/ruby/logstash-input-courier/.gitignore @@ -1,5 +1,3 @@ /pkg -/spec/tmp -/vendor /logstash /logstash.tar.gz diff --git a/ruby/logstash-input-courier/.rubocop.yml b/ruby/logstash-input-courier/.rubocop.yml index 8abc6afad..8c1447f43 100644 --- a/ruby/logstash-input-courier/.rubocop.yml +++ b/ruby/logstash-input-courier/.rubocop.yml @@ -1,5 +1,7 @@ AllCops: NewCops: enable +Layout/LineLength: + Max: 140 Metrics/AbcSize: Enabled: false Metrics/BlockLength: @@ -10,13 +12,11 @@ Metrics/ClassLength: Enabled: false Metrics/CyclomaticComplexity: Enabled: false -Metrics/LineLength: - Max: 140 Metrics/PerceivedComplexity: Enabled: false Metrics/MethodLength: Enabled: false -Naming/Filename: +Naming/FileName: Enabled: false Style/FrozenStringLiteralComment: Enabled: false diff --git a/ruby/logstash-input-courier/.travis.yml b/ruby/logstash-input-courier/.travis.yml deleted file mode 100644 index 4d7b710ff..000000000 --- a/ruby/logstash-input-courier/.travis.yml +++ /dev/null @@ -1,6 +0,0 @@ -language: ruby - -rvm: - - 1.9.3 - - 2.0.0 - - jruby-19mode diff --git a/ruby/logstash-input-courier/Gemfile.lock b/ruby/logstash-input-courier/Gemfile.lock index a7cac599f..af95b3b61 100644 --- a/ruby/logstash-input-courier/Gemfile.lock +++ b/ruby/logstash-input-courier/Gemfile.lock @@ -1,15 +1,15 @@ PATH remote: ../log-courier specs: - log-courier (2.7.2) + log-courier (2.7.3) cabin (~> 0.6) multi_json (~> 1.10) PATH remote: . specs: - logstash-input-courier (2.7.2-java) - log-courier (~> 2.7.2) + logstash-input-courier (2.7.3-java) + log-courier (~> 2.7.3) logstash-codec-plain logstash-core-plugin-api (>= 1.60, <= 2.99) @@ -47,7 +47,7 @@ PATH GEM remote: https://rubygems.org/ specs: - ast (2.4.0) + ast (2.4.2) cabin (0.9.0) chronic_duration (0.10.6) numerizer (~> 0.1.1) @@ -89,7 +89,6 @@ GEM gems (1.2.0) i18n (1.8.10) concurrent-ruby (~> 1.0) - jaro_winkler (1.5.4-java) jrjackson (0.4.11-java) jruby-openssl (0.10.7-java) kramdown (1.14.0) @@ -122,9 +121,9 @@ GEM nio4r (2.5.8-java) numerizer (0.1.1) openssl_pkcs8_pure (0.0.0.2) - parallel (1.19.1) - parser (2.7.0.4) - ast (~> 2.4.0) + parallel (1.21.0) + parser (3.0.2.0) + ast (~> 2.4.1) polyglot (0.3.5) pry (0.14.1-java) coderay (~> 1.1) @@ -137,6 +136,7 @@ GEM rack rainbow (3.0.0) rake (13.0.1) + regexp_parser (2.1.1) rexml (3.2.5) rspec (3.9.0) rspec-core (~> 3.9.0) @@ -153,15 +153,18 @@ GEM rspec-support (3.9.2) rspec-wait (0.0.9) rspec (>= 3, < 4) - rubocop (0.80.1) - jaro_winkler (~> 1.5.1) + rubocop (1.22.2) parallel (~> 1.10) - parser (>= 2.7.0.1) + parser (>= 3.0.0.0) rainbow (>= 2.2.2, < 4.0) + regexp_parser (>= 1.8, < 3.0) rexml + rubocop-ast (>= 1.12.0, < 2.0) ruby-progressbar (~> 1.7) - unicode-display_width (>= 1.4.0, < 1.7) - ruby-progressbar (1.10.1) + unicode-display_width (>= 1.4.0, < 3.0) + rubocop-ast (1.12.0) + parser (>= 3.0.1.1) + ruby-progressbar (1.11.0) ruby2_keywords (0.0.5) rubyzip (1.3.0) sinatra (2.1.0) @@ -176,7 +179,7 @@ GEM tilt (2.0.10) treetop (1.6.11) polyglot (~> 0.3) - unicode-display_width (1.6.1) + unicode-display_width (2.1.0) PLATFORMS java diff --git a/ruby/logstash-input-courier/LICENSE b/ruby/logstash-input-courier/LICENSE deleted file mode 100644 index a67adf1f4..000000000 --- a/ruby/logstash-input-courier/LICENSE +++ /dev/null @@ -1,192 +0,0 @@ -Log Courier project licence: - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - Copyright 2014 Jason Woods. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/ruby/logstash-input-courier/lib/logstash/inputs/courier.rb b/ruby/logstash-input-courier/lib/logstash/inputs/courier.rb index 0b58d7ce4..587a036df 100644 --- a/ruby/logstash-input-courier/lib/logstash/inputs/courier.rb +++ b/ruby/logstash-input-courier/lib/logstash/inputs/courier.rb @@ -60,14 +60,6 @@ class Courier < LogStash::Inputs::Base # Max packet size config :max_packet_size, validate: :number - # The size of the internal queue for each peer - # - # Sent payloads will be dropped when the queue is full - # - # This setting should max the max_pending_payloads Log Courier - # configuration - config :peer_recv_queue, validate: :number - # Add additional fields to events that identity the peer # # This setting is only effective with the tcp and tls transports @@ -85,6 +77,7 @@ def register require 'log-courier/server' @log_courier = LogCourier::Server.new options + nil end # Logstash < 2.0.0 shutdown raises LogStash::ShutdownSignal in this thread @@ -97,11 +90,12 @@ def run(output_queue) decorate event output_queue << event end + nil end - # Logstash >= 2.0.0 shutdown - def stop + def close @log_courier.stop + nil end private @@ -120,6 +114,7 @@ def add_plugin_options(result) ].each do |k| result[k] = send(k) end + result end def add_override_options(result) diff --git a/ruby/logstash-input-courier/logstash-input-courier.gemspec b/ruby/logstash-input-courier/logstash-input-courier.gemspec index fa383fbfe..17653d185 100644 --- a/ruby/logstash-input-courier/logstash-input-courier.gemspec +++ b/ruby/logstash-input-courier/logstash-input-courier.gemspec @@ -1,24 +1,23 @@ # Add platform conditions around java-only dependencies so GitHub dependency chart that is MRI only (I think) still works Gem::Specification.new do |gem| gem.name = 'logstash-input-courier' - gem.version = '2.7.2' + gem.version = '2.7.3' gem.description = 'Courier Input Logstash Plugin' - gem.summary = - 'Receive events from Log Courier and Logstash using the Courier protocol' - gem.homepage = 'https://github.com/driskell/logstash-input-courier' + gem.summary = 'Receive events from Log Courier and Logstash using the Courier protocol' + gem.homepage = 'https://github.com/driskell/log-courier' gem.authors = ['Jason Woods'] gem.email = ['devel@jasonwoods.me.uk'] gem.platform = 'java' if RUBY_PLATFORM == 'java' gem.licenses = ['Apache-2.0'] gem.rubyforge_project = 'nowarning' gem.require_paths = ['lib'] - gem.files = %w[ - lib/logstash/inputs/courier.rb + gem.files = [ + 'lib/logstash/inputs/courier.rb', ] gem.metadata = { 'logstash_plugin' => 'true', 'logstash_group' => 'input' } - gem.add_runtime_dependency 'log-courier', '~> 2.7.2' + gem.add_runtime_dependency 'log-courier', '~> 2.7.3' gem.add_runtime_dependency 'logstash-codec-plain' if RUBY_PLATFORM == 'java' gem.add_runtime_dependency 'logstash-core-plugin-api', '>= 1.60', '<= 2.99' if RUBY_PLATFORM == 'java' end diff --git a/ruby/logstash-input-courier/spec/courier_spec.rb b/ruby/logstash-input-courier/spec/courier_spec.rb deleted file mode 100644 index c93160d56..000000000 --- a/ruby/logstash-input-courier/spec/courier_spec.rb +++ /dev/null @@ -1,31 +0,0 @@ -require 'logstash/devutils/rspec/spec_helper' -require 'logstash/inputs/courier' -require 'lib/rspec_configure' - -describe LogStash::Inputs::Courier do - include_context 'Helpers' - - context 'logstash-input-courier' do - it 'receives connections and generates events' do - @plugin = LogStash::Inputs::Courier.new( - 'port' => 12_345, - 'ssl_certificate' => @ssl_cert.path, - 'ssl_key' => @ssl_key.path, - ) - @plugin.register - @thread = Thread.new do - @plugin.run @event_queue - end - - client = start_client('127.0.0.1', 12_345) - client.publish message: 'This is a test message' - receive_and_check 1 do |e| - expect(e.get('message')).to eq 'This is a test message' - end - - shutdown_client - @plugin.do_stop - @thread.join - end - end -end diff --git a/ruby/logstash-input-courier/spec/gem_spec.rb b/ruby/logstash-input-courier/spec/gem_spec.rb new file mode 100644 index 000000000..05e24443a --- /dev/null +++ b/ruby/logstash-input-courier/spec/gem_spec.rb @@ -0,0 +1,61 @@ +# Copyright 2014-2021 Jason Woods. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require 'logstash/devutils/rspec/spec_helper' +require 'log-courier/rspec/spec_helper' +require 'logstash/inputs/courier' + +describe LogStash::Inputs::Courier do + include_context 'LogCourier' + + context 'logstash-input-courier' do + it 'receives connections and generates events' do + @plugin = LogStash::Inputs::Courier.new( + 'port' => 12_345, + 'ssl_certificate' => @ssl_cert.path, + 'ssl_key' => @ssl_key.path, + ) + @plugin.register + + @thread = Thread.new do + @plugin.run @event_queue + end + + client = start_client( + port: 12_345, + ) + + # Allow 60 seconds + Timeout.timeout(60) do + 5_000.times do |i| + client.publish 'message' => "gem line test #{i}", 'host' => @host, 'path' => 'gemfile.log' + end + + # Receive and check + i = 0 + receive_and_check(total: 5_000) do |e| + e = e.to_hash + expect(e['message']).to eq "gem line test #{i}" + expect(e['host']).to eq @host + expect(e['path']).to eq 'gemfile.log' + i += 1 + end + end + + shutdown_client + @plugin.close + @thread.join + end + end +end diff --git a/ruby/logstash-input-courier/spec/lib/helpers/common.rb b/ruby/logstash-input-courier/spec/lib/helpers/common.rb deleted file mode 100644 index 1cf09527d..000000000 --- a/ruby/logstash-input-courier/spec/lib/helpers/common.rb +++ /dev/null @@ -1,193 +0,0 @@ -# encoding: utf-8 - -# Copyright 2014-2016 Jason Woods. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -require 'thread' -require 'cabin' -require 'log-courier/client' -require 'log-courier/server' - -# Common helpers for testing both ruby client and the courier -shared_context 'Helpers' do - before :all do - Thread.abort_on_exception = true - - @transport = 'tls' - - @ssl_cert = File.open(File.join(TEMP_PATH, 'ssl_cert'), 'w') - @ssl_key = File.open(File.join(TEMP_PATH, 'ssl_key'), 'w') - @ssl_csr = File.open(File.join(TEMP_PATH, 'ssl_csr'), 'w') - - # Generate the ssl key - system('openssl req -config spec/lib/openssl.cnf -new -batch -keyout ' \ - "#{@ssl_key.path} -out #{@ssl_csr.path}") - system('openssl x509 -extfile spec/lib/openssl.cnf -extensions ' \ - "extensions_section -req -days 365 -in #{@ssl_csr.path} -signkey " \ - "#{@ssl_key.path} -out #{@ssl_cert.path}") - end - - after :all do - [@ssl_cert, @ssl_key, @ssl_csr].each do |f| - File.unlink(f.path) if File.file?(f.path) - end - end - - before :each do - # When we add a file we log it here, so after we can remove them - @files = [] - - @event_queue = SizedQueue.new 10_000 - @servers = {} - @server_counts = {} - @server_threads = {} - - @clients = {} - end - - after :each do - shutdown_server - shutdown_client - end - - def create_logger(id, type) - logger = Cabin::Channel.new - logger.subscribe STDOUT - logger['type'] = type.to_s - logger['instance'] = id - logger.level = :debug - logger - end - - def start_server(args = {}) - args = { id: '__default__', transport: nil }.merge!(args) - id = args[:id] - - # Reset server for each test - @servers[id] = LogCourier::Server.new( - transport: args[:transport].nil? ? @transport : args[:transport], - ssl_certificate: @ssl_cert.path, ssl_key: @ssl_key.path, - logger: create_logger(id, :server) - ) - - @server_counts[id] = 0 - @server_threads[id] = start_server_thread(id) - @servers[id] - end - - def start_server_thread(id) - Thread.new do - begin - @servers[id].run do |event| - @server_counts[id] += 1 - @event_queue << event - end - rescue LogCourier::ShutdownSignal - 0 - end - end - end - - # A helper to shutdown a Log Courier server - def shutdown_server(which = nil) - which = if which.nil? - @servers.keys - else - [which] - end - shutdown_multiple_servers(which) - end - - def shutdown_multiple_servers(which) - which.each do |id| - @server_threads[id].raise LogCourier::ShutdownSignal - @server_threads[id].join - @server_threads.delete id - @server_counts.delete id - @servers.delete id - end - end - - def server_port(id = '__default__') - @servers[id].port - end - - def server_count(id = '__default__') - @server_counts[id] - end - - def start_client(address, port, args = {}) - args = { - id: '__default__', - transport: nil - }.merge!(args) - - id = args[:id] - - @clients[id] = LogCourier::Client.new( - transport: args[:transport].nil? ? @transport : args[:transport], - addresses: [address], port: port, ssl_ca: @ssl_cert.path, - logger: create_logger(id, :client) - ) - end - - def shutdown_client(which = nil) - which = if which.nil? - @clients.keys - else - [which] - end - shutdown_multiple_clients(which) - end - - def shutdown_multiple_clients(which) - which.each do |id| - @clients[id].shutdown - @clients.delete id - end - end - - def receive_and_check(total, &block) - orig_total = total - total = receive_and_check_events(total, &block) - - # Fancy calculation to give a nice "expected" output of expected # of events - expect(orig_total - total).to eq orig_total - end - - def receive_and_check_events(total, &block) - waited = 0 - while total > 0 && waited <= EVENT_WAIT_COUNT - if @event_queue.empty? - sleep(EVENT_WAIT_TIME) - waited += 1 - next - end - - waited = 0 - total -= read_events(&block) - end - total - end - - def read_events(&block) - processed = 0 - until @event_queue.empty? - e = @event_queue.pop - processed += 1 - yield e unless block.nil? - end - processed - end -end diff --git a/ruby/logstash-input-courier/spec/lib/openssl.cnf b/ruby/logstash-input-courier/spec/lib/openssl.cnf deleted file mode 100644 index 6ae3eb911..000000000 --- a/ruby/logstash-input-courier/spec/lib/openssl.cnf +++ /dev/null @@ -1,41 +0,0 @@ -[req] -default_bits = 2048 - -encrypt_key = no - -distinguished_name = distinguished_name - -x509_extensions = extensions_section - -[extensions_section] -subjectKeyIdentifier=hash - -authorityKeyIdentifier=keyid:always,issuer:always - -basicConstraints = critical,CA:true - -keyUsage = digitalSignature, keyCertSign, cRLSign, nonRepudiation, digitalSignature, keyEncipherment - -subjectAltName = IP:127.0.0.1 - -[distinguished_name] -countryName = Country Name -countryName_default = GB - -stateOrProvinceName = State -stateOrProvinceName_default = London - -localityName = Locality -localityName_default = London - -organizationName = Organisation -organizationName_default = Log Courier - -organizationalUnitName = Organisational Unit -organizationalUnitName_default = RSpec - -emailAddress = Email Address -emailAddress_default = - -commonName = Common Name -commonName_default = localhost diff --git a/ruby/logstash-input-courier/spec/lib/rspec_configure.rb b/ruby/logstash-input-courier/spec/lib/rspec_configure.rb deleted file mode 100644 index b44ad3322..000000000 --- a/ruby/logstash-input-courier/spec/lib/rspec_configure.rb +++ /dev/null @@ -1,43 +0,0 @@ -# Copyright 2014-2021 Jason Woods. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# $LOAD_PATH << File.join( -# File.dirname(File.dirname(File.dirname(__FILE__))), 'lib') - -require 'lib/helpers/common' - -TEMP_PATH = File.join(File.dirname(File.dirname(__FILE__)), 'tmp') -EVENT_WAIT_COUNT = 50 -EVENT_WAIT_TIME = 0.5 - -RSpec.configure do |config| - config.before :all do - FileUtils.rm_r(TEMP_PATH) if File.directory?(TEMP_PATH) - Dir.mkdir(TEMP_PATH) - Dir.mkdir(File.join(TEMP_PATH, 'logs')) - puts "\n\n" - end - - config.after :all do - FileUtils.rm_r(TEMP_PATH) if File.directory?(TEMP_PATH) - end - - config.before :each do - puts "\n\n" - end - - config.after :each do - puts "\n\n" - end -end diff --git a/ruby/logstash-input-courier/version.txt b/ruby/logstash-input-courier/version.txt deleted file mode 100644 index 4dae2985b..000000000 --- a/ruby/logstash-input-courier/version.txt +++ /dev/null @@ -1 +0,0 @@ -1.10.1 diff --git a/ruby/logstash-output-courier/.gitignore b/ruby/logstash-output-courier/.gitignore index bd6bea3fa..af5f2f795 100644 --- a/ruby/logstash-output-courier/.gitignore +++ b/ruby/logstash-output-courier/.gitignore @@ -1,5 +1,3 @@ -/.bundle -/.vagrant /pkg -/spec/tmp -/vendor +/logstash +/logstash.tar.gz diff --git a/ruby/logstash-output-courier/.rspec b/ruby/logstash-output-courier/.rspec new file mode 100644 index 000000000..16f9cdb01 --- /dev/null +++ b/ruby/logstash-output-courier/.rspec @@ -0,0 +1,2 @@ +--color +--format documentation diff --git a/ruby/logstash-output-courier/.rubocop.yml b/ruby/logstash-output-courier/.rubocop.yml new file mode 100644 index 000000000..8c1447f43 --- /dev/null +++ b/ruby/logstash-output-courier/.rubocop.yml @@ -0,0 +1,30 @@ +AllCops: + NewCops: enable +Layout/LineLength: + Max: 140 +Metrics/AbcSize: + Enabled: false +Metrics/BlockLength: + Enabled: false +Metrics/BlockNesting: + Enabled: false +Metrics/ClassLength: + Enabled: false +Metrics/CyclomaticComplexity: + Enabled: false +Metrics/PerceivedComplexity: + Enabled: false +Metrics/MethodLength: + Enabled: false +Naming/FileName: + Enabled: false +Style/FrozenStringLiteralComment: + Enabled: false +Style/SymbolArray: + Enabled: false +Style/TrailingCommaInArguments: + EnforcedStyleForMultiline: consistent_comma +Style/TrailingCommaInArrayLiteral: + EnforcedStyleForMultiline: consistent_comma +Style/TrailingCommaInHashLiteral: + EnforcedStyleForMultiline: consistent_comma diff --git a/ruby/logstash-output-courier/.travis.yml b/ruby/logstash-output-courier/.travis.yml deleted file mode 100644 index 4d7b710ff..000000000 --- a/ruby/logstash-output-courier/.travis.yml +++ /dev/null @@ -1,6 +0,0 @@ -language: ruby - -rvm: - - 1.9.3 - - 2.0.0 - - jruby-19mode diff --git a/ruby/logstash-output-courier/Gemfile b/ruby/logstash-output-courier/Gemfile index 324d547ad..325c5fe61 100644 --- a/ruby/logstash-output-courier/Gemfile +++ b/ruby/logstash-output-courier/Gemfile @@ -2,12 +2,16 @@ source 'https://rubygems.org' gemspec -gem 'rake', :group => 'test' -gem 'rubocop', :group => 'test' +group :development do + gem 'rake' + gem 'rubocop' + gem 'logstash-devutils' + gem 'logstash-core', :path => './logstash/logstash-core' + gem 'logstash-core-plugin-api', :path => "./logstash/logstash-core-plugin-api" + gem 'log-courier', :path => '../log-courier' +end -# Profiler for MRI -gem 'ruby-prof', '~> 0.15', :platforms => :mri, :group => 'test' - -# Tests -gem 'rspec', '~> 3.1', :group => 'test' -gem 'rspec-core', '~> 3.1', :group => 'test' +group :test do + gem 'rspec', '~> 3.1' + gem 'rspec-core', '~> 3.1' +end diff --git a/ruby/logstash-output-courier/Gemfile.lock b/ruby/logstash-output-courier/Gemfile.lock index dcb02bd2e..f6de3a64f 100644 --- a/ruby/logstash-output-courier/Gemfile.lock +++ b/ruby/logstash-output-courier/Gemfile.lock @@ -1,63 +1,143 @@ +PATH + remote: ../log-courier + specs: + log-courier (2.7.3) + cabin (~> 0.6) + multi_json (~> 1.10) + PATH remote: . specs: - logstash-output-courier (1.8.4.pre.1.pre.g45888c8) - log-courier (~> 1.9) - logstash-core (>= 1.4, < 3) + logstash-output-courier (2.7.3-java) + log-courier (~> 2.7.3) + logstash-codec-plain + logstash-core-plugin-api (>= 1.60, <= 2.99) + +PATH + remote: logstash/logstash-core-plugin-api + specs: + logstash-core-plugin-api (2.1.16-java) + logstash-core (= 7.7.0) + +PATH + remote: logstash/logstash-core + specs: + logstash-core (7.7.0-java) + chronic_duration (~> 0.10) + clamp (~> 0.6) + concurrent-ruby (~> 1) + elasticsearch (~> 5) + filesize (~> 0.2) + gems (~> 1) + i18n (~> 1) + jrjackson (= 0.4.11) + jruby-openssl (~> 0.10) + manticore (~> 0.6) + minitar (~> 0.8) + mustermann (~> 1.0.3) + pry (~> 0.12) + puma (~> 4) + rack (~> 2) + rubyzip (~> 1) + sinatra (~> 2) + stud (~> 0.0.19) + thread_safe (~> 0.3.6) + treetop (~> 1) GEM remote: https://rubygems.org/ specs: - ast (2.1.0) - astrolabe (1.3.1) - parser (~> 2.2) - cabin (0.7.2) + ast (2.4.2) + cabin (0.9.0) + chronic_duration (0.10.6) + numerizer (~> 0.1.1) clamp (0.6.5) - coderay (1.1.0) - concurrent-ruby (0.9.2-java) + coderay (1.1.3) + concurrent-ruby (1.1.9) diff-lcs (1.2.5) - ffi (1.9.10-java) - ffi-rzmq (2.0.4) - ffi-rzmq-core (>= 1.0.1) - ffi-rzmq-core (1.0.4) - ffi (~> 1.9) - filesize (0.0.4) - gems (0.8.3) - i18n (0.6.9) - jrjackson (0.3.7) - jruby-openssl (0.9.12-java) - log-courier (1.9.0) - cabin (~> 0.6) - ffi-rzmq (~> 2.0) - multi_json (~> 1.10) - logstash-core (2.0.0-java) - cabin (~> 0.7.0) - clamp (~> 0.6.5) - concurrent-ruby (~> 0.9.1) - filesize (= 0.0.4) - gems (~> 0.8.3) - i18n (= 0.6.9) - jrjackson (~> 0.3.6) - jruby-openssl (>= 0.9.11) - minitar (~> 0.5.4) - pry (~> 0.10.1) - stud (~> 0.0.19) - thread_safe (~> 0.3.5) - treetop (< 1.5.0) - method_source (0.8.2) - minitar (0.5.4) - multi_json (1.11.2) - parser (2.2.3.0) - ast (>= 1.1, < 3.0) + elasticsearch (5.0.5) + elasticsearch-api (= 5.0.5) + elasticsearch-transport (= 5.0.5) + elasticsearch-api (5.0.5) + multi_json + elasticsearch-transport (5.0.5) + faraday + multi_json + faraday (1.8.0) + faraday-em_http (~> 1.0) + faraday-em_synchrony (~> 1.0) + faraday-excon (~> 1.1) + faraday-httpclient (~> 1.0.1) + faraday-net_http (~> 1.0) + faraday-net_http_persistent (~> 1.1) + faraday-patron (~> 1.0) + faraday-rack (~> 1.0) + multipart-post (>= 1.2, < 3) + ruby2_keywords (>= 0.0.4) + faraday-em_http (1.0.0) + faraday-em_synchrony (1.0.0) + faraday-excon (1.1.0) + faraday-httpclient (1.0.1) + faraday-net_http (1.0.1) + faraday-net_http_persistent (1.2.0) + faraday-patron (1.0.0) + faraday-rack (1.0.0) + ffi (1.15.4-java) + filesize (0.2.0) + fivemat (1.3.7) + gem_publisher (1.5.0) + gems (1.2.0) + i18n (1.8.10) + concurrent-ruby (~> 1.0) + jrjackson (0.4.11-java) + jruby-openssl (0.10.7-java) + kramdown (1.14.0) + logstash-codec-plain (3.1.0) + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-mixin-ecs_compatibility_support (~> 1.3) + logstash-mixin-event_support (~> 1.0) + logstash-devutils (2.2.1-java) + fivemat + gem_publisher + kramdown (= 1.14.0) + logstash-codec-plain + logstash-core (>= 6.3) + minitar + rake + rspec (~> 3.0) + rspec-wait + stud (>= 0.0.20) + logstash-mixin-ecs_compatibility_support (1.3.0-java) + logstash-core (>= 6.0.0) + logstash-mixin-event_support (1.0.1-java) + logstash-core (>= 6.8) + manticore (0.7.1-java) + openssl_pkcs8_pure + method_source (1.0.0) + minitar (0.9) + multi_json (1.15.0) + multipart-post (2.1.1) + mustermann (1.0.3) + nio4r (2.5.8-java) + numerizer (0.1.1) + openssl_pkcs8_pure (0.0.0.2) + parallel (1.21.0) + parser (3.0.2.0) + ast (~> 2.4.1) polyglot (0.3.5) - powerpack (0.1.1) - pry (0.10.3-java) - coderay (~> 1.1.0) - method_source (~> 0.8.1) - slop (~> 3.4) + pry (0.14.1-java) + coderay (~> 1.1) + method_source (~> 1.0) spoon (~> 0.0) - rainbow (2.0.0) + puma (4.3.10-java) + nio4r (~> 2.0) + rack (2.2.3) + rack-protection (2.1.0) + rack + rainbow (3.0.0) rake (10.4.2) + regexp_parser (2.1.1) + rexml (3.2.5) rspec (3.3.0) rspec-core (~> 3.3.0) rspec-expectations (~> 3.3.0) @@ -71,32 +151,50 @@ GEM diff-lcs (>= 1.2.0, < 2.0) rspec-support (~> 3.3.0) rspec-support (3.3.0) - rubocop (0.35.0) - astrolabe (~> 1.3) - parser (>= 2.2.3.0, < 3.0) - powerpack (~> 0.1) - rainbow (>= 1.99.1, < 3.0) + rspec-wait (0.0.9) + rspec (>= 3, < 4) + rubocop (1.22.2) + parallel (~> 1.10) + parser (>= 3.0.0.0) + rainbow (>= 2.2.2, < 4.0) + regexp_parser (>= 1.8, < 3.0) + rexml + rubocop-ast (>= 1.12.0, < 2.0) ruby-progressbar (~> 1.7) - ruby-progressbar (1.7.5) - slop (3.6.0) - spoon (0.0.4) + unicode-display_width (>= 1.4.0, < 3.0) + rubocop-ast (1.12.0) + parser (>= 3.0.1.1) + ruby-progressbar (1.11.0) + ruby2_keywords (0.0.5) + rubyzip (1.3.0) + sinatra (2.1.0) + mustermann (~> 1.0) + rack (~> 2.2) + rack-protection (= 2.1.0) + tilt (~> 2.0) + spoon (0.0.6) ffi - stud (0.0.22) - thread_safe (0.3.5-java) - treetop (1.4.15) - polyglot - polyglot (>= 0.3.1) + stud (0.0.23) + thread_safe (0.3.6-java) + tilt (2.0.10) + treetop (1.6.11) + polyglot (~> 0.3) + unicode-display_width (2.1.0) PLATFORMS java + universal-java-14 DEPENDENCIES + log-courier! + logstash-core! + logstash-core-plugin-api! + logstash-devutils logstash-output-courier! rake rspec (~> 3.1) rspec-core (~> 3.1) rubocop - ruby-prof (~> 0.15) BUNDLED WITH - 1.10.6 + 2.2.29 diff --git a/ruby/logstash-output-courier/LICENSE b/ruby/logstash-output-courier/LICENSE deleted file mode 100644 index a67adf1f4..000000000 --- a/ruby/logstash-output-courier/LICENSE +++ /dev/null @@ -1,192 +0,0 @@ -Log Courier project licence: - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - Copyright 2014 Jason Woods. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/ruby/logstash-output-courier/Rakefile b/ruby/logstash-output-courier/Rakefile index d889eca4e..c78b0e00c 100644 --- a/ruby/logstash-output-courier/Rakefile +++ b/ruby/logstash-output-courier/Rakefile @@ -4,26 +4,18 @@ require 'rubygems/package_task' gemspec = Gem::Specification.load('logstash-output-courier.gemspec') Gem::PackageTask.new(gemspec).define -task :default => [:deploy] do +task :default => [:install] do require 'rspec/core/rake_task' RSpec::Core::RakeTask.new(:spec) Rake::Task[:spec].invoke end -task :deploy do - Bundler.with_clean_env do - sh 'bundle install --deployment' - end +task :install do + sh 'bundle install --jobs 4 --retry 3' end -task :update do - Bundler.with_clean_env do - sh 'bundle install --no-deployment --path vendor/bundle' - end -end - -task :release => [:package] do - sh "gem push pkg/logstash-output-courier-#{gemspec.version}.gem" +task release: [:package] do + sh "gem push pkg/logstash-output-courier-#{gemspec.version}-java.gem" end task :clean do diff --git a/ruby/logstash-output-courier/download_logstash.sh b/ruby/logstash-output-courier/download_logstash.sh new file mode 100755 index 000000000..808d62d10 --- /dev/null +++ b/ruby/logstash-output-courier/download_logstash.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +set -eo pipefail + +SOURCE="https://artifacts.elastic.co/downloads/logstash/logstash-7.7.0.tar.gz" +CHECKSUM="970740adc47551d7967b9841cc39d15f2cbdcd46c2fee1f84b5688fac266fdcd2202cbb10d3a10cf3768606f693ed2e4fc79e91d293a3295083718bafaa7bc9d" + +echo "===== Downloading =====" +wget --progress dot:giga "$SOURCE" -O logstash.tar.gz +echo "===== Verifying checksum =====" +shasum -a 512 -c <(echo "$CHECKSUM logstash.tar.gz") +echo "===== Extracting =====" +mkdir -p logstash +tar -C logstash --strip-components 1 -xzf logstash.tar.gz +echo "===== Testing =====" +./logstash/bin/logstash --version +./logstash/vendor/jruby/bin/jruby --version +echo "===== Completed =====" diff --git a/ruby/logstash-output-courier/lib/logstash/outputs/courier.rb b/ruby/logstash-output-courier/lib/logstash/outputs/courier.rb index 2bdb72be8..a52dc06f9 100644 --- a/ruby/logstash-output-courier/lib/logstash/outputs/courier.rb +++ b/ruby/logstash-output-courier/lib/logstash/outputs/courier.rb @@ -1,6 +1,4 @@ -# encoding: utf-8 - -# Copyright 2014 Jason Woods. +# Copyright 2014-2021 Jason Woods. # # This file is a modification of code from Logstash Forwarder. # Copyright 2012-2013 Jordan Sissel and contributors. @@ -26,17 +24,15 @@ module Outputs class Courier < LogStash::Outputs::Base config_name 'courier' - # Compatibility with Logstash 1.4 requires milestone - if Gem::Version.new(LOGSTASH_VERSION) < Gem::Version.new('1.5.0') - milestone 2 - end - # The list of addresses Log Courier should send to - config :hosts, validate: :array, required: true + config :addresses, validate: :array, required: true # The port to connect to config :port, validate: :number, required: true + # The transport type to use + config :transport, validate: :string, default: 'tls' + # CA certificate for validation of the server config :ssl_ca, validate: :path, required: true @@ -49,35 +45,32 @@ class Courier < LogStash::Outputs::Base # SSL key passphrase to use config :ssl_key_passphrase, validate: :password + # Set minimum TLS version + config :min_tls_version, validate: :number, default: 1.2 + # Maximum number of events to spool before forcing a flush config :spool_size, validate: :number, default: 1024 # Maximum time to wait for a full spool before forcing a flush config :idle_timeout, validate: :number, default: 5 - public - def register @logger.info 'Starting courier output' require 'log-courier/client' @client = LogCourier::Client.new(options) + nil end def receive(event) return unless output?(event) - @client.publish event.to_hash - end - # Logstash < 2.0.0 shutdown - def teardown - close + @client.publish event.to_hash + nil end - # Logstash >= 2.0.0 shutdown def close @client.shutdown - finished nil end @@ -85,14 +78,12 @@ def close def options result = {} - [ - :logger, :addresses, :port, :ssl_ca, :ssl_certificate, :ssl_key, - :ssl_key_passphrase, :spool_size, :idle_timeout + :logger, :addresses, :port, :transport, :ssl_ca, :ssl_certificate, :ssl_key, + :ssl_key_passphrase, :spool_size, :idle_timeout, :min_tls_version, ].each do |k| result[k] = send(k) end - result end diff --git a/ruby/logstash-output-courier/logstash-output-courier.gemspec b/ruby/logstash-output-courier/logstash-output-courier.gemspec index d83c0b74e..d1faa793f 100644 --- a/ruby/logstash-output-courier/logstash-output-courier.gemspec +++ b/ruby/logstash-output-courier/logstash-output-courier.gemspec @@ -1,32 +1,23 @@ -# Pull version from git if we're cloned (git command sure to exist) -# Otherwise, if in an archive, use version.txt, which is the last stable version -if File.directory? '.git' - version = \ - `git describe | sed 's/-\([0-9][0-9]*\)-\([0-9a-z][0-9a-z]*\)$/-\1.\2/g'` - version.sub!(/^v/, '') -else - version = IO.read 'version.txt' -end - -version.chomp! - +# Add platform conditions around java-only dependencies so GitHub dependency chart that is MRI only (I think) still works Gem::Specification.new do |gem| gem.name = 'logstash-output-courier' - gem.version = version + gem.version = '2.7.3' gem.description = 'Courier Output Logstash Plugin' gem.summary = 'Transmit events from one Logstash instance to another using the Courier protocol' - gem.homepage = 'https://github.com/driskell/logstash-output-courier' + gem.homepage = 'https://github.com/driskell/log-courier' gem.authors = ['Jason Woods'] gem.email = ['devel@jasonwoods.me.uk'] - gem.licenses = ['Apache'] + gem.platform = 'java' if RUBY_PLATFORM == 'java' + gem.licenses = ['Apache-2.0'] gem.rubyforge_project = 'nowarning' gem.require_paths = ['lib'] - gem.files = %w( - lib/logstash/outputs/courier.rb - ) + gem.files = [ + 'lib/logstash/outputs/courier.rb', + ] gem.metadata = { 'logstash_plugin' => 'true', 'logstash_group' => 'output' } - gem.add_runtime_dependency 'logstash-core', '>= 1.4', '< 3' - gem.add_runtime_dependency 'log-courier', '~> 1.9' + gem.add_runtime_dependency 'log-courier', '~> 2.7.3' + gem.add_runtime_dependency 'logstash-codec-plain' if RUBY_PLATFORM == 'java' + gem.add_runtime_dependency 'logstash-core-plugin-api', '>= 1.60', '<= 2.99' if RUBY_PLATFORM == 'java' end diff --git a/ruby/logstash-output-courier/spec/gem_spec.rb b/ruby/logstash-output-courier/spec/gem_spec.rb new file mode 100644 index 000000000..95c2fb2d8 --- /dev/null +++ b/ruby/logstash-output-courier/spec/gem_spec.rb @@ -0,0 +1,59 @@ +# Copyright 2014-2021 Jason Woods. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require 'logstash/devutils/rspec/spec_helper' +require 'log-courier/rspec/spec_helper' +require 'logstash/outputs/courier' + +describe LogStash::Outputs::Courier do + include_context 'LogCourier' + + before :all do + LogStash::Logging::Logger.configure_logging 'debug' + end + + context 'logstash-output-courier' do + it 'sends events' do + @plugin = LogStash::Outputs::Courier.new( + 'hosts' => ['127.0.0.1'], + 'port' => 12_345, + 'ssl_ca' => @ssl_cert.path, + ) + @plugin.register + + start_server( + port: 12_345, + ) + + # Allow 60 seconds + Timeout.timeout(60) do + 5_000.times do |i| + @plugin.receive 'message' => "gem line test #{i}", 'host' => @host, 'path' => 'gemfile.log' + end + + # Receive and check + i = 0 + receive_and_check(total: 5_000) do |e| + expect(e['message']).to eq "gem line test #{i}" + expect(e['host']).to eq @host + expect(e['path']).to eq 'gemfile.log' + i += 1 + end + end + + @plugin.close + shutdown_server + end + end +end diff --git a/ruby/logstash-output-courier/spec/lib/common.rb b/ruby/logstash-output-courier/spec/lib/common.rb deleted file mode 100644 index 4b5d3f877..000000000 --- a/ruby/logstash-output-courier/spec/lib/common.rb +++ /dev/null @@ -1,46 +0,0 @@ -# encoding: utf-8 - -# Copyright 2014 Jason Woods. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -$LOAD_PATH << File.join(File.dirname(File.dirname(File.dirname(__FILE__))), 'lib') - -require 'lib/helpers/common' -require 'lib/logfile' - -TEMP_PATH = File.join(File.dirname(File.dirname(__FILE__)), 'tmp') -STARTUP_WAIT_TIME = 2 -EVENT_WAIT_COUNT = 50 -EVENT_WAIT_TIME = 0.5 - -RSpec.configure do |config| - config.before :all do - FileUtils.rm_r(TEMP_PATH) if File.directory?(TEMP_PATH) - Dir.mkdir(TEMP_PATH) - Dir.mkdir(File.join(TEMP_PATH, 'logs')) - puts "\n\n" - end - - config.after :all do - FileUtils.rm_r(TEMP_PATH) if File.directory?(TEMP_PATH) - end - - config.before :each do - puts "\n\n" - end - - config.after :each do - puts "\n\n" - end -end diff --git a/ruby/logstash-output-courier/spec/lib/helpers/common.rb b/ruby/logstash-output-courier/spec/lib/helpers/common.rb deleted file mode 100644 index 5ae29a277..000000000 --- a/ruby/logstash-output-courier/spec/lib/helpers/common.rb +++ /dev/null @@ -1,209 +0,0 @@ -# encoding: utf-8 - -# Copyright 2014 Jason Woods. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -require 'thread' -require 'cabin' -require 'log-courier/server' - -# Common helpers for testing both ruby client and the courier -shared_context 'Helpers' do - before :all do - Thread.abort_on_exception = true - - @transport = 'tls' - - @ssl_cert = File.open(File.join(TEMP_PATH, 'ssl_cert'), 'w') - @ssl_key = File.open(File.join(TEMP_PATH, 'ssl_key'), 'w') - @ssl_csr = File.open(File.join(TEMP_PATH, 'ssl_csr'), 'w') - - # Generate the ssl key - system("openssl req -config spec/lib/openssl.cnf -new -batch -keyout #{@ssl_key.path} -out #{@ssl_csr.path}") - system("openssl x509 -extfile spec/lib/openssl.cnf -extensions extensions_section -req -days 365 -in #{@ssl_csr.path} -signkey #{@ssl_key.path} -out #{@ssl_cert.path}") - end - - after :all do - [@ssl_cert, @ssl_key, @ssl_csr].each do |f| - File.unlink(f.path) if File.file?(f.path) - end - end - - before :each do - # When we add a file we log it here, so after we can remove them - @files = [] - - @event_queue = SizedQueue.new 10_000 - - @servers = {} - @server_counts = {} - @server_threads = {} - - start_server - end - - after :each do - # Remove any files we created for the test - @files.each do |f| - f.close - end - - @files = [] - - shutdown_server - end - - # A helper that starts a Log Courier server - def start_server(args = {}) - args = { - id: '__default__', - transport: nil - }.merge!(args) - - id = args[:id] - - logger = Cabin::Channel.new - logger.subscribe STDOUT - logger['instance'] = id - logger.level = :debug - - raise 'Server already initialised' if @servers.key?(id) - - # Reset server for each test - @servers[id] = LogCourier::Server.new( - transport: args[:transport].nil? ? @transport : args[:transport], - ssl_certificate: @ssl_cert.path, - ssl_key: @ssl_key.path, - curve_secret_key: '1XQgjDjkw?YP=$f61HKe%g+AEbe 0 && waited <= EVENT_WAIT_COUNT - if @event_queue.length == 0 - sleep(EVENT_WAIT_TIME) - waited += 1 - next - end - - waited = 0 - while @event_queue.length != 0 - e = @event_queue.pop - total -= 1 - next unless check - if block.nil? - found = @files.find do |f| - next unless f.pending? - f.logged?({event: e}.merge!(args)) - end - expect(found).to_not be_nil, "Event received not recognised: #{e}" - else - block.call e - end - end - end - - # Fancy calculation to give a nice "expected" output of expected num of events - expect(orig_total - total).to eq orig_total - end -end diff --git a/ruby/logstash-output-courier/spec/lib/logfile.rb b/ruby/logstash-output-courier/spec/lib/logfile.rb deleted file mode 100644 index a2e93b975..000000000 --- a/ruby/logstash-output-courier/spec/lib/logfile.rb +++ /dev/null @@ -1,145 +0,0 @@ -# encoding: utf-8 - -# Copyright 2014 Jason Woods. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Helper class that will log to a file and also validate the received entries -class LogFile - attr_reader :count - attr_reader :path - - def initialize(path) - @file = File.open(path, 'a+') - @path = path - @orig_path = path - @count = 0 - - @host = Socket.gethostname - @next = 1 - @gaps = {} - end - - def close - @file.close unless @file.closed? - File.unlink(@path) if File.file?(@path) - end - - def rename(dst) - # Close first and then rename, then reopen, so we work on Windows - @file.close - File.rename @path, dst - @file.reopen dst, 'a+' - @path = dst - end - - def log(num = 1) - num.times do |i| - i += @count + @next - @file.puts @orig_path + " test event #{i}" - @file.flush - end - @count += num - self - end - - def log_partial_start - @file.write @orig_path - @file.flush - self - end - - def log_partial_end - i = @count + @next - @file.puts " test event #{i}" - @file.flush - @count += 1 - self - end - - def skip(num = @count) - @count -= num - @next += num - self - end - - def pending? - @count != 0 || !@gaps.empty? - end - - def logged?(args = {}) - args = { - event: { 'host' => nil }, - check_file: true, - check_order: true, - host: @host - }.merge!(args) - - event = args[:event] - - return false if event['host'] != args[:host] - return false if args[:check_file] && event['path'] != @orig_path - - if args[:check_order] - # Regular simple test that follows the event number - return false if event['message'] != @orig_path + " test event #{@next}" - else - # For when the numbers might not be in order - if event['message'] != @orig_path + " test event #{@next}" - match = /\A#{Regexp.escape(@orig_path)} test event (?\d+)\z/.match(event['message']) - return false if match.nil? - number = match['number'].to_i - return false if number >= @next + count - if @gaps.key?(number) - if @gaps[number] != number - @gaps[number + 1] = @gaps[number] - end - @gaps.delete number - return true - end - fs = nil - fe = nil - @gaps.each do |s, e| - next if number < s || number > e - fs = s - fe = e - break - end - unless fs.nil? - if number == fs && number == fe - @gaps.delete number - elsif number == fs - @gaps[fs + 1] = fe - @gaps.delete fs - elsif number == fe - @gaps[fs] = fe - 1 - else - @gaps[fs] = number - 1 - @gaps[number + 1] = fe - end - return true - end - return false if number < @next - @gaps[@next] = number - 1 - @count -= (number + 1) - @next - @next = number + 1 - return true - end - end - - # Count and return - @count -= 1 - @next += 1 - true - end -end diff --git a/ruby/logstash-output-courier/spec/lib/openssl.cnf b/ruby/logstash-output-courier/spec/lib/openssl.cnf deleted file mode 100644 index 6ae3eb911..000000000 --- a/ruby/logstash-output-courier/spec/lib/openssl.cnf +++ /dev/null @@ -1,41 +0,0 @@ -[req] -default_bits = 2048 - -encrypt_key = no - -distinguished_name = distinguished_name - -x509_extensions = extensions_section - -[extensions_section] -subjectKeyIdentifier=hash - -authorityKeyIdentifier=keyid:always,issuer:always - -basicConstraints = critical,CA:true - -keyUsage = digitalSignature, keyCertSign, cRLSign, nonRepudiation, digitalSignature, keyEncipherment - -subjectAltName = IP:127.0.0.1 - -[distinguished_name] -countryName = Country Name -countryName_default = GB - -stateOrProvinceName = State -stateOrProvinceName_default = London - -localityName = Locality -localityName_default = London - -organizationName = Organisation -organizationName_default = Log Courier - -organizationalUnitName = Organisational Unit -organizationalUnitName_default = RSpec - -emailAddress = Email Address -emailAddress_default = - -commonName = Common Name -commonName_default = localhost diff --git a/ruby/logstash-output-courier/version.txt b/ruby/logstash-output-courier/version.txt deleted file mode 100644 index f8e233b27..000000000 --- a/ruby/logstash-output-courier/version.txt +++ /dev/null @@ -1 +0,0 @@ -1.9.0