From f9c36295b141085e000fddc1714b32d1d29384f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcos=20Ag=C3=BCero?= Date: Thu, 5 Sep 2019 12:04:42 +0200 Subject: [PATCH 1/6] Updated to API version 7.3 * Added support for LEEF 2.0 and custom field delimiter * Code synced with the newer versions of CEF plugins * Updated CI to logstash standars * Some more things, sure. --- .travis.yml | 24 ++- ci/build.sh | 21 ++ ci/setup.sh | 26 +++ docker-compose.yml | 2 +- lib/logstash/codecs/leef.rb | 266 ++++++++++++++++++------- logstash-codec-leef-3.0.1-java.gem.old | Bin 13312 -> 0 bytes spec/codecs/leef_spec.rb | 243 ++++++++++++++-------- 7 files changed, 414 insertions(+), 168 deletions(-) create mode 100644 ci/build.sh create mode 100644 ci/setup.sh delete mode 100644 logstash-codec-leef-3.0.1-java.gem.old diff --git a/.travis.yml b/.travis.yml index 97a2f46..dc96273 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,9 +1,21 @@ +--- sudo: false language: ruby cache: bundler -jdk: - - oraclejdk8 -rvm: - - jruby-1.7.25 -script: - - bundle exec rspec spec +matrix: + include: + - rvm: jruby-9.1.13.0 + env: LOGSTASH_BRANCH=master + - rvm: jruby-9.1.13.0 + env: LOGSTASH_BRANCH=7.0 + - rvm: jruby-9.1.13.0 + env: LOGSTASH_BRANCH=6.7 + - rvm: jruby-9.1.13.0 + env: LOGSTASH_BRANCH=6.6 + - rvm: jruby-1.7.27 + env: LOGSTASH_BRANCH=5.6 + fast_finish: true +install: true +script: ci/build.sh +jdk: oraclejdk8 +before_install: gem install bundler -v '< 2' diff --git a/ci/build.sh b/ci/build.sh new file mode 100644 index 0000000..06caffd --- /dev/null +++ b/ci/build.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# version: 1 +######################################################## +# +# AUTOMATICALLY GENERATED! DO NOT EDIT +# +######################################################## +set -e + +echo "Starting build process in: `pwd`" +source ./ci/setup.sh + +if [[ -f "ci/run.sh" ]]; then + echo "Running custom build script in: `pwd`/ci/run.sh" + source ./ci/run.sh +else + echo "Running default build scripts in: `pwd`/ci/build.sh" + bundle install + bundle exec rake vendor + bundle exec rspec spec +fi diff --git a/ci/setup.sh b/ci/setup.sh new file mode 100644 index 0000000..835fa43 --- /dev/null +++ b/ci/setup.sh @@ -0,0 +1,26 @@ +#!/bin/bash +# version: 1 +######################################################## +# +# AUTOMATICALLY GENERATED! DO NOT EDIT +# +######################################################## +set -e +if [ "$LOGSTASH_BRANCH" ]; then + echo "Building plugin using Logstash source" + BASE_DIR=`pwd` + echo "Checking out branch: $LOGSTASH_BRANCH" + git clone -b $LOGSTASH_BRANCH https://github.com/elastic/logstash.git ../../logstash --depth 1 + printf "Checked out Logstash revision: %s\n" "$(git -C ../../logstash rev-parse HEAD)" + cd ../../logstash + echo "Building plugins with Logstash version:" + cat versions.yml + echo "---" + # We need to build the jars for that specific version + echo "Running gradle assemble in: `pwd`" + ./gradlew assemble + cd $BASE_DIR + export LOGSTASH_SOURCE=1 +else + echo "Building plugin using released gems on rubygems" +fi diff --git a/docker-compose.yml b/docker-compose.yml index f5b0cd1..b79a7e8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,5 @@ devenv: - image: hpess/devenv-jruby:master + image: jruby:latest entrypoint: /bin/bash volumes: - ./:/storage diff --git a/lib/logstash/codecs/leef.rb b/lib/logstash/codecs/leef.rb index b46d430..86a4085 100644 --- a/lib/logstash/codecs/leef.rb +++ b/lib/logstash/codecs/leef.rb @@ -1,5 +1,6 @@ # encoding: utf-8 require "logstash/codecs/base" +require "logstash/util/charset" require "json" require "socket" require "time" @@ -10,12 +11,16 @@ class LogStash::Codecs::LEEF < LogStash::Codecs::Base config_name "leef" - - # Field is to disable the leef header, which can be helpful for generating WCollect messages over syslog - config :leefheader, :validate => :boolean, :default => true + + LEEF_V1_DELIMITER = "\t" + + config :leefversion, :validate => :string, :default => "2.0" + + config :leefdelimiter, :validate => :string, :default => LEEF_V1_DELIMITER # Field to enable the default syslog header, which uses the default `%{host}` field for hostname and the timestamp is generated by the codec parsing time. If no value is set the hostname is set to the `hostname` value where logstash is running. config :syslogheader, :validate => :boolean, :default => true + config :sysloghost, :validate => :string, :default => "logstash" # Device vendor field in LEEF header. The new value can include `%{foo}` strings @@ -28,7 +33,7 @@ class LogStash::Codecs::LEEF < LogStash::Codecs::Base # Device version field in LEEF header. The new value can include `%{foo}` strings # to help you build a new value from other parts of the event. - config :version, :validate => :string, :default => "5.0.0" + config :version, :validate => :string, :default => "4.0.0" # EventID field in LEEF header. The new value can include `%{foo}` strings # to help you build a new value from other parts of the event. @@ -59,11 +64,67 @@ class LogStash::Codecs::LEEF < LogStash::Codecs::Base # Fields to be included in LEEF extension part as key/value pairs config :fields, :validate => :array, :default => [] -HEADER_FIELDS = ['leef_version', 'leef_vendor', 'leef_product', 'leef_device_version', 'leef_eventid'] + # If raw_data_field is set, during decode of an event an additional field with + # the provided name is added, which contains the raw data. + config :raw_data_field, :validate => :string + + + # Common Header fields for LEEF. In leefVersion=LEEF:2.0 there is a new optional field leefDelimiter + HEADER_FIELDS = ['leefVersion', 'productVendor', 'deviceProduct','deviceVersion','deviceEventId'] + + # SYSLOG_HEADER_PATTERN = /^(?:<\d+>)?\s*([A-Z][a-z]{2}\s+\d+\s\d+:\d+:\d+)\s(.*?)\s/ + SYSLOG_HEADER_PATTERN = /^([A-Z][a-z]{2}\s+\d+\s+\d+:\d+:\d+)\s(.*?)\s+/ + + # A LEFT Header is a sequence of zero or more: + # - backslash-escaped pipes; OR + # - backslash-escaped backslashes; OR + # - non-pipe characters + HEADER_PATTERN = /(?:\\\||\\\\|[^|])*?/ + HEADER_ESCAPE_CAPTURE = /\\([\\|])/ + + # Cache of a scanner pattern that _captures_ a HEADER followed by an unescaped pipe + HEADER_SCANNER = /(#{HEADER_PATTERN})#{Regexp.quote('|')}/ + + LEEF_DELIM_PATTERN = /^(.|0?[xX][0-9A-Fa-f]{2,4})#{Regexp.quote('|')}/ + + + # Cache of a gsub pattern that matches a backslash-escaped backslash or backslash-escaped equals, _capturing_ the escaped character + EXTENSION_VALUE_ESCAPE_CAPTURE = /\\([\\=])/ + + # While the original CEF spec calls out that extension keys must be alphanumeric and must not contain spaces, + # in practice many "CEF" producers like the Arcsight smart connector produce non-legal keys including underscores, + # commas, periods, and square-bracketed index offsets. + # + # To support this, we look for a specific sequence of characters that are followed by an equals sign. This pattern + # will correctly identify all strictly-legal keys, and will also match those that include a dot "subkey" + # + # That sequence must begin with one or more `\w` (word: alphanumeric + underscore), which _optionally_ may be followed + # by "subkey" sequence consisting of a literal dot (`.`) followed by a non-whitespace character, then one or more word + # characters, and then one or more characters that do not convey semantic meaning within CEF (e.g., literal-pipe (`|`), + # whitespace (`\s`), literal-dot (`.`), literal-equals (`=`), or literal-backslash ('\')). + EXTENSION_KEY_PATTERN = /(?:\w+(?:\.[^\s]\w+[^\|\s\.\=\\]+)?(?==))/ + + # Some CEF extension keys seen in the wild use an undocumented array-like syntax that may not be compatible with + # the Event API's strict-mode FieldReference parser (e.g., `fieldname[0]`). + # Cache of a `String#sub` pattern matching array-like syntax and capturing both the base field name and the + # array-indexing portion so we can convert to a valid FieldReference (e.g., `[fieldname][0]`). + EXTENSION_KEY_ARRAY_CAPTURE = /^([^\[\]]+)((?:\[[0-9]+\])+)$/ # '[\1]\2' + public - def initialize(params={}) - super(params) + def register + # LEEF input MUST be UTF-8, per the LEEF documentation: + # https://www.ibm.com/support/knowledgecenter/SS42VS_DSM/com.ibm.dsm.doc/c_LEEF_Format_Guide_intro.html + @utf8_charset = LogStash::Util::Charset.new('UTF-8') + @utf8_charset.logger = self.logger + + if @delimiter + # Logstash configuration doesn't have built-in support for escaping, + # so we implement it here. Feature discussion for escaping is here: + # https://github.com/elastic/logstash/issues/1645 + @delimiter = @delimiter.gsub("\\r", "\r").gsub("\\n", "\n") + @buffer = FileWatch::BufferedTokenizer.new(@delimiter) + end end private @@ -73,77 +134,117 @@ def store_header_field(event,field_name,field_data) end public - def decode(data) + def decode(data, &block) + if @delimiter + @buffer.extract(data).each do |line| + handle(line, &block) + end + else + handle(data, &block) + end + end + + public + def handle(data, &block) + event = LogStash::Event.new + event.set(raw_data_field, data) unless raw_data_field.nil? + + @utf8_charset.convert(data) + + # Several of the many operations in the rest of this method will fail when they encounter UTF8-tagged strings + # that contain invalid byte sequences; fail early to avoid wasted work. + fail('invalid byte sequence in UTF-8') unless data.valid_encoding? + # Strip any quotations at the start and end, flex connectors seem to send this if data[0] == "\"" data = data[1..-2] end - event = LogStash::Event.new - # Split by the pipes, pipes in the extension part are perfectly valid and do not need escaping - # The better solution for the splitting regex would be /(? e + @logger.error("Failed to decode LEEF payload. Generating failure event with payload in message field.", :error => e.message, :backtrace => e.backtrace, :data => data) + yield LogStash::Event.new("message" => data, "tags" => ["_leefparsefailure"]) end public def encode(event) # "LEEF:1.0|Elastic|Logstash|2.3.3|EventID|" - - if self.class.get_config["syslogheader"][:default] == true + + sheader = "" + + if @syslogheader time = Time.new - syslogtime = time.strftime("%b %d %H:%M:%S") + syslogtime = sanitize_header_field(event.get('syslogTime')) + syslogtime = time.strftime("%b %d %H:%M:%S") if syslogtime == "" syslogtime = "<13>" + syslogtime + sysloghost = sanitize_header_field(event.sprintf(@sysloghost)) if sysloghost == "" sysloghost = Socket.gethostname end + + sheader = "#{syslogtime} #{sysloghost} " end + leefversion = get_leefversion(event) + vendor = sanitize_header_field(event.sprintf(@vendor)) - vendor = self.class.get_config["vendor"][:default] if vendor == "" + vendor = self.class.get_config["vendor"][:default] if vendor == "" product = sanitize_header_field(event.sprintf(@product)) product = self.class.get_config["product"][:default] if product == "" @@ -154,43 +255,56 @@ def encode(event) eventid = sanitize_header_field(event.sprintf(@eventid)) eventid = self.class.get_config["eventid"][:default] if eventid == "" - #name = sanitize_header_field(event.sprintf(@name)) - #name = self.class.get_config["name"][:default] if name == "" + headers = ["LEEF:#{leefversion}", vendor, product, version, eventid] - # :sev is deprecated and therefore only considered if :severity equals the default setting or is invalid - #severity = sanitize_severity(event, @severity) - #if severity == self.class.get_config["severity"][:default] - # Use deprecated setting sev - # severity = sanitize_severity(event, @sev) - # end + leefdelimiter = get_leefdelimiter(event) + if leefversion == "2.0" + headers << leefdelimiter + end - # Should also probably set the fields sent - - if @syslogheader == true && @leefheader == true - sheader = [syslogtime, sysloghost].join(" ") - header = ["LEEF:1.0", vendor, product, version, eventid].join("|") - values = @fields.map {|fieldname| get_value(fieldname, event)}.compact.join(" ") - - @on_event.call(event, "#{sheader} #{header}|#{values}\n") - - elsif @syslogheader == true && @leefheader == false - sheader = [syslogtime, sysloghost].join(" ") - values = @fields.map {|fieldname| get_value(fieldname, event)}.compact.join(" ") - @on_event.call(event, "#{sheader} #{values}\n") - elsif @syslogheader == false && @leefheader == false - values = @fields.map {|fieldname| get_value(fieldname, event)}.compact.join(" ") - @on_event.call(event, "#{values}\n") - else - # default behaviour LEEF - header = ["LEEF:1.0", vendor, product, version, eventid].join("|") - values = @fields.map {|fieldname| get_value(fieldname, event)}.compact.join(" ") + delim = get_leefdelimiter_value(leefdelimiter) - @on_event.call(event, "#{header}|#{values}\n") + header = headers.join("|") + values = @fields.map {|fieldname| get_value(fieldname, event)}.compact.join(delim) + + @on_event.call(event, "#{sheader}#{header}|#{values}#{@delimiter}") + + end + + private + def get_leefversion(event) + leefversion = sanitize_header_field(event.sprintf(@leefversion)) + leefversion = self.class.get_config["leefversion"][:default] if leefversion == "" + + return leefversion + end + + private + def get_leefdelimiter(event) + if get_leefversion(event) == "2.0" + leefdelimiter = event.get('leefDelimiter') + leefdelimiter = @leefdelimiter if leefdelimiter.nil? + leefdelimiter = self.class.get_config["leefdelimiter"][:default] if leefdelimiter == "" + else + leefdelimiter = LEEF_V1_DELIMITER end + + return leefdelimiter end private + def get_leefdelimiter_value(leefdelimiter) + if leefdelimiter.length > 1 + # delimiter can be expressed as hexadecimal value as 'x20' or '0x20' + # or even in utf '0x1234' + leefdelimiter = '0' + leefdelimiter if leefdelimiter[0] != '0' + leefdelimiter = [leefdelimiter.to_i(16)].pack('U*') + end + return leefdelimiter + end + + private # Escape pipes and backslashes in the header. Equal signs are ok. # Newlines are forbidden. def sanitize_header_field(value) diff --git a/logstash-codec-leef-3.0.1-java.gem.old b/logstash-codec-leef-3.0.1-java.gem.old deleted file mode 100644 index 7055668af72f0372e4ec32df031f3e358535ef1b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 13312 zcmeHtWpG_hlI0UKGqc6a%#0Q@Gx&)uMvGY%Gcz-@^u%B>GqWs~CGFR{KYF%jX7^7| z>}*8mpSqP9HzVuRy(bEegN3V!xrwU@la&|XzsgwtG7b(7z~9=x>>pcJ4t90`D?1A- z8!HDFH#a+gg_WIygB?K1@?TZZzs>9B;%egjS4rQkEiCN+ZQviC|8MdCHMYMG?jK(N z@7g7af(5)z(P)EzE^0ae=UGrcu8d5+6!nMkIEZ3I0a#n=6wh>R=wSm@4!$f84c5n8 zR-N}AXWd&^6w3IR6eaI@-@aVsbh;GK6e!!X8w`1QA+;+ji4Qrjy~`!amb#v^BaT%`V?2o%Sfn= zW#f1n+aL(zn~;&I<-?m`L;@3oA9M_z0!MoMD4?EBmh+Z{!(ZRaK7$^;WZ$g17sY$y z!4b>A)tT}H4|^)9>0_Y0ItALG4@uz9ohhUa8Qq$QdC)!{w|Z9XB1Epn6`!iuo3^cB zy5D2cm;G%l`;bwB)4fM^!lUiw^2xtJ(%~XtQ#yv(vNI8p&M+gYMmQwlaM}YVTTI_) zTz-$9=REl&XP!Lf;Up)f3U^z8d(dP91U z=I~P3B%P}wiiMVn&xl_sKVb*XiNn^v$^MSqp)>5PxPyH+^1FQK1$_j4!%KzihSSE= z5+U1)z5ds!rgjL(O1f~~l}E_J4Q=$++S&8tW8d7zGla5)(|XU=!!IkgMpZv%1Vg`Z z>=+I`^>l7ba_utNv?G?|TUb-nV3MVKhu%2mWbnv4*bECb@-=u;=<=qojxpqXa!vmY zx?Szv7NwIw?#${JL8sMla8s4x@LB>iS#t2N_qbm8agk8^oRZz7GRg8#gobwO9FQcOgB~oW&jeo0O*TlNM5bg%pcaysOG3 zG=iVG!

1cF2WlB|dCn`?|E@pmKe?{|$^CyZ;Qu@RXJh4N`*-}$&c*pJ{{JUW^?#V| z@BDuu`0lqplJK(Z6UnHNS|lGtl6a|&vy|Y#tNC-Kytru@LjxW%m69A)q%Oc*XW{z$ zwPT!PoHNi_*_pku^O;;mw#l5Hw;@LI3Uu+8Oc68X7F=W2E? zdNnjZXvq8B`SV|DzEwV2R&Lq=1wCI~G1h=B{K`G1)wM!{)adzm06Hs->1?XYnu;0a zEAG2h<4mau>r`C3Ivg-=W&7`4Q+|(OmZM>l-wWz58)nTCb|w$4E^3d;$J~|M2^z9< zA0Y`m!Pear(AKcmQ54UMUV^!y6C>fg?R{t*>bL@c^>j`ACWU>O4XK^AIv{I;Oip~) z?RwD9SzfE9Ay*KKCC@Ibn{DV#^2hGvUFe2si!n>bWg0|ShRWhr6v^r~xQjJ3qLgOY zr5|iNKcsOjRo)p&XJ@Tl01^@dNU%}dZos~Em%n5Cka4U=7g^7IjXm>0yfG$< z!&C7aXsa`}4QRIk%OjiWr%(wSB^x+S5o};&hs+?+pf)*)y0n}}>5A9eou5u&y+xYH z>cuxwDyfv8hE5{{#H&z8}jjuf)n%3@z;1>{Hg5;7z ze@Yp8rtDa|M}(1!T{2kL9Pd0~k;%)L(7iUl{!j zd%V4G{H}**V6L~W{mI_9w=P`oUV9)bDeyVMLIL`LQcAr29xo_F=wyT)h6Eqx4+j?1H@m8?gW;NU=yF%d&_9# zr7Hl85&HAba9(LoQma04KOa{rVP_9i=#|Rn_15`JI`hC$U2yyyUt=|dovfevL?$5A zp|mA$)=S~7XFV`tC{!keoj_pSkc%IX=RMpIFbuFWhsRZs{@rWNblp%y?Ng6Z7aLJ@ zBL0CRvXp7Sfj0fwp!)kD&piGh_63SlPUP-HO zGNvks5XSJi_FzYj_^(3`({c+xv;s!eenT=c&3*|t4PZcb3?809@Y0j9L-Y78(Z@rr zkAi{-hfO3#CgF?9ca^&mdvtCM{`4_PQ?@1emC#B-7K=h;HsbTn>!nib?Dd?3r0` z$az@gjQV1MLD~K4irMc5UP?DzqZKQ8RTbmS{`C2pxf#a?-2H02MA=`dSU>vrgmM!4 z>C#_g9Xd$MM!%zenT`1+j0FG&a8a(*Vk^L1F00d9hEfSgG%heG44@zC&nHw%s9lCa zXWRR`6EXd84?tW6Oy3FZm?V6TvCho8vVj8cI{H?uj8_Bk3}7?Z|Rq=xYi!RH5 zh~U$^$`#=QBKl|;t}hz9-`TwD8+%)gnBUE=ccwaXQC5WAw|cidz3W;#XSTYn9PPcE zzt=roG5+zIa^L#+>D&ACsFCeNpvX%~w9*>DkH0U&C&VH5mSqLKoI`MF6NuNoKT7Dr z1?|n`cq_0|$S(%V2I}*o=Y3p2RCR9W58ZeP6SKfUD!oD zuR-*SlK;?%Dt4OE22RX8diHkyJ_Bv?BYqfkf98I3=Q^JDuP!}*|McARcQO_W<(DfS z6ec8wJfgM#jgxSR$XbOXMIO=rLw&W*-iH5lCm>RgLv1dItQ8-d1aHq(|Lw;ICzabM zf4NwqbvL2y0>+9wIU#4E`WgZntfa;YEF$-v?pQ9ZQKD$jglsei)nFf8RU<9;7{{+I z8kVMkn=C~NmPEKB_>?uWOb;tulY){1d~pmX(@=F9gIofFi)1TFTvCd3?MQ=fG-k>U zN!2a<@CtY=VsUtV+=a85V%`%Hlf|{1kgJ)YMHl>-OA@TX@4@sC1SO0N>!C+Zd5aY5 zoG4;_Dp$N2+iXEr^0m2lcss8Ff>%8DfTbd$aMaB(7H((M&hS!V=rKZ5mE=h3-38Va zWLNd5bfV<4NE0;&Hg6`YusU2auOuR7B8)8l1P%}yYW3JAWrKyJC?Gp?6o3>BTdAXF zo%#h#G9?=C4&@6p!+Db>w)VaYCcjY9d~IRr!3i$nbAVpH?>N>4?Z~L}eQ|==I*uco ziY5>7H$oVqMlcemzL`X&S&Mo;M7M9&vq%{23glzZ>b78pI$sXVcH3(HgaF|!pe1rHRsfwY8D=Nc==MmsB>=ukj*n(ztk{d zCYq(||7oE0T3bY8#4rxslt@r2OD&}f1cTH^)Tl7q9G~tpjP5LT-)X-mV)?NIb|Hv{ zo{|P7{af)kqR*g<0sJ?6fWYO@Pl1tFR+I=*XG6L>+rep2Swn^d$oQHfl~Q?Bl1A=2 z-YhBDKb+sZkWug!?s^Y19Ctw+-47Y z+qSje5WKKIg(rSqoJv0MiNogOmH2Cf@;x(bK4485gE&v{jctc!TLHe6K0VNTso!&d z_D`^)*+d!`_}et{wgiNGnig|N1^IRv^el+GY|Ms_6fs%GGE7dPE)(GUVK)@~ zyXK58WG*Mm(sEHA1Ol0=j^Dvy(+!ZP*Vj(YB_L#-_Ckt4q|30XwCD^#->TT;6n-UN zj)RV6{8Hy03Agj9YeIX^CzT0JriBP^M_tci-H#fzNmd4Gd)9^`JCI@zV}p@-)Tq47 z+$~a~Q=9`5%t>fUk#N$3hM;-s_zCFn)oU6ZF2+0jO%A^(704QAMy<^s{B%nwq?apJ zD2}a{ZByE*g5)}1fq{HMW65!h70&z<>V6$TiXUA#0or7gprAVD87(8%a z@^++oSil6A8??kAIBBSWeZdP^(P<%ktO)J2Zt;$5!Y*q%X)KM>aRE=ng5U^9L)c_E z2z_ZS!+_hW@vwu=!aCO!vIr3`aL|x4Wec2hsQkQ~>>3pDi7|6H)m;Fh?T=vnsJ;j1 z^?Zf%)saiG%PWqfS$#$p_atNNo4WiUE?!NgY6}L+{TJoDZQotzUyHKRx`BO-NlcQf z$r%XcRh0J8YcQ8`^cXTFd92?>Xns3nx1)jv*jucaIF$UX6qHce0I^2^aeRp7N{T)f zKr-gbL8nE{Y3@o{y+3@Hld)1MT`9Qly5*h>Hg#Q`NaabP2w@4#N#20yz9slN%MHg^ zeEi6nD}Fl{G5@n`Ku8^gbzCyk5qia#$$b9QL={;aX`45#V1m60%Ny-kjk2V9+d#{Y z@=BJTCWm{K1@T<3Q7B}^L?5sKjLcp~Q?ky~D$k7gWZi-xSfS{~xW_HO@73b^E;}B< zN0{)QvkFgKC4=}ONK1w%1ipmc@qDzq5d=y*gwq%OJM1t1eA%Tpg6E%}4U2}LPobZm zVGYi-=lt-}F}3n?3RGOlmQjJUeIk*S{%ARwue3Mus3q`X1edfb&`$7mn6Qo)>}p@g zEZonv7izD+8(wzxDv>`SsEvlFBQv$cUxcD%M;dEkD4rjmZ+B=v-ppq?24|bT zguBn%3lj*~edmh-&9vyoFrJ@V#RRsG5$FY1CoFnS@@Hn(W~$X`PL^aDl9Vo|-k{*p@>7jUF z0C$NH9o6M}fuq{Y#Qt6zb8*{rk#BcT;}Qu%akHr_3L!5Y-@N862;qF3IY zL;PGM$16Gw*IrLaWd=y#wdVRKD0uj>?>FJMpxe4?YWQLkR5I2GXGirG(Vq~~D06QT zE})+nOg|uL4PrgIl6iE@OI01A%DL_YNHg+EVIc!0xbeUHsE+<3TtEyFOBq zfj&0Zt@UKfq;8C&rIX*G0e=%4(?3)+cA6=T$0yT zkEdasA}wbG$4>u!M^kR(%w|3uZc@t!(h65ZT6`pb?uD+tbQW4rm%`_nDE447IZ3kz zM)5d5hwhbvrmhbpFiRE*_?oEZU{dHFr9m}ioP}>90S3EH<>8EohQh|_YxKsQxC&@d zW|K#8ZCe@_rd1Q|+R~P~he`}Ni{{yRruvLlja)cGqZ$@`(@A+5@ZNpu=#bd6V{#go zL5Fvd{B7>ZlhIhD8y_!Ij^VeJM0umgk4j4>%YpgQKSjzi(`ZIQygPf4(Y%AO(C6W9 zE#4Z}1rL#~HrL%&^%XoTyl+V(L>v6ozlK{|+t@4VIS6yW3}KT1fDG5KB7IcvHh!vf_Et@ zWn#kBWuYP?M15OODVRAJ#e3Ch$}ofTHOK*0!gcJ-LEV6j*(#yWBb08#W|-Wy!Zj11 z`+`K6TAGSQaLlC|6AhB%C!vx8iFtXUs*3qJ#> zEGr|B@h4V0KQi?~_?VOw+s}ZwS)1T`Od-O;rueVGEP?PxGB3IQ5^!|hJGS66(Ne{PtV% zg7BYZ^pR_?w0itQt9bTdWMizv5Xay(Xb_2JDg;YwGTO&=LDcx~EWpFE9C|4i9-|7x zai81x=wZpcq?q90xE&D+mU`8OW>HKE_Y>le%4L4?P!abuS>%K&Q66f+Meco^8TZO$ zo#K+!cPm+{;j^K3h?k-eam*9~Gc=?MMG+_HjYtS;DnW+R2fAkP zy2N-F6^mP2k$tyO(U=J>h)g38;l0m;ZSc@Vd@ymU5y^#Ahp=w0i-^T;Bx>_oW^jhO zJ_4J)(Ri*QV4yksI3t8rE%48uCf{zE)8r;)Ff1t<7&ihE%VXUL%->b2T{Ai%8cBv< zk*HKQk`tn&9Nt4a$wZR>I$@~=ztFA9Tu|tNAmFJBb9&{adk-YCohsa^szbQd4La&Y z-Qq%TQ_WDDw8G#&7yz9^?*Zb_@iVk(IGYJYUnf?A=wz&y7CACgg{(rVTDa~TRrc{l?;P%TFXThQORRW>SAL2zeg`oL! zwZQ5#D&m~ZZ3#c4?W#R%@_@{zgta(CH2tK>K4I)&F44u4tm*qEVI75BgplzEd(N!M zDt%g+RbDIn=XlcoHEx++_PBX9QZmiCD}^c#L9$CarCi*5WcLz}@zQkNn;oNwn|s;S zB3)qUB(_+t=8p3j`gN;t#;g+o)H@e!`IE>jFvZ&^QOJ&NiV2+g67Jrz>5rmgC-#MD z7ssxAdBg||DxPW*$r;uV`3jTUT)OOs5gDft57B>!*bh(BWaU+VckN7%l4_N)9+?>F zE{7d93D^3#mR;I3NR7EY0equaA|PB@3*25r+`kSVctYk7@21+m>;?!DHiQ9NNIu%$ z9eefXtLK(st3x87U!=EEAs%J%CQk9rIMm;<*O$;+FZ5C^B8(Z7vLHd^iyQ0s}>`> z(uxKw9Z9DOs*2CQ@Od*vA%hLUpm$-x7KI5Wy|~-VV0?12W!t-o@~hqCQ|HgfIM=o5 z+17L@OgB#VEJE4FQu4dDhPW&${`wt5butrFT3U+sLx^f|uOl2yiNHqQ%!t_8)o=}< zBzEj{B98xdzvC~3>z9e`t-CPx3L}=Tt9l>C7aHZ=G^=g;PGbR_72Zv9>_|h=Lz#@2_B> zpqB13bcw8;X$m*ff&}yGNXRSKdRx4Q{-b_K+Jiw@qmy1eXbfVgXAPG?4J-{?w~yDz4^OVg4fm$ zq>2{N=ZdTdNR0QfrSa+Z19PGX&Q`_ zkxspF^5Ysfs+U3HDKP{$Q8$uXC89m<80W^jxj|Ip)>9_{75w;X^NhcU<~_P*4Q#f( zQ1&;a5P$GAbXZ(xw8e5X0qY8O$^Dtyi|lQzkY0YTHU&!hLs=U4xnc4_lH(;T|FM0v zU#Bay2Pp{j@Hs2@ke|JN@_ygVlS|Rsb_inAf?=A{? zl{daXxTTEs&I!BiMSo<=yUh(d#g3OA;n^~?ztD*>)7FlYes0%wAR{x}49{c&JoJb8f=NH*qlR1og4aU3!^~O%oR#Zsn3-opE@D*Xo zD1KxtR6P{!jRnZo2^G~yH&6Pl7+*L0$K?blS;$k#3ew{0m_tN{k(Rv^Znemdhr33uCKqHraBa2gb`kwhN_C=x>73oc2ueM zabnq4qTH6Z-1~Xzoby|TtNonDzHFqVecSp7;#D|i~i715OpVmZ9&Ql$bXP1|KtBC0aZlXA9i;%*)jM0<^#5bh`+2k(*EwaObp{SC;~89ho4`FL2JwA`<@D9X z$dHrX%1mk((qM>|ZNyVA(V3<&`@(e3p?dqv)5ioGz?x@{=e;^%~5l8m`A4&OckVE}YE3Iof_jHQ- zQDkI?B)sT5eWTdDi_}8}?5Ut3L_RTe9RdU5UD-)s%8l^s_&u07-qzA-&N%k1hG#yj zB7e0rTv9iMzM`tTmWq|7MJdPGk;@fwPR5eDKSO4h|=(zH`|$p$%z6B4SfLs4c~WQz&96y<|- zu5z99_ip$X+ncu_BH~j%f)N!w%(4pU5`h7FVRkyl^#~ZNUxutr%6ET#alNrMcz@W; zY*8PlxH~S`h@=>%56V-dbErA{`w+*QCslrfiAQI@%7AmGi2|J~`gao)Z}B~elz{}* zIk7a4d-0~qBbf70{`N{PCZCvJcUC4KURp90n8N*%vAy zSa%jQNo?`?2bW)&V{J!HEgA&iIF9HNM~!O8;f4?C{EQGNeRP2vn( zldQ$ZVtU8y1uXE5Lz+mgy@*a`K+5{DabCPaTd9Z7D=`T=9fW)fw>nK)*L3fkdKG;; zcD2%bR^|BTP)|Autc+tfUmmWT`#=75m~o%ks92 zZP>8>Cv;B5uaLYd;PDq?SMNZ@G6kYVfIziSM}xq#>4NlhD$l)sH*~k577n8U?BN}^ z#ZuEj0l!{QCfa*}iaZWIg1aZ@ZGv(qgZSH^28|&n@#4oyRpaAVca8iS%K-2zwWh-B zH6=F3+|hLl|4sj=rMv(H&RhtQ%W#V4+Qq2zna{g3tdUenn4_0>-TlFhlCI-Q8d{GS zi=t33B9wU-fsQM~B<7v2R6ETs_SM5{A3TJo{X?XsW>SKa9tK-5+m)TDvZg`J{ndVd z-1V43;bS+gEolR5Z~sJ4N1u#rbwWlpC5(`@?k3ZuYsCa-85$Kdn>-PMCiuL|SmboS zsdgKF*G|jsrcfy4*N+!%*ff)2+dgl}(E_w)ww^}Q@Wx~?_b7n77Wnd&y(OUknoY>^ z*mT@7%igkK`4NUYqXvA726sT42l1$X(<5oHa1szEEm^1GbsQ6Dq;DBuq>oKbd>n0g zM{!K`^@!soMB?igFif7vg-f|#*2urk|GKB1a+%G;#PNX&(Cx`r$=E(sahmeY346m| zctW5RGdutjCsOa$Mq)8ueAdiq_=d49e&YKBe6UTUt+nUvco)avv$&JQ(k^Kqk+JK@ z{ED~wZ}+Z3UA#iPqV*CAq*D_d_IO2bBkI?H>?~^?8f`#2LdfKyVtjiG!*~v9PZ9?> zgHS&kU4WXs$G!P&Bxx`|z?0g8sX2l2BBx5<>4!3B61*|Cu3Y2du3OaCiyG)#t|A6m@;F9FF|@}LIJD@)L`6$<)>mljYBmq zvDINTTQ}Km{4cGDxG+sxH^CxXc6(!Q-NN8V)1`acf1SNOFnG zG6+V70DAbjkn8&oklY;F1Zwa`F3U7pd5YTYtQEOsc;mPMN|fAIE*Oe}HneNPMz^eu zlCe5VM`8ItHlJ?U6(XEin;3m)`Vdo8G|U{4rs)pAPO6rLosNcaR1Zq{o1ugR@}@n> z`L9=P3(Qy<7pzb9pr9(1D5#r!!>F}_;Ql)EHi%kIxiKWZ)fh`&x8~k3KBPZ62Y7{B zIt$OnXh-?N@1A}YEn(4i7oRhw!^!uj%Ey@#GvHk+wUA;{uaOU>O=Ese69-DQ?zG)r zTEJE`Bpv=@+^5c~&H9eKsGJB4=)x9B9{$-Dl)amJyHQzcJs7s>S!t0A z^y;bk9t;3SnNhIu3|pJ-a_esBtKSUdPglpq__{ADYhU4`{M6vyWwB9x*tk%A&{-L7 zj;AP}rW|hf+Ba&xCgqx2MrzIJ6WAlTFAJo1E*xf5Fgr*XcIV!;RaI1_@Fh`xXz{2Y zV7a@VZI;%$p8xT4{&|$$#>`2!DCsLsyoZsqrhq&89N)DIe=TPuCKvNLMO$_)wDi{E z_!&fhxmC0ky>wiNeWM9FgFi=S6lq-j`%sqsxcGCA$AMpgcG0d(+o8EmPTFNoV z=eq0bfdDZ3RI6rbh=5@6!x?xm+v{)>H+hbZ}sf;mR zQ703p9TRoT>-jVW_4EYv=V7g3jE&`05q)B3$RP!6TpBf{Z-UzZnk$xtArPlk2XLZ7 zUrsS@rwxg4#>Q=ZOga2YUh!b{uH%0-cKiwXm$2va69P~@G6)Np{0~(=|GCr5+QQ7v z#m&Km$@O*>A(K-S^MWj{;vPBh^J4{6GfD0 z*V;wmDWz)mvm6{~1b^qFF^~PKnR^_FQKRQ39|zihJU@mJ06x9tdP;)m{r-Hsb#W^~ zngFvXw~i?*Dsw1A#nv6AgGjpDn2)e>jR@1yX3Ht33=D`TbXsz%lV`6$-}$e<(?fhL z%j3zS*Ti%a!iDPHnsfJ!pCDDcZSG`? diff --git a/spec/codecs/leef_spec.rb b/spec/codecs/leef_spec.rb index 8d4aafc..ac26657 100644 --- a/spec/codecs/leef_spec.rb +++ b/spec/codecs/leef_spec.rb @@ -11,15 +11,15 @@ end context "#encode" do - subject(:codec) { LogStash::Codecs::LEEF.new } - + subject(:codec) { LogStash::Codecs::LEEF.new("leefversion" => "1.0", "syslogheader" => false)} + let(:results) { [] } it "should not fail if fields is nil" do codec.on_event{|data, newdata| results << newdata} event = LogStash::Event.new("foo" => "bar") codec.encode(event) - expect(results.first).to match(/^LEEF:1.0\|Elastic\|Logstash\|2.3.3\|Logstash\|$/m) + expect(results.first).to match(/^LEEF:1.0\|Elastic\|Logstash\|4.0.0\|Logstash\|$/m) end it "should assert all header fields are present" do @@ -27,11 +27,12 @@ codec.fields = [] event = LogStash::Event.new("foo" => "bar") codec.encode(event) - expect(results.first).to match(/^LEEF:1.0\|Elastic\|Logstash\|2.3.3\|Logstash\|$/m) + expect(results.first).to match(/^LEEF:1.0\|Elastic\|Logstash\|4.0.0\|Logstash\|$/m) end it "should use default values for empty header fields" do codec.on_event{|data, newdata| results << newdata} + codec.leefversion = "" codec.vendor = "" codec.product = "" codec.version = "" @@ -39,19 +40,21 @@ codec.fields = [] event = LogStash::Event.new("foo" => "bar") codec.encode(event) - expect(results.first).to match(/^LEEF:1.0\|Elastic\|Logstash\|2.3.3\|Logstash\|$/m) + expect(results.first).to match(/^LEEF:2.0\|Elastic\|Logstash\|4.0.0\|Logstash\|\t\|$/m) end it "should use configured values for header fields" do codec.on_event{|data, newdata| results << newdata} + codec.leefversion = "2.0" + codec.leefdelimiter = "^" codec.vendor = "vendor" codec.product = "product" - codec.version = "2.0" + codec.version = "1.0" codec.eventid = "eventid" codec.fields = [] event = LogStash::Event.new("foo" => "bar") codec.encode(event) - expect(results.first).to match(/^LEEF:1.0\|vendor\|product\|2.0\|eventid\|$/m) + expect(results.first).to match(/^LEEF:2.0\|vendor\|product\|1.0\|eventid\|\^\|$/m) end it "should use sprintf for header fields" do @@ -71,7 +74,7 @@ codec.fields = [ "foo", "bar" ] event = LogStash::Event.new("foo" => "foo value", "bar" => "bar value") codec.encode(event) - expect(results.first).to match(/^LEEF:1.0\|Elastic\|Logstash\|2.3.3\|Logstash\|foo=foo value bar=bar value$/m) + expect(results.first).to match(/^LEEF:1.0\|Elastic\|Logstash\|4.0.0\|Logstash\|foo=foo value bar=bar value$/m) end it "should ignore fields in fields if not present in event" do @@ -79,7 +82,7 @@ codec.fields = [ "foo", "bar", "baz" ] event = LogStash::Event.new("foo" => "foo value", "baz" => "baz value") codec.encode(event) - expect(results.first).to match(/^LEEF:1.0\|Elastic\|Logstash\|2.3.3\|Logstash\|foo=foo value baz=baz value$/m) + expect(results.first).to match(/^LEEF:1.0\|Elastic\|Logstash\|4.0.0\|Logstash\|foo=foo value baz=baz value$/m) end it "should sanitize header fields" do @@ -99,7 +102,7 @@ codec.fields = [ "f o\no", "@b-a_r" ] event = LogStash::Event.new("f o\no" => "foo value", "@b-a_r" => "bar value") codec.encode(event) - expect(results.first).to match(/^LEEF:1.0\|Elastic\|Logstash\|2.3.3\|Logstash\|foo=foo value bar=bar value$/m) + expect(results.first).to match(/^LEEF:1.0\|Elastic\|Logstash\|4.0.0\|Logstash\|foo=foo value bar=bar value$/m) end it "should sanitize extension values" do @@ -107,7 +110,7 @@ codec.fields = [ "foo", "bar", "baz" ] event = LogStash::Event.new("foo" => "foo\\value\n", "bar" => "bar=value\r") codec.encode(event) - expect(results.first).to match(/^LEEF:1.0\|Elastic\|Logstash\|2.3.3\|Logstash\|foo=foo\\\\value\\n bar=bar\\=value\\n$/m) + expect(results.first).to match(/^LEEF:1.0\|Elastic\|Logstash\|4.0.0\|Logstash\|foo=foo\\\\value\\n bar=bar\\=value\\n$/m) end it "should encode a hash value" do @@ -115,7 +118,7 @@ codec.fields = [ "foo" ] event = LogStash::Event.new("foo" => { "bar" => "bar value", "baz" => "baz value" }) codec.encode(event) - foo = results.first[/^LEEF:1.0\|Elastic\|Logstash\|2.3.3\|Logstash\|foo=(.*)$/, 1] + foo = results.first[/^LEEF:1.0\|Elastic\|Logstash\|4.0.0\|Logstash\|foo=(.*)$/, 1] expect(foo).not_to be_nil foo_hash = JSON.parse(foo) expect(foo_hash).to eq({"bar" => "bar value", "baz" => "baz value"}) @@ -126,7 +129,7 @@ codec.fields = [ "foo" ] event = LogStash::Event.new("foo" => [ "bar", "baz" ]) codec.encode(event) - foo = results.first[/^LEEF:1.0\|Elastic\|Logstash\|2.3.3\|Logstash\|foo=(.*)$/, 1] + foo = results.first[/^LEEF:1.0\|Elastic\|Logstash\|4.0.0\|Logstash\|foo=(.*)$/, 1] expect(foo).not_to be_nil foo_array = JSON.parse(foo) expect(foo_array).to eq(["bar", "baz"]) @@ -137,7 +140,7 @@ codec.fields = [ "foo" ] event = LogStash::Event.new("foo" => [ { "bar" => "bar value" }, "baz" ]) codec.encode(event) - foo = results.first[/^LEEF:1.0\|Elastic\|Logstash\|2.3.3\|Logstash\|foo=(.*)$/, 1] + foo = results.first[/^LEEF:1.0\|Elastic\|Logstash\|4.0.0\|Logstash\|foo=(.*)$/, 1] expect(foo).not_to be_nil foo_array = JSON.parse(foo) expect(foo_array).to eq([{"bar" => "bar value"}, "baz"]) @@ -148,7 +151,7 @@ codec.fields = [ "foo" ] event = LogStash::Event.new("foo" => LogStash::Timestamp.new) codec.encode(event) - expect(results.first).to match(/^LEEF:1.0\|Elastic\|Logstash\|2.3.3\|Logstash\|foo=[0-9TZ.:-]+$/m) + expect(results.first).to match(/^LEEF:1.0\|Elastic\|Logstash\|4.0.0\|Logstash\|foo=[0-9TZ.:-]+$/m) end end @@ -208,129 +211,114 @@ end end - context "#decode" do - let (:message) { "LEEF:1.0|security|threatmanager|1.0|100|src=10.0.0.192 dst=12.121.122.82 spt=1232" } + context "#decode v1" do + subject(:codec) { LogStash::Codecs::LEEF.new } + + let(:results) { [] } + + let (:message) { "LEEF:1.0|security|threatmanager|1.0|100|src=10.0.0.192\tdst=12.121.122.82\tspt=1232" } def validate(e) insist { e.is_a?(LogStash::Event) } - insist { e.get('leef_version') } == "1.0" - insist { e.get('leef_device_version') } == "1.0" - insist { e.get('leef_eventid') } == "100" + insist { e.get('leefVersion') } == "1.0" end it "should parse the leef headers" do subject.decode(message) do |e| - validate(e) - ext = e.get('leef_ext') - insist { e.get("leef_vendor") } == "security" - insist { e.get("leef_product") } == "threatmanager" + insist { e.get('leefVersion') } == "1.0" + insist { e.get('deviceVersion') } == "1.0" + insist { e.get('deviceEventId') } == "100" + insist { e.get("productVendor") } == "security" + insist { e.get("deviceProduct") } == "threatmanager" end end it "should parse the leef body" do subject.decode(message) do |e| - ext = e.get('leef_ext') - insist { ext['src'] } == "10.0.0.192" - insist { ext['dst'] } == "12.121.122.82" - insist { ext['spt'] } == "1232" - end - end - - let (:no_ext) { "LEEF:1.0|security|threatmanager|1.0|100|" } - it "should be OK with no extension dictionary" do - subject.decode(no_ext) do |e| validate(e) - insist { e.get("leef_ext") } == nil - end + insist { e.get('src') } == "10.0.0.192" + insist { e.get('dst') } == "12.121.122.82" + insist { e.get('spt') } == "1232" + end end - let (:missing_headers) { "LEEF:1.0|||1.0|100|src=10.0.0.192 dst=12.121.122.82 spt=1232" } + let (:missing_headers) { "LEEF:1.0|||1.0|100|src=10.0.0.192\tdst=12.121.122.82\tspt=1232" } it "should be OK with missing LEEF headers (multiple pipes in sequence)" do subject.decode(missing_headers) do |e| validate(e) - insist { e.get("leef_vendor") } == "" - insist { e.get("leef_product") } == "" + insist { e.get("productVendor") } == "" + insist { e.get("deviceProduct") } == "" + insist { e.get('src') } == "10.0.0.192" + insist { e.get('dst') } == "12.121.122.82" + insist { e.get('spt') } == "1232" end end - let (:leading_whitespace) { "LEEF:1.0|security|threatmanager|1.0|100| src=10.0.0.192 dst=12.121.122.82 spt=1232" } + let (:leading_whitespace) { "LEEF:1.0|security|threatmanager|1.0|100| src=10.0.0.192\tdst=12.121.122.82\tspt=1232" } it "should strip leading whitespace from the message" do subject.decode(leading_whitespace) do |e| validate(e) + insist { e.get('src') } == "10.0.0.192" + insist { e.get('dst') } == "12.121.122.82" + insist { e.get('spt') } == "1232" end end let (:escaped_pipes) { 'LEEF:1.0|security|threatmanager|1.0|100|moo=this\|has an escaped pipe' } it "should be OK with escaped pipes in the message" do subject.decode(escaped_pipes) do |e| - ext = e.get('leef_ext') - insist { ext['moo'] } == 'this\|has an escaped pipe' + insist { e.get('moo') } == 'this\|has an escaped pipe' end end let (:pipes_in_message) {'LEEF:1.0|security|threatmanager|1.0|100|moo=this|has an pipe'} it "should be OK with not escaped pipes in the message" do subject.decode(pipes_in_message) do |e| - ext = e.get('leef_ext') - insist { ext['moo'] } == 'this|has an pipe' + insist { e.get('moo') } == 'this|has an pipe' end end let (:escaped_equal_in_message) {'LEEF:1.0|security|threatmanager|1.0|100|moo=this \=has escaped \= equals\='} it "should be OK with escaped equal in the message" do subject.decode(escaped_equal_in_message) do |e| - ext = e.get('leef_ext') - insist { ext['moo'] } == 'this =has escaped = equals=' + insist { e.get('moo') } == 'this =has escaped = equals=' end end let (:escaped_backslash_in_header) {'LEEF:1.0|secu\\\\rity|threat\\\\manager|1.\\\\0|10\\\\0|'} it "should be OK with escaped backslash in the headers" do subject.decode(escaped_backslash_in_header) do |e| - insist { e.get("leef_version") } == '1.0' - insist { e.get("leef_vendor") } == 'secu\\rity' - insist { e.get("leef_product") } == 'threat\\manager' - insist { e.get("leef_device_version") } == '1.\\0' - insist { e.get("leef_eventid") } == '10\\0' + insist { e.get("leefVersion") } == '1.0' + insist { e.get("productVendor") } == 'secu\\rity' + insist { e.get("deviceProduct") } == 'threat\\manager' + insist { e.get("deviceVersion") } == '1.\\0' + insist { e.get("deviceEventId") } == '10\\0' end end let (:escaped_backslash_in_header_edge_case) {'LEEF:1.0|security\\\\\\||threatmanager\\\\|1.0|100|'} it "should be OK with escaped backslash in the headers (edge case: escaped slash in front of pipe)" do subject.decode(escaped_backslash_in_header_edge_case) do |e| - validate(e) - insist { e.get("leef_vendor") } == 'security\\|' - insist { e.get("leef_product") } == 'threatmanager\\' + insist { e.get("productVendor") } == 'security\\|' + insist { e.get("deviceProduct") } == 'threatmanager\\' end end - - # let (:escaped_pipes_in_header) {'LEEF:1.0|secu\\|rity|threatmanager\\||1.\\|0|10\\|0|'} - # it "should be OK with escaped pipes in the headers" do - # subject.decode(escaped_pipes_in_header) do |e| - # insist { e.get("leef_version") } == '0' - # insist { e.get("leef_vendor") } == 'secu|rity' - # insist { e.get("leef_product") } == 'threatmanager|' - # insist { e.get("leef_device_version") } == '1.|0' - # insist { e.get("leef_eventid") } == '10|0' - # end - # end let (:escaped_pipes_in_header) {'LEEF:1.0|secu\\|rity|threatmanager\\||1.\\|0|10\\|0|'} it "should be OK with escaped pipes in the headers" do subject.decode(escaped_pipes_in_header) do |e| - insist { e.get("leef_version") } == '1.0' - insist { e.get("leef_vendor") } == 'secu|rity' - insist { e.get("leef_product") } == 'threatmanager|' - insist { e.get("leef_device_version") } == '1.|0' - insist { e.get("leef_eventid") } == '10|0' + insist { e.get("leefVersion") } == '1.0' + insist { e.get("productVendor") } == 'secu|rity' + insist { e.get("deviceProduct") } == 'threatmanager|' + insist { e.get("deviceVersion") } == '1.|0' + insist { e.get("deviceEventId") } == '10|0' end end let (:escaped_backslash_in_message) {'LEEF:1.0|security|threatmanager|1.0|100|moo=this \\\\has escaped \\\\ backslashs\\\\'} it "should be OK with escaped backslashs in the message" do subject.decode(escaped_backslash_in_message) do |e| - ext = e.get('leef_ext') - insist { ext['moo'] } == 'this \\has escaped \\ backslashs\\' + insist { e.get('moo') } == 'this \\has escaped \\ backslashs\\' end end @@ -338,19 +326,89 @@ def validate(e) it "should be OK with equal in the headers" do subject.decode(equal_in_header) do |e| validate(e) - insist { e.get("leef_product") } == "threatmanager=equal" + insist { e.get("deviceProduct") } == "threatmanager=equal" end end - let (:syslog) { "Syslogdate Sysloghost LEEF:1.0|security|threatmanager|1.0|100|src=10.0.0.192 dst=12.121.122.82 spt=1232" } + let (:syslog) { "Aug 1 12:00:00 sysloghost LEEF:1.0|security|threatmanager|1.0|100|src=10.0.0.192\tdst=12.121.122.82\tspt=1232" } it "Should detect headers before LEEF starts" do subject.decode(syslog) do |e| validate(e) - insist { e.get('syslog') } == 'Syslogdate Sysloghost' + insist { e.get('syslogTime') } == 'Aug 1 12:00:00' + insist { e.get('syslogHost') } == 'sysloghost' + insist { e.get('src') } == "10.0.0.192" + insist { e.get('dst') } == "12.121.122.82" + insist { e.get('spt') } == "1232" end end end + context "#decode v2" do + subject(:codec) { LogStash::Codecs::LEEF.new } + + let (:message) { "LEEF:2.0|security|threatmanager|1.0|100|^|src=10.0.0.192^dst=12.121.122.82^spt=1232" } + + def validate(e) + insist { e.is_a?(LogStash::Event) } + insist { e.get('leefVersion') } == "2.0" + end + + it "should parse the leef headers" do + subject.decode(message) do |e| + insist { e.get('deviceVersion') } == "1.0" + insist { e.get('deviceEventId') } == "100" + insist { e.get("productVendor") } == "security" + insist { e.get("deviceProduct") } == "threatmanager" + insist { e.get("leefDelimiter") } == "^" + end + end + + it "should parse the leef body" do + subject.decode(message) do |e| + validate(e) + insist { e.get('src') } == "10.0.0.192" + insist { e.get('dst') } == "12.121.122.82" + insist { e.get('spt') } == "1232" + end + end + + let (:default_delimiter) {"LEEF:2.0|security|threatmanager|1.0|100|src=10.0.0.192\tdst=12.121.122.82\tspt=1232"} + it "should use default delimiter" do + subject.decode(default_delimiter) do |e| + validate(e) + expect(codec.send(:get_leefdelimiter_value, "\t")).to be == "\t" + insist { e.get('src') } == "10.0.0.192" + insist { e.get('dst') } == "12.121.122.82" + insist { e.get('spt') } == "1232" + end + end + + let (:utf8_delimiter) {'LEEF:2.0|security|threatmanager|1.0|100|0x241d|src=10.0.0.192␝dst=12.121.122.82␝spt=1232'} + it "should use default delimiter" do + subject.decode(utf8_delimiter) do |e| + validate(e) + insist { e.get("leefDelimiter") } == "0x241d" + insist { e.get('src') } == "10.0.0.192" + insist { e.get('dst') } == "12.121.122.82" + insist { e.get('spt') } == "1232" + end + end + end + + + context "get v2 delmiter value" do + subject(:codec) { LogStash::Codecs::LEEF.new } + + it "should get delimiter value" do + + expect(codec.send(:get_leefdelimiter_value, "\t")).to be == "\t" + expect(codec.send(:get_leefdelimiter_value, "\x21")).to be == "!" + expect(codec.send(:get_leefdelimiter_value, "x21")).to be == "!" + expect(codec.send(:get_leefdelimiter_value, "0x21")).to be == "!" + expect(codec.send(:get_leefdelimiter_value, "0x241d")).to be == "␝" + end + end + context "encode and decode" do subject(:codec) { LogStash::Codecs::LEEF.new } @@ -358,19 +416,34 @@ def validate(e) it "should return an equal event if encoded and decoded again" do codec.on_event{|data, newdata| results << newdata} - codec.vendor = "%{leef_vendor}" - codec.product = "%{leef_product}" - codec.version = "%{leef_device_version}" - codec.eventid = "%{leef_eventid}" + codec.syslogheader = false + codec.leefversion = "%{leefVersion}" + codec.leefdelimiter = "%{leefDelimiter}" + codec.vendor = "%{productVendor}" + codec.product = "%{deviceProduct}" + codec.version = "%{deviceVersion}" + codec.eventid = "%{deviceEventId}" codec.fields = [ "foo" ] - event = LogStash::Event.new("leef_vendor" => "vendor", "leef_product" => "product", "leef_device_version" => "2.0", "leef_eventid" => "eventid", "foo" => "bar") + + event = LogStash::Event.new( + "leefVersion" => "2.0", + "leefDelimiter" => "^", + "productVendor" => "vendor", + "deviceProduct" => "product", + "deviceVersion" => "2.0", + "deviceEventId" => "eventid", + "foo" => "bar" + ) + codec.encode(event) codec.decode(results.first) do |e| - expect(e.get('leef_vendor')).to be == event.get('leef_vendor') - expect(e.get('leef_product')).to be == event.get('leef_product') - expect(e.get('leef_device_version')).to be == event.get('leef_device_version') - expect(e.get('leef_eventid')).to be == event.get('leef_eventid') - expect(e.get('[leef_ext][foo]')).to be == event.get('foo') + expect(e.get('leefVersion')).to be == event.get('leefVersion') + expect(e.get('leefDelimiter')).to be == event.get('leefDelimiter') + expect(e.get('productVendor')).to be == event.get('productVendor') + expect(e.get('deviceProduct')).to be == event.get('deviceProduct') + expect(e.get('deviceVersion')).to be == event.get('deviceVersion') + expect(e.get('deviceEventId')).to be == event.get('deviceEventId') + expect(e.get('foo')).to be == event.get('foo') end end end From 2d11d2524b61c06b2059c22849e28f3a00f2b5ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcos=20Ag=C3=BCero?= Date: Thu, 5 Sep 2019 14:14:41 +0200 Subject: [PATCH 2/6] Updated version --- logstash-codec-leef.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/logstash-codec-leef.gemspec b/logstash-codec-leef.gemspec index e029e4d..d6a938d 100644 --- a/logstash-codec-leef.gemspec +++ b/logstash-codec-leef.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-codec-leef' - s.version = '3.0.1' + s.version = '4.0' s.platform = 'java' s.licenses = ['Apache License (2.0)'] s.summary = "LEEF codec to parse and encode LEEF formated logs" From 80d403a592c728e3393b50dc5dd8886b7ba3fbf3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcos=20Ag=C3=BCero?= Date: Fri, 6 Sep 2019 09:03:08 +0200 Subject: [PATCH 3/6] Removed debug message --- lib/logstash/codecs/leef.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/logstash/codecs/leef.rb b/lib/logstash/codecs/leef.rb index 86a4085..c22b44a 100644 --- a/lib/logstash/codecs/leef.rb +++ b/lib/logstash/codecs/leef.rb @@ -173,7 +173,6 @@ def handle(data, &block) unprocessed_data = data HEADER_FIELDS.each do |field_name| match_data = HEADER_SCANNER.match(unprocessed_data) - @logger.debug(match_data.inspect) break if match_data.nil? # missing fields escaped_field_value = match_data[1] From a6d1546277a1dd2e1df6d30f870ae8740bcfc6fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcos=20Ag=C3=BCero?= Date: Mon, 9 Sep 2019 10:41:55 +0200 Subject: [PATCH 4/6] Ignore empty lines. Do not add failed decode events --- lib/logstash/codecs/leef.rb | 6 +++++- spec/codecs/leef_spec.rb | 9 +++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/lib/logstash/codecs/leef.rb b/lib/logstash/codecs/leef.rb index c22b44a..153b21b 100644 --- a/lib/logstash/codecs/leef.rb +++ b/lib/logstash/codecs/leef.rb @@ -137,6 +137,7 @@ def store_header_field(event,field_name,field_data) def decode(data, &block) if @delimiter @buffer.extract(data).each do |line| + next if /^\s*$/.match(line) # Skip empty lines handle(line, &block) end else @@ -146,11 +147,14 @@ def decode(data, &block) public def handle(data, &block) + raise "Empty line" if /^\s*$/.match/data + event = LogStash::Event.new event.set(raw_data_field, data) unless raw_data_field.nil? @utf8_charset.convert(data) + # Several of the many operations in the rest of this method will fail when they encounter UTF8-tagged strings # that contain invalid byte sequences; fail early to avoid wasted work. fail('invalid byte sequence in UTF-8') unless data.valid_encoding? @@ -217,7 +221,7 @@ def handle(data, &block) rescue => e @logger.error("Failed to decode LEEF payload. Generating failure event with payload in message field.", :error => e.message, :backtrace => e.backtrace, :data => data) - yield LogStash::Event.new("message" => data, "tags" => ["_leefparsefailure"]) + # yield LogStash::Event.new("message" => data, "tags" => ["_leefparsefailure"]) end public diff --git a/spec/codecs/leef_spec.rb b/spec/codecs/leef_spec.rb index ac26657..02409d9 100644 --- a/spec/codecs/leef_spec.rb +++ b/spec/codecs/leef_spec.rb @@ -393,6 +393,15 @@ def validate(e) insist { e.get('spt') } == "1232" end end + + let (:empty_lines) {["", " ", " ", " \r"]} + it "should ignore empty lines" do + empty_lines.each do |line| + subject.decode(line) do |e| + expect(e).to be_nil + end + end + end end From d5b0ab4c5005276ddfbf2425010de13427d23245 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcos=20Ag=C3=BCero?= Date: Tue, 10 Sep 2019 12:26:52 +0200 Subject: [PATCH 5/6] Added error_event config parameter --- lib/logstash/codecs/leef.rb | 50 +++++++++++++------------------------ 1 file changed, 18 insertions(+), 32 deletions(-) diff --git a/lib/logstash/codecs/leef.rb b/lib/logstash/codecs/leef.rb index 153b21b..96037f1 100644 --- a/lib/logstash/codecs/leef.rb +++ b/lib/logstash/codecs/leef.rb @@ -39,28 +39,6 @@ class LogStash::Codecs::LEEF < LogStash::Codecs::Base # to help you build a new value from other parts of the event. config :eventid, :validate => :string, :default => "Logstash" - # Name field in LEEF header. The new value can include `%{foo}` strings - # to help you build a new value from other parts of the event. - #config :name, :validate => :string, :default => "Logstash" - - # Deprecated severity field for LEEF header. The new value can include `%{foo}` strings - # to help you build a new value from other parts of the event. - # - # This field is used only if :severity is unchanged set to the default value. - # - # Defined as field of type string to allow sprintf. The value will be validated - # to be an integer in the range from 0 to 10 (including). - # All invalid values will be mapped to the default of 6. - #config :sev, :validate => :string, :default => "6", :deprecated => "This setting is being deprecated, use :severity instead." - - # Severity field in LEEF header. The new value can include `%{foo}` strings - # to help you build a new value from other parts of the event. - # - # Defined as field of type string to allow sprintf. The value will be validated - # to be an integer in the range from 0 to 10 (including). - # All invalid values will be mapped to the default of 6. - #config :severity, :validate => :string, :default => "6" - # Fields to be included in LEEF extension part as key/value pairs config :fields, :validate => :array, :default => [] @@ -68,6 +46,9 @@ class LogStash::Codecs::LEEF < LogStash::Codecs::Base # the provided name is added, which contains the raw data. config :raw_data_field, :validate => :string + # If error_event is true, generate an error event if parsing fails. Payload is placed on :raw_data_field if available + config :error_event, :validate => :boolean, :default => false + # Common Header fields for LEEF. In leefVersion=LEEF:2.0 there is a new optional field leefDelimiter HEADER_FIELDS = ['leefVersion', 'productVendor', 'deviceProduct','deviceVersion','deviceEventId'] @@ -150,7 +131,7 @@ def handle(data, &block) raise "Empty line" if /^\s*$/.match/data event = LogStash::Event.new - event.set(raw_data_field, data) unless raw_data_field.nil? + event.set(raw_data_field, data) unless @raw_data_field.nil? @utf8_charset.convert(data) @@ -159,22 +140,22 @@ def handle(data, &block) # that contain invalid byte sequences; fail early to avoid wasted work. fail('invalid byte sequence in UTF-8') unless data.valid_encoding? + # Use a scanning parser to capture the HEADER_FIELDS"""!!!|@@@@@@ + unprocessed_data = data + # Strip any quotations at the start and end, flex connectors seem to send this - if data[0] == "\"" - data = data[1..-2] + if unprocessed_data[0] == "\"" + unprocessed_data = unprocessed_data[1..-2] end - # Search for syslog header - SYSLOG_HEADER_PATTERN.match(data) do |syslog| + SYSLOG_HEADER_PATTERN.match(unprocessed_data) do |syslog| event.set('syslogTime', syslog[1]) event.set('syslogHost', syslog[2]) - data = syslog.post_match + unprocessed_data = syslog.post_match @syslogheader = true end - # Use a scanning parser to capture the HEADER_FIELDS"""!!!|@@@@@@ - unprocessed_data = data HEADER_FIELDS.each do |field_name| match_data = HEADER_SCANNER.match(unprocessed_data) break if match_data.nil? # missing fields @@ -220,8 +201,13 @@ def handle(data, &block) yield event rescue => e - @logger.error("Failed to decode LEEF payload. Generating failure event with payload in message field.", :error => e.message, :backtrace => e.backtrace, :data => data) - # yield LogStash::Event.new("message" => data, "tags" => ["_leefparsefailure"]) + @logger.error("Failed to decode LEEF payload.", :error => e.message, :backtrace => e.backtrace, :data => data) + data_field = if @raw_data_field.nil? then "raw_data" else @raw_data_field end + if @error_event + @logger.error("Generating error event.", :data => data) + yield LogStash::Event.new("message" => e.message, "tags" => ["_leefparsefailure"], data_field => data) if @error_event + end + end public From 250ec1dea66bca60ab9934992a143e642c640758 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcos=20Ag=C3=BCero?= Date: Tue, 10 Sep 2019 12:27:22 +0200 Subject: [PATCH 6/6] Cleanup --- lib/logstash/codecs/leef.rb | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/lib/logstash/codecs/leef.rb b/lib/logstash/codecs/leef.rb index 96037f1..127bb0e 100644 --- a/lib/logstash/codecs/leef.rb +++ b/lib/logstash/codecs/leef.rb @@ -118,7 +118,6 @@ def store_header_field(event,field_name,field_data) def decode(data, &block) if @delimiter @buffer.extract(data).each do |line| - next if /^\s*$/.match(line) # Skip empty lines handle(line, &block) end else @@ -128,7 +127,7 @@ def decode(data, &block) public def handle(data, &block) - raise "Empty line" if /^\s*$/.match/data + raise "Empty line" if /^\s*$/.match(data) event = LogStash::Event.new event.set(raw_data_field, data) unless @raw_data_field.nil? @@ -359,19 +358,4 @@ def get_value(fieldname, event) end end - #def sanitize_severity(event, severity) - # severity = sanitize_header_field(event.sprintf(severity)).strip - # severity = self.class.get_config["severity"][:default] unless valid_severity?(severity) - # severity = severity.to_i.to_s - #end - - #def valid_severity?(sev) - # f = Float(sev) - # check if it's an integer or a float with no remainder - # and if the value is between 0 and 10 (inclusive) - # (f % 1 == 0) && f.between?(0,10) - #rescue TypeError, ArgumentError - # false - #end - end