Skip to content

Commit

Permalink
Fixes for new 7.10 rsa2elk datasets (elastic#21240)
Browse files Browse the repository at this point in the history 
* Fix bad unicode character used in juniper/netscreen

Some parsers from netwitness wrongly use &#x092 XML entity as a quote
character. This entity translates to UNICODE codepoint U+0092 (PRIVATE
USE 2), which is not printable and can cause problems.

My understanding is that this is the result of either:
- Device logs are encoded in the windows-1252 codepage, or
- Log parsers originally written in windows-1252 codepage.

In this codepage, \x92 represents a quotation mark similar to the
ASCII \x27 single quotation mark (').

I believe someone misunderstood XML's &#xNNN entity as escaping a byte value,
instead of a UNICODE codepoint.

As it is unclear if the original logs contain this special quote, or it's the
result of writting the parsers in a Windows editor, it's better to replace
it's usage with empty captures that skip over this quote.

* Update pipelines for new 7.10 rsa2elk datasets

The original pipelines had been generated with some debugging comments
in them, which made them much larger than necessary.

(cherry picked from commit 24e972f)
  • Loading branch information
@adriansr
adriansr committed Sep 29, 2020
1 parent a98d839 commit 49eb59c
Show file tree
Hide file tree
Showing 12 changed files with 8,297 additions and 13,681 deletions.
159 changes: 53 additions & 106 deletions x-pack/filebeat/module/barracuda/spamfirewall/config/pipeline.js
View file

Large diffs are not rendered by default.

204 changes: 68 additions & 136 deletions x-pack/filebeat/module/cisco/meraki/config/pipeline.js
View file

Large diffs are not rendered by default.

1,140 changes: 380 additions & 760 deletions x-pack/filebeat/module/citrix/netscaler/config/pipeline.js
View file

Large diffs are not rendered by default.

864 changes: 288 additions & 576 deletions x-pack/filebeat/module/cyberark/corepas/config/pipeline.js
View file

Large diffs are not rendered by default.

3 changes: 1 addition & 2 deletions x-pack/filebeat/module/f5/bigipafm/config/pipeline.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,7 @@ var map_getEventCategoryActivity = {

var dup1 = constant("Deny");

var hdr1 = // "Pattern{Field(hfld1,true), Constant(' '), Field(hfld2,true), Constant(' '), Field(hhostname,true), Constant(' '), Field(hfld3,true), Constant(' '), Field(hfld4,true), Constant(' '), Field(hfld5,true), Constant(' [F5@'), Field(hfld6,true), Constant(' '), Field(payload,false)}"
match("HEADER#0:0001", "message", "%{hfld1->} %{hfld2->} %{hhostname->} %{hfld3->} %{hfld4->} %{hfld5->} [F5@%{hfld6->} %{payload}", processor_chain([
var hdr1 = match("HEADER#0:0001", "message", "%{hfld1->} %{hfld2->} %{hhostname->} %{hfld3->} %{hfld4->} %{hfld5->} [F5@%{hfld6->} %{payload}", processor_chain([
setc("header_id","0001"),
setc("messageid","BIGIP_AFM"),
]));
Expand Down
630 changes: 210 additions & 420 deletions x-pack/filebeat/module/fortinet/fortimail/config/pipeline.js
View file

Large diffs are not rendered by default.

66 changes: 22 additions & 44 deletions x-pack/filebeat/module/fortinet/fortimanager/config/pipeline.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -124,8 +124,7 @@ var dup23 = lookup({
key: dup15,
});

var hdr1 = // "Pattern{Constant('date='), Field(hdate,true), Constant(' time='), Field(htime,true), Constant(' devname='), Field(hdevice,true), Constant(' device_id='), Field(hfld1,true), Constant(' log_id='), Field(id,true), Constant(' type='), Field(hfld2,true), Constant(' subtype='), Field(hfld3,true), Constant(' pri='), Field(hseverity,true), Constant(' '), Field(payload,false)}"
match("HEADER#0:0001", "message", "date=%{hdate->} time=%{htime->} devname=%{hdevice->} device_id=%{hfld1->} log_id=%{id->} type=%{hfld2->} subtype=%{hfld3->} pri=%{hseverity->} %{payload}", processor_chain([
var hdr1 = match("HEADER#0:0001", "message", "date=%{hdate->} time=%{htime->} devname=%{hdevice->} device_id=%{hfld1->} log_id=%{id->} type=%{hfld2->} subtype=%{hfld3->} pri=%{hseverity->} %{payload}", processor_chain([
setc("header_id","0001"),
call({
dest: "nwparser.messageid",
Expand All @@ -137,26 +136,22 @@ match("HEADER#0:0001", "message", "date=%{hdate->} time=%{htime->} devname=%{hde
}),
]));

var hdr2 = // "Pattern{Constant('logver='), Field(hfld1,true), Constant(' date='), Field(hdate,true), Constant(' time='), Field(htime,true), Constant(' log_id='), Field(id,true), Constant(' '), Field(payload,false)}"
match("HEADER#1:0002", "message", "logver=%{hfld1->} date=%{hdate->} time=%{htime->} log_id=%{id->} %{payload}", processor_chain([
var hdr2 = match("HEADER#1:0002", "message", "logver=%{hfld1->} date=%{hdate->} time=%{htime->} log_id=%{id->} %{payload}", processor_chain([
setc("header_id","0002"),
dup1,
]));

var hdr3 = // "Pattern{Constant('date='), Field(hdate,true), Constant(' time='), Field(htime,true), Constant(' logver='), Field(fld1,true), Constant(' '), Field(payload,false)}"
match("HEADER#2:0003", "message", "date=%{hdate->} time=%{htime->} logver=%{fld1->} %{payload}", processor_chain([
var hdr3 = match("HEADER#2:0003", "message", "date=%{hdate->} time=%{htime->} logver=%{fld1->} %{payload}", processor_chain([
setc("header_id","0003"),
dup1,
]));

var hdr4 = // "Pattern{Constant('logver='), Field(hfld1,true), Constant(' dtime='), Field(hdatetime,true), Constant(' devid='), Field(hfld2,true), Constant(' devname='), Field(hdevice,true), Constant(' '), Field(payload,false)}"
match("HEADER#3:0004", "message", "logver=%{hfld1->} dtime=%{hdatetime->} devid=%{hfld2->} devname=%{hdevice->} %{payload}", processor_chain([
var hdr4 = match("HEADER#3:0004", "message", "logver=%{hfld1->} dtime=%{hdatetime->} devid=%{hfld2->} devname=%{hdevice->} %{payload}", processor_chain([
setc("header_id","0004"),
dup2,
]));

var hdr5 = // "Pattern{Constant('logver='), Field(hfld1,true), Constant(' devname="'), Field(hdevice,false), Constant('" devid="'), Field(hfld2,false), Constant('" '), Field(payload,false)}"
match("HEADER#4:0005", "message", "logver=%{hfld1->} devname=\"%{hdevice}\" devid=\"%{hfld2}\" %{payload}", processor_chain([
var hdr5 = match("HEADER#4:0005", "message", "logver=%{hfld1->} devname=\"%{hdevice}\" devid=\"%{hfld2}\" %{payload}", processor_chain([
setc("header_id","0005"),
dup2,
]));
Expand All @@ -169,8 +164,7 @@ var select1 = linear_select([
hdr5,
]);

var part1 = // "Pattern{Constant('user='), Field(fld1,true), Constant(' adom='), Field(domain,true), Constant(' user='), Field(username,true), Constant(' ui='), Field(fld2,true), Constant(' action='), Field(action,true), Constant(' status='), Field(event_state,true), Constant(' msg="'), Field(event_description,false), Constant('"')}"
match("MESSAGE#0:fortinetmgr:01", "nwparser.payload", "user=%{fld1->} adom=%{domain->} user=%{username->} ui=%{fld2->} action=%{action->} status=%{event_state->} msg=\"%{event_description}\"", processor_chain([
var part1 = match("MESSAGE#0:fortinetmgr:01", "nwparser.payload", "user=%{fld1->} adom=%{domain->} user=%{username->} ui=%{fld2->} action=%{action->} status=%{event_state->} msg=\"%{event_description}\"", processor_chain([
dup3,
dup4,
dup5,
Expand All @@ -183,8 +177,7 @@ match("MESSAGE#0:fortinetmgr:01", "nwparser.payload", "user=%{fld1->} adom=%{dom

var msg1 = msg("fortinetmgr:01", part1);

var part2 = // "Pattern{Constant('user='), Field(username,true), Constant(' adom='), Field(domain,true), Constant(' msg="'), Field(event_description,false), Constant('"')}"
match("MESSAGE#1:fortinetmgr", "nwparser.payload", "user=%{username->} adom=%{domain->} msg=\"%{event_description}\"", processor_chain([
var part2 = match("MESSAGE#1:fortinetmgr", "nwparser.payload", "user=%{username->} adom=%{domain->} msg=\"%{event_description}\"", processor_chain([
dup3,
dup4,
dup5,
Expand All @@ -197,42 +190,33 @@ match("MESSAGE#1:fortinetmgr", "nwparser.payload", "user=%{username->} adom=%{do

var msg2 = msg("fortinetmgr", part2);

var part3 = // "Pattern{Constant('user="'), Field(username,false), Constant('" userfrom='), Field(fld7,true), Constant(' msg="'), Field(p0,false)}"
match("MESSAGE#2:fortinetmgr:04/0", "nwparser.payload", "user=\"%{username}\" userfrom=%{fld7->} msg=\"%{p0}");
var part3 = match("MESSAGE#2:fortinetmgr:04/0", "nwparser.payload", "user=\"%{username}\" userfrom=%{fld7->} msg=\"%{p0}");

var part4 = // "Pattern{Constant('User'), Field(p0,false)}"
match("MESSAGE#2:fortinetmgr:04/1_0", "nwparser.p0", "User%{p0}");
var part4 = match("MESSAGE#2:fortinetmgr:04/1_0", "nwparser.p0", "User%{p0}");

var part5 = // "Pattern{Constant('user'), Field(p0,false)}"
match("MESSAGE#2:fortinetmgr:04/1_1", "nwparser.p0", "user%{p0}");
var part5 = match("MESSAGE#2:fortinetmgr:04/1_1", "nwparser.p0", "user%{p0}");

var select2 = linear_select([
part4,
part5,
]);

var part6 = // "Pattern{Field(,false), Constant('''), Field(fld3,false), Constant('' with profile ''), Field(fld4,false), Constant('' '), Field(fld5,true), Constant(' from '), Field(fld6,false), Constant('('), Field(hostip,false), Constant(')'), Field(p0,false)}"
match("MESSAGE#2:fortinetmgr:04/2", "nwparser.p0", "%{}'%{fld3}' with profile '%{fld4}' %{fld5->} from %{fld6}(%{hostip})%{p0}");
var part6 = match("MESSAGE#2:fortinetmgr:04/2", "nwparser.p0", "%{}'%{fld3}' with profile '%{fld4}' %{fld5->} from %{fld6}(%{hostip})%{p0}");

var part7 = // "Pattern{Constant('."'), Field(p0,false)}"
match("MESSAGE#2:fortinetmgr:04/3_0", "nwparser.p0", ".\"%{p0}");
var part7 = match("MESSAGE#2:fortinetmgr:04/3_0", "nwparser.p0", ".\"%{p0}");

var part8 = // "Pattern{Constant('"'), Field(p0,false)}"
match("MESSAGE#2:fortinetmgr:04/3_1", "nwparser.p0", "\"%{p0}");
var part8 = match("MESSAGE#2:fortinetmgr:04/3_1", "nwparser.p0", "\"%{p0}");

var select3 = linear_select([
part7,
part8,
]);

var part9 = // "Pattern{Field(,false), Constant('adminprof='), Field(p0,false)}"
match("MESSAGE#2:fortinetmgr:04/4", "nwparser.p0", "%{}adminprof=%{p0}");
var part9 = match("MESSAGE#2:fortinetmgr:04/4", "nwparser.p0", "%{}adminprof=%{p0}");

var part10 = // "Pattern{Field(fld2,true), Constant(' sid='), Field(sid,true), Constant(' user_type="'), Field(profile,false), Constant('"')}"
match("MESSAGE#2:fortinetmgr:04/5_0", "nwparser.p0", "%{fld2->} sid=%{sid->} user_type=\"%{profile}\"");
var part10 = match("MESSAGE#2:fortinetmgr:04/5_0", "nwparser.p0", "%{fld2->} sid=%{sid->} user_type=\"%{profile}\"");

var part11 = // "Pattern{Field(fld2,false)}"
match_copy("MESSAGE#2:fortinetmgr:04/5_1", "nwparser.p0", "fld2");
var part11 = match_copy("MESSAGE#2:fortinetmgr:04/5_1", "nwparser.p0", "fld2");

var select4 = linear_select([
part10,
Expand Down Expand Up @@ -268,8 +252,7 @@ var all1 = all_match({

var msg3 = msg("fortinetmgr:04", all1);

var part12 = // "Pattern{Constant('user='), Field(username,true), Constant(' userfrom='), Field(fld4,true), Constant(' msg="'), Field(event_description,false), Constant('" adminprof='), Field(fld2,false)}"
match("MESSAGE#3:fortinetmgr:02", "nwparser.payload", "user=%{username->} userfrom=%{fld4->} msg=\"%{event_description}\" adminprof=%{fld2}", processor_chain([
var part12 = match("MESSAGE#3:fortinetmgr:02", "nwparser.payload", "user=%{username->} userfrom=%{fld4->} msg=\"%{event_description}\" adminprof=%{fld2}", processor_chain([
dup3,
dup4,
dup5,
Expand All @@ -282,8 +265,7 @@ match("MESSAGE#3:fortinetmgr:02", "nwparser.payload", "user=%{username->} userfr

var msg4 = msg("fortinetmgr:02", part12);

var part13 = // "Pattern{Constant('user="'), Field(username,false), Constant('" msg="Login from ssh:'), Field(fld1,true), Constant(' for '), Field(fld2,true), Constant(' from '), Field(saddr,true), Constant(' port '), Field(sport,false), Constant('" remote_ip="'), Field(daddr,false), Constant('" remote_port='), Field(dport,true), Constant(' valid='), Field(fld3,true), Constant(' authmsg="'), Field(result,false), Constant('" extrainfo='), Field(fld5,false)}"
match("MESSAGE#4:fortinetmgr:03", "nwparser.payload", "user=\"%{username}\" msg=\"Login from ssh:%{fld1->} for %{fld2->} from %{saddr->} port %{sport}\" remote_ip=\"%{daddr}\" remote_port=%{dport->} valid=%{fld3->} authmsg=\"%{result}\" extrainfo=%{fld5}", processor_chain([
var part13 = match("MESSAGE#4:fortinetmgr:03", "nwparser.payload", "user=\"%{username}\" msg=\"Login from ssh:%{fld1->} for %{fld2->} from %{saddr->} port %{sport}\" remote_ip=\"%{daddr}\" remote_port=%{dport->} valid=%{fld3->} authmsg=\"%{result}\" extrainfo=%{fld5}", processor_chain([
dup11,
dup4,
dup5,
Expand All @@ -302,22 +284,18 @@ match("MESSAGE#4:fortinetmgr:03", "nwparser.payload", "user=\"%{username}\" msg=

var msg5 = msg("fortinetmgr:03", part13);

var part14 = // "Pattern{Constant('user="'), Field(username,false), Constant('" userfrom="'), Field(fld1,false), Constant('"msg="'), Field(p0,false)}"
match("MESSAGE#5:fortinetmgr:05/0", "nwparser.payload", "user=\"%{username}\" userfrom=\"%{fld1}\"msg=\"%{p0}");
var part14 = match("MESSAGE#5:fortinetmgr:05/0", "nwparser.payload", "user=\"%{username}\" userfrom=\"%{fld1}\"msg=\"%{p0}");

var part15 = // "Pattern{Constant('dev='), Field(fld2,false), Constant(',vdom='), Field(fld3,false), Constant(',type='), Field(fld4,false), Constant(',key='), Field(fld5,false), Constant(',act='), Field(action,false), Constant(',pkgname='), Field(fld7,false), Constant(',allowaccess='), Field(fld8,false), Constant('"'), Field(p0,false)}"
match("MESSAGE#5:fortinetmgr:05/1_0", "nwparser.p0", "dev=%{fld2},vdom=%{fld3},type=%{fld4},key=%{fld5},act=%{action},pkgname=%{fld7},allowaccess=%{fld8}\"%{p0}");
var part15 = match("MESSAGE#5:fortinetmgr:05/1_0", "nwparser.p0", "dev=%{fld2},vdom=%{fld3},type=%{fld4},key=%{fld5},act=%{action},pkgname=%{fld7},allowaccess=%{fld8}\"%{p0}");

var part16 = // "Pattern{Field(event_description,false), Constant('"'), Field(p0,false)}"
match("MESSAGE#5:fortinetmgr:05/1_1", "nwparser.p0", "%{event_description}\"%{p0}");
var part16 = match("MESSAGE#5:fortinetmgr:05/1_1", "nwparser.p0", "%{event_description}\"%{p0}");

var select5 = linear_select([
part15,
part16,
]);

var part17 = // "Pattern{Field(domain,false), Constant('" adom="')}"
match("MESSAGE#5:fortinetmgr:05/2", "nwparser.p0", "%{domain}\" adom=\"");
var part17 = match("MESSAGE#5:fortinetmgr:05/2", "nwparser.p0", "%{domain}\" adom=\"");

var all2 = all_match({
processors: [
Expand Down
Loading

0 comments on commit 49eb59c

Please sign in to comment.