more work on the telnet detection feature

cisagov · Apr 14, 2020 · 7ca7add · 7ca7add
1 parent d921cf9
commit 7ca7add
Show file tree

Hide file tree

Showing 7 changed files with 173 additions and 149 deletions.
diff --git a/README.md b/README.md
@@ -204,7 +204,7 @@ Malcolm leverages the following excellent open source tools, among others.
     * Amazon.com, Inc.'s [ICS protocol](https://github.com/amzn?q=zeek) analyzers
     * Andrew Klaus's [zeek-httpattacks](https://github.com/precurse/zeek-httpattacks) plugin for detecting noncompliant HTTP requests
     * Corelight's [bro-xor-exe](https://github.com/corelight/bro-xor-exe-plugin) plugin
-    * Corelight's [community ID](https://github.com/corelight/bro-community-id) flow hashing plugin
+    * Corelight's [community ID](https://github.com/corelight/zeek-community-id) flow hashing plugin
     * Cybera's [Sniffpass](https://github.com/cybera/zeek-sniffpass) plugin for detecting cleartext passwords in HTTP POST requests
     * J-Gras' [Zeek::AF_Packet](https://github.com/J-Gras/zeek-af_packet-plugin) plugin
     * Lexi Brent's [EternalSafety](https://github.com/lexibrent/zeek-EternalSafety) plugin
@@ -891,7 +891,7 @@ The Moloch interface displays both Zeek logs and Moloch sessions alongside each
 
 A few fields of particular mention that help limit returned results to those Zeek logs and Moloch session records generated from the same network connection are [Community ID](https://github.com/corelight/community-id-spec) (`communityId` and `zeek.community_id` in Moloch and Zeek, respectively) and Zeek's [connection UID](https://docs.zeek.org/en/stable/examples/logs/#using-uids) (`zeek.uid`), which Malcolm maps to Moloch's `rootId` field.
 
-Community ID is specification for standard flow hashing [published by Corelight](https://github.com/corelight/community-id-spec) with the intent of making it easier to pivot from one dataset (e.g., Moloch sessions) to another (e.g., Zeek `conn.log` entries). In Malcolm both Moloch and [Zeek](https://github.com/corelight/bro-community-id) populate this value, which makes it possible to filter for a specific network connection and see both data sources' results for that connection.
+Community ID is specification for standard flow hashing [published by Corelight](https://github.com/corelight/community-id-spec) with the intent of making it easier to pivot from one dataset (e.g., Moloch sessions) to another (e.g., Zeek `conn.log` entries). In Malcolm both Moloch and [Zeek](https://github.com/corelight/zeek-community-id) populate this value, which makes it possible to filter for a specific network connection and see both data sources' results for that connection.
 
 The `rootId` field is used by Moloch to link session records together when a particular session has too many packets to be represented by a single session. When normalizing Zeek logs to Moloch's schema, Malcolm piggybacks on `rootId` to store Zeek's [connection UID](https://docs.zeek.org/en/stable/examples/logs/#using-uids) to crossreference entries across Zeek log types. The connection UID is also stored in `zeek.uid`.
 

diff --git a/kibana/dashboards/0ad3d7c2-3441-485e-9dfe-dbb22e84e576.json b/kibana/dashboards/0ad3d7c2-3441-485e-9dfe-dbb22e84e576.json
diff --git a/kibana/dashboards/95479950-41f2-11ea-88fa-7151df485405.json b/kibana/dashboards/95479950-41f2-11ea-88fa-7151df485405.json
diff --git a/logstash/pipelines/zeek/11_zeek_logs.conf b/logstash/pipelines/zeek/11_zeek_logs.conf
@@ -1118,6 +1118,36 @@ filter {
       add_field =>  { "[zeek_cols][service]" => "ldap" }
     }
 
+  } else if ([source] == "login") {
+    #############################################################################################################################
+    # login.log
+    # custom login.log module (rudimentary, telnet/rlogin/rsh analyzers are old and not the greatest)
+
+    dissect {
+      id => "dissect_zeek_login"
+      # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP
+      mapping => {
+        "[message]" => "%{[zeek_cols][ts]}	%{[zeek_cols][uid]}	%{[zeek_cols][orig_h]}	%{[zeek_cols][orig_p]}	%{[zeek_cols][resp_h]}	%{[zeek_cols][resp_p]}	%{[zeek_cols][proto]}	%{[zeek_cols][success]}	%{[zeek_cols][confused]}	%{[zeek_cols][user]}	%{[zeek_cols][client_user]}	%{[zeek_cols][password]}"
+      }
+    }
+    if ("_dissectfailure" in [tags]) {
+      mutate {
+        id => "mutate_split_zeek_login"
+        # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP
+        split => { "[message]" => "	" }
+      }
+      ruby {
+        id => "ruby_zip_zeek_login"
+        init => "$zeek_login_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'success', 'confused', 'user', 'client_user', 'password' ]"
+        code => "event.set('[zeek_cols]', $zeek_login_field_names.zip(event.get('[message]')).to_h)"
+      }
+    }
+
+    if ([zeek_cols][proto]) and ([zeek_cols][proto] != '-') and ([zeek_cols][proto] != '(empty)') and ([zeek_cols][proto] != '') {
+      mutate { id => "mutate_add_field_zeek_login_service"
+               add_field => { "[zeek_cols][service]" => "%{[zeek_cols][proto]}" } }
+    }
+
   } else if ([source] == "modbus") {
     #############################################################################################################################
     # modbus.log
@@ -2316,36 +2346,6 @@ filter {
       }
     }
 
-  } else if ([source] == "telnet") {
-    #############################################################################################################################
-    # telnet.log
-    # custom telnet.log module (rudimentary, still a lot to be improved since the analyzers are pretty confused)
-
-    dissect {
-      id => "dissect_zeek_telnet"
-      # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP
-      mapping => {
-        "[message]" => "%{[zeek_cols][ts]}	%{[zeek_cols][uid]}	%{[zeek_cols][orig_h]}	%{[zeek_cols][orig_p]}	%{[zeek_cols][resp_h]}	%{[zeek_cols][resp_p]}	%{[zeek_cols][success]}	%{[zeek_cols][confused]}	%{[zeek_cols][user]}	%{[zeek_cols][client_user]}	%{[zeek_cols][password]}"
-      }
-    }
-    if ("_dissectfailure" in [tags]) {
-      mutate {
-        id => "mutate_split_zeek_telnet"
-        # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP
-        split => { "[message]" => "	" }
-      }
-      ruby {
-        id => "ruby_zip_zeek_telnet"
-        init => "$zeek_telnet_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'success', 'confused', 'user', 'client_user', 'password' ]"
-        code => "event.set('[zeek_cols]', $zeek_telnet_field_names.zip(event.get('[message]')).to_h)"
-      }
-    }
-
-    mutate {
-      id => "mutate_add_fields_zeek_telnet"
-      add_field =>  { "[zeek_cols][service]" => "telnet" }
-    }
-
   } else if ([source] == "tunnel") {
     #############################################################################################################################
     # tunnel.log

diff --git a/moloch/wise/source.zeeklogs.js b/moloch/wise/source.zeeklogs.js
diff --git a/zeek/config/local.zeek b/zeek/config/local.zeek
@@ -45,7 +45,7 @@ redef SOCKS::default_capture_password = T;
 @load policy/protocols/modbus/track-memmap
 @load policy/protocols/modbus/known-masters-slaves
 @load policy/protocols/mqtt
-@load telnet.zeek
+@load login.zeek
 # @load frameworks/files/detect-MHR
 
 # custom packages installed manually

diff --git a/zeek/config/telnet.zeek → zeek/config/login.zeek b/zeek/config/telnet.zeek → zeek/config/login.zeek
@@ -1,12 +1,12 @@
-module Telnet;
+module Login;
 
-# log telnet, rlogin, and rsh events to telnet.log
+# log telnet, rlogin, and rsh events to login.log
 
 export {
 
   redef enum Log::ID += {
-    ## The telnet protocol logging stream identifier
-    Log_TELNET
+    ## The logging stream identifier
+    Log_LOGIN
   };
 
   type Info : record {
@@ -17,6 +17,8 @@ export {
     ## The connection's 4-tuple of endpoint addresses/port
     id              : conn_id &log;
 
+    ## proto (telnet, rlogin, or rsh)
+    proto           : string &log &optional;
     ## login_success event was seen (successful login)
     success         : bool &log &default = F;
     ## login_confused event was seen (successful login)
@@ -28,18 +30,18 @@ export {
     ## password given for login attempt
     password        : string &log &optional;
 
-    ## whether or not a line has been written to telnet.log
+    ## whether or not a line has been written to login.log
     logged          : bool &default = F;
   };
 
-  ## Event that can be handled to access the :zeek:type:`Telnet::Info`
+  ## Event that can be handled to access the :zeek:type:`Login::Info`
   ## record as it is sent on to the logging framework.
-  global log_telnet : event(rec : Info);
+  global log_login : event(rec : Info);
 }
 
 # Add the state tracking information variable to the connection record
 redef record connection += {
-  telnet : Info &optional;
+  login : Info &optional;
 };
 
 ###############################################
@@ -117,23 +119,39 @@ redef login_timeouts = {
 ###############################################
 
 # telnet, rlogin, rsh
-const telnet_port = { 23/tcp };
-const rlogin_port = { 513/tcp };
-const rsh_port = { 514/tcp };
-redef likely_server_ports += { telnet_port, rlogin_port, rsh_port };
-
-# set_telnet_session - if has not yet been registered in the connection, instantiate
-# the Info record and assign in c$telnet
-function set_telnet_session(c : connection) {
-  if ( ! c?$telnet ) {
+const telnet_port = 23/tcp;
+const telnet_ports = { telnet_port };
+const rlogin_port = 513/tcp;
+const rlogin_ports = { rlogin_port };
+const rsh_port = 514/tcp;
+const rsh_ports = { rsh_port };
+redef likely_server_ports += { telnet_ports, rlogin_ports, rsh_ports };
+
+# set_login_session - if has not yet been registered in the connection, instantiate
+# the Info record and assign in c$login
+function set_login_session(c : connection) {
+  if ( ! c?$login ) {
     local s : Info = [$ts = network_time(), $uid = c$uid, $id = c$id];
-    c$telnet = s;
-    add c$service["telnet"];
+    switch c$id$resp_p {
+      case telnet_port:
+        s$proto = "telnet";
+        add c$service["telnet"];
+        break;
+      case rlogin_port:
+        s$proto = "rlogin";
+        add c$service["rlogin"];
+        break;
+      case rsh_port:
+        s$proto = "rsh";
+        add c$service["rsh"];
+        break;
+    }
+    c$login = s;
   }
 }
 
-# telnet_message - log to telnet.log
-function telnet_message(s : Info) {
+# login_message - log to login.log
+function login_message(s : Info) {
 
   # strip some values that can happen in a "confused" state that aren't really valid values
   if (( s?$user ) && (( s$user == "" ) || ( s$user == "<none>" ) || ( s$user == "<timeout>" )))
@@ -142,70 +160,76 @@ function telnet_message(s : Info) {
     delete s$client_user;
   if (( s?$password ) && (( s$password == "" ) || ( s$password == "<none>" ) || ( s$password == "<timeout>" )))
     delete s$password;
+  if (( s?$proto ) && ( s$proto == "" ))
+    delete s$proto;
 
   s$ts = network_time();
-  Log::write(Telnet::Log_TELNET, s);
+  Log::write(Login::Log_LOGIN, s);
   s$logged = T;
 }
 
-# create log stream for telnet.log and register telnet, rlogin, and rsh analyzers
+# create log stream for login.log and register telnet, rlogin, and rsh analyzers
 event zeek_init() &priority = 5 {
-  Log::create_stream(Telnet::Log_TELNET, [$columns = Info, $ev = log_telnet, $path = "telnet"]);
-  Analyzer::register_for_ports(Analyzer::ANALYZER_TELNET, telnet_port);
-  Analyzer::register_for_ports(Analyzer::ANALYZER_RLOGIN, rlogin_port);
-  Analyzer::register_for_ports(Analyzer::ANALYZER_RSH, rsh_port);
+  Log::create_stream(Login::Log_LOGIN, [$columns = Info, $ev = log_login, $path = "login"]);
+  Analyzer::register_for_ports(Analyzer::ANALYZER_TELNET, telnet_ports);
+  Analyzer::register_for_ports(Analyzer::ANALYZER_RLOGIN, rlogin_ports);
+  Analyzer::register_for_ports(Analyzer::ANALYZER_RSH, rsh_ports);
 }
 
 # login_confused - Generated when tracking of Telnet/Rlogin authentication failed
 # https://docs.zeek.org/en/current/scripts/base/bif/plugins/Zeek_Login.events.bif.zeek.html#id-login_confused
 event login_confused(c : connection, msg : string, line : string) &priority = 5 {
   # print "login_confused", msg, line;
 
-  set_telnet_session(c);
+  set_login_session(c);
 
-  c$telnet$confused = T;
+  c$login$confused = T;
 }
 
 # login_failure - Generated when tracking of Telnet/Rlogin authentication failed
 # https://docs.zeek.org/en/current/scripts/base/bif/plugins/Zeek_Login.events.bif.zeek.html#id-login_failure
 event login_failure(c : connection, user : string, client_user : string, password : string, line : string) &priority = 5 {
   # print "login_failure", user, client_user, password, line;
 
-  set_telnet_session(c);
+  set_login_session(c);
 
-  if (c$telnet$user == "")
-    c$telnet$user = user;
-  if (c$telnet$client_user == "")
-    c$telnet$client_user = client_user;
-  if (c$telnet$password == "")
-    c$telnet$password = password;
+  if ((!c$login?$user) || (c$login$user == ""))
+    c$login$user = user;
+  if ((!c$login?$client_user) || (c$login$client_user == ""))
+    c$login$client_user = client_user;
+  if ((!c$login?$password) || (c$login$password == ""))
+    c$login$password = password;
 
-  telnet_message(c$telnet);
+  login_message(c$login);
 }
 
 # login_success - Generated for successful Telnet/Rlogin logins
 # https://docs.zeek.org/en/current/scripts/base/bif/plugins/Zeek_Login.events.bif.zeek.html#id-login_success
 event login_success(c : connection, user : string, client_user : string, password : string, line : string) &priority = 5 {
   # print "login_success", user, client_user, password, line;
 
-  set_telnet_session(c);
+  set_login_session(c);
+
+  c$login$success = T;
+  c$login$user = user;
+  c$login$client_user = client_user;
 
-  c$telnet$success = T;
-  c$telnet$user = user;
-  c$telnet$client_user = client_user;
-  c$telnet$password = password;
+  # it appears for a successful login with rsh where client_user was checked, what we're getting in
+  # the "password" field is actually not the password, but the first line of data
+  if ((c$login$proto != "rsh") || (c$login$client_user == ""))
+    c$login$password = password;
 
-  telnet_message(c$telnet);
+  login_message(c$login);
 }
 
 event connection_state_remove(c : connection) &priority = -5 {
-  if (c?$telnet) {
+  if (c?$login) {
 
-    if ( c$telnet$logged == F) {
-      telnet_message(c$telnet);
+    if ( c$login$logged == F) {
+      login_message(c$login);
     }
 
-    delete c$telnet;
+    delete c$login;
   }
 }