Add username and email indicators to pantherlog (#2117)

* Add email scanner and indicator * Add usename indicator and scanner * Add tests for indicator scanners * mage gen fmt Co-authored-by: panther-bot <[email protected]>
glerb · Nov 27, 2020 · 8624f93 · 8624f93
1 parent 4c39080
commit 8624f93
Show file tree

Hide file tree

Showing 3 changed files with 116 additions and 5 deletions.
diff --git a/internal/log_analysis/log_processor/pantherlog/meta.go b/internal/log_analysis/log_processor/pantherlog/meta.go
@@ -70,11 +70,13 @@ const (
 	FieldAWSInstanceID
 	FieldAWSARN
 	FieldAWSTag
+	FieldEmail
+	FieldUsername
 )
 
 // ScanValues implements ValueScanner interface
 func (id FieldID) ScanValues(w ValueWriter, input string) {
-	w.WriteValues(id, input)
+	w.WriteValues(id, strings.TrimSpace(input))
 }
 
 // CoreFields are the 'core' fields Panther adds to each log.
@@ -196,15 +198,25 @@ func init() {
 		NameJSON:    "p_any_aws_tags",
 		Description: "Panther added field with collection of AWS Tags associated with the row",
 	})
-	MustRegisterScanner("ip", ValueScannerFunc(ScanIPAddress), FieldIPAddress)
+	MustRegisterIndicator(FieldEmail, FieldMeta{
+		Name:        "PantherAnyEmails",
+		NameJSON:    "p_any_emails",
+		Description: "Panther added field with collection of email addresses associated with the row",
+	})
+	MustRegisterIndicator(FieldUsername, FieldMeta{
+		Name:        "PantherAnyUsernames",
+		NameJSON:    "p_any_usernames",
+		Description: "Panther added field with collection of usernames associated with the row",
+	})
+	MustRegisterScannerFunc("ip", ScanIPAddress, FieldIPAddress)
 	MustRegisterScanner("domain", FieldDomainName, FieldDomainName)
 	MustRegisterScanner("md5", FieldMD5Hash, FieldMD5Hash)
 	MustRegisterScanner("sha1", FieldSHA1Hash, FieldSHA1Hash)
 	MustRegisterScanner("sha256", FieldSHA256Hash, FieldSHA256Hash)
-	MustRegisterScanner("hostname", ValueScannerFunc(ScanHostname), FieldDomainName, FieldIPAddress)
-	MustRegisterScanner("url", ValueScannerFunc(ScanURL), FieldDomainName, FieldIPAddress)
+	MustRegisterScannerFunc("hostname", ScanHostname, FieldDomainName, FieldIPAddress)
+	MustRegisterScannerFunc("url", ScanURL, FieldDomainName, FieldIPAddress)
 	MustRegisterScanner("trace_id", FieldTraceID, FieldTraceID)
-	MustRegisterScanner("net_addr", ValueScannerFunc(ScanNetworkAddress), FieldIPAddress, FieldDomainName)
+	MustRegisterScannerFunc("net_addr", ScanNetworkAddress, FieldIPAddress, FieldDomainName)
 	MustRegisterScannerFunc("aws_arn", ScanARN,
 		FieldAWSARN,
 		FieldAWSInstanceID,
@@ -213,6 +225,8 @@ func init() {
 	MustRegisterScannerFunc("aws_account_id", ScanAWSAccountID, FieldAWSAccountID)
 	MustRegisterScannerFunc("aws_instance_id", ScanAWSInstanceID, FieldAWSInstanceID)
 	MustRegisterScannerFunc("aws_tag", ScanAWSTag, FieldAWSTag)
+	MustRegisterScannerFunc("email", ScanEmail, FieldEmail)
+	MustRegisterScanner("username", FieldUsername, FieldUsername)
 }
 
 // MustRegisterIndicator allows modules to define their own indicator fields.

diff --git a/internal/log_analysis/log_processor/pantherlog/scanners.go b/internal/log_analysis/log_processor/pantherlog/scanners.go
@@ -118,6 +118,7 @@ func LookupScanner(name string) (scanner ValueScanner, fields []FieldID) {
 
 // ScanURL scans a URL string for domain or ip address
 func ScanURL(dest ValueWriter, input string) {
+	input = strings.TrimSpace(input)
 	if input == "" {
 		return
 	}
@@ -130,6 +131,7 @@ func ScanURL(dest ValueWriter, input string) {
 
 // ScanHostname scans `input` for either an ip address or a domain name value.
 func ScanHostname(w ValueWriter, input string) {
+	input = strings.TrimSpace(input)
 	if checkIPAddress(input) {
 		w.WriteValues(FieldIPAddress, input)
 	} else {
@@ -181,3 +183,10 @@ func (m multiScanner) ScanValues(w ValueWriter, input string) {
 		scanner.ScanValues(w, input)
 	}
 }
+
+func ScanEmail(w ValueWriter, input string) {
+	if pos := strings.IndexByte(input, '@'); pos != -1 {
+		input = strings.TrimSpace(input)
+		w.WriteValues(FieldEmail, input)
+	}
+}
diff --git a/internal/log_analysis/log_processor/pantherlog/scanners_test.go b/internal/log_analysis/log_processor/pantherlog/scanners_test.go
@@ -1,5 +1,11 @@
 package pantherlog
 
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
 /**
  * Panther is a Cloud-Native SIEM for the Modern Security Team.
  * Copyright (C) 2020 Panther Labs Inc
@@ -17,3 +23,85 @@ package pantherlog
  * You should have received a copy of the GNU Affero General Public License
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
+
+func TestScanIPAddress(t *testing.T) {
+	assert := require.New(t)
+	w := &ValueBuffer{}
+
+	ScanIPAddress(w, "foo")
+	assert.True(w.IsEmpty())
+	ScanIPAddress(w, "")
+	assert.True(w.IsEmpty())
+	ScanIPAddress(w, " ")
+	assert.True(w.IsEmpty())
+	ScanIPAddress(w, "[email protected]")
+	assert.True(w.IsEmpty())
+	ScanIPAddress(w, "23.23.23.23")
+	assert.False(w.IsEmpty())
+	assert.Equal([]string{"23.23.23.23"}, w.Get(FieldIPAddress))
+}
+
+func TestScanHostname(t *testing.T) {
+	assert := require.New(t)
+	w := &ValueBuffer{}
+
+	ScanHostname(w, "")
+	assert.True(w.IsEmpty())
+	ScanHostname(w, " ")
+	assert.True(w.IsEmpty())
+	ScanHostname(w, "23.23.23.23")
+	assert.False(w.IsEmpty())
+	assert.Equal([]string{"23.23.23.23"}, w.Get(FieldIPAddress))
+	w.Reset()
+	ScanHostname(w, "foo ")
+	assert.False(w.IsEmpty())
+	assert.Equal([]string{"foo"}, w.Get(FieldDomainName))
+}
+
+func TestScanURL(t *testing.T) {
+	assert := require.New(t)
+	w := &ValueBuffer{}
+	ScanURL(w, "")
+	assert.True(w.IsEmpty())
+	ScanURL(w, " ")
+	assert.True(w.IsEmpty())
+	ScanURL(w, "23.23.23.23")
+	assert.True(w.IsEmpty())
+	ScanURL(w, "http://23.23.23.23/foo")
+	assert.Equal([]string{"23.23.23.23"}, w.Get(FieldIPAddress))
+	w.Reset()
+	ScanURL(w, "foo ")
+	assert.True(w.IsEmpty())
+	ScanURL(w, "http://foo")
+	assert.False(w.IsEmpty())
+	assert.Equal([]string{"foo"}, w.Get(FieldDomainName))
+}
+
+func TestScanEmail(t *testing.T) {
+	assert := require.New(t)
+	w := &ValueBuffer{}
+	ScanEmail(w, "foo")
+	assert.True(w.IsEmpty())
+	ScanEmail(w, "")
+	assert.True(w.IsEmpty())
+	ScanEmail(w, "  ")
+	assert.True(w.IsEmpty())
+	ScanEmail(w, "23.23.23.23")
+	assert.True(w.IsEmpty())
+	ScanEmail(w, "[email protected]")
+	assert.False(w.IsEmpty())
+	ScanEmail(w, "[email protected] ")
+	assert.Equal([]string{"[email protected]"}, w.Get(FieldEmail))
+}
+
+func TestScanDomain(t *testing.T) {
+	assert := require.New(t)
+	w := &ValueBuffer{}
+	FieldDomainName.ScanValues(w, "")
+	assert.True(w.IsEmpty())
+	FieldDomainName.ScanValues(w, "  ")
+	assert.True(w.IsEmpty())
+	FieldDomainName.ScanValues(w, "foo ")
+	assert.False(w.IsEmpty())
+	assert.Equal([]string{"foo"}, w.Get(FieldDomainName))
+}