Skip to content

Commit

Permalink
Use string slice for PantherAnyString (#2309)
Browse files Browse the repository at this point in the history 
* Use string slice for PantherAnyString

* Add test for omitempty behavior

* Be conservative and clone the value instead of modifying it inplace

Co-authored-by: Kostas Papageorgiou <[email protected]>
  • Loading branch information
@alxarch
alxarch and Kostas Papageorgiou authored Dec 18, 2020
1 parent ebb3a60 commit e515277
Show file tree
Hide file tree
Showing 6 changed files with 87 additions and 167 deletions.
11 changes: 5 additions & 6 deletions internal/log_analysis/log_processor/pantherlog/result_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -123,15 +123,14 @@ func TestOldResults(t *testing.T) {
Host: box.String("2.1.1.1"),
Timestamp: (*timestamp.RFC3339)(&tm),
PantherLog: parsers.PantherLog{
PantherLogType: box.String("Foo"),
PantherRowID: box.String("id"),
PantherEventTime: (*timestamp.RFC3339)(&tm),
PantherParseTime: (*timestamp.RFC3339)(&now),
PantherAnyIPAddresses: parsers.NewPantherAnyString(),
PantherLogType: box.String("Foo"),
PantherRowID: box.String("id"),
PantherEventTime: (*timestamp.RFC3339)(&tm),
PantherParseTime: (*timestamp.RFC3339)(&now),
},
}
event.SetEvent(&event)
parsers.AppendAnyString(event.PantherAnyIPAddresses, "1.1.1.1", "2.1.1.1")
parsers.AppendAnyString(&event.PantherAnyIPAddresses, "1.1.1.1", "2.1.1.1")

api := buildAPI()
result := event.Result()
Expand Down
2 changes: 1 addition & 1 deletion internal/log_analysis/log_processor/parsers/awslogs/extractor_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,9 +35,9 @@ func TestAWSExtractor(t *testing.T) {
expectedEvent := AWSPantherLog{}
expectedEvent.AppendAnyAWSARNs(
"arn:aws:cloudtrail:us-west-2:888888888888:trail/panther-lab-cloudtrail",
"arn:aws:iam::123456789012:instance-profile/ArnLike",
"arn:aws:iam::123456789012:instance-profile/EC2Dev",
"arn:aws:ec2:region:111122223333:instance/i-0072230f74b3a798e",
"arn:aws:iam::123456789012:instance-profile/ArnLike",
"arn:aws:ec2:region:111122223333:instance/",
)
expectedEvent.AppendAnyAWSInstanceIds("i-081de1d7604b11e4a", "i-0072230f74b3a798e" /* from ARN */)
Expand Down
28 changes: 8 additions & 20 deletions internal/log_analysis/log_processor/parsers/awslogs/pantherlog.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,10 +32,10 @@ var (
type AWSPantherLog struct {
parsers.PantherLog

PantherAnyAWSAccountIds *parsers.PantherAnyString `json:"p_any_aws_account_ids,omitempty" description:"Panther added field with collection of aws account ids associated with the row"`
PantherAnyAWSInstanceIds *parsers.PantherAnyString `json:"p_any_aws_instance_ids,omitempty" description:"Panther added field with collection of aws instance ids associated with the row"`
PantherAnyAWSARNs *parsers.PantherAnyString `json:"p_any_aws_arns,omitempty" description:"Panther added field with collection of aws arns associated with the row"`
PantherAnyAWSTags *parsers.PantherAnyString `json:"p_any_aws_tags,omitempty" description:"Panther added field with collection of aws tags associated with the row"`
PantherAnyAWSAccountIds parsers.PantherAnyString `json:"p_any_aws_account_ids,omitempty" description:"Panther added field with collection of aws account ids associated with the row"`
PantherAnyAWSInstanceIds parsers.PantherAnyString `json:"p_any_aws_instance_ids,omitempty" description:"Panther added field with collection of aws instance ids associated with the row"`
PantherAnyAWSARNs parsers.PantherAnyString `json:"p_any_aws_arns,omitempty" description:"Panther added field with collection of aws arns associated with the row"`
PantherAnyAWSTags parsers.PantherAnyString `json:"p_any_aws_tags,omitempty" description:"Panther added field with collection of aws tags associated with the row"`
}

func (pl *AWSPantherLog) AppendAnyAWSAccountIdPtrs(values ...*string) { // nolint
Expand All @@ -51,10 +51,7 @@ func (pl *AWSPantherLog) AppendAnyAWSAccountIds(values ...string) {
if !awsAccountIDRegex.MatchString(value) {
continue
}
if pl.PantherAnyAWSAccountIds == nil { // lazy create
pl.PantherAnyAWSAccountIds = parsers.NewPantherAnyString()
}
parsers.AppendAnyString(pl.PantherAnyAWSAccountIds, value)
parsers.AppendAnyString(&pl.PantherAnyAWSAccountIds, value)
}
}

Expand All @@ -67,10 +64,7 @@ func (pl *AWSPantherLog) AppendAnyAWSInstanceIdPtrs(values ...*string) { // noli
}

func (pl *AWSPantherLog) AppendAnyAWSInstanceIds(values ...string) {
if pl.PantherAnyAWSInstanceIds == nil { // lazy create
pl.PantherAnyAWSInstanceIds = parsers.NewPantherAnyString()
}
parsers.AppendAnyString(pl.PantherAnyAWSInstanceIds, values...)
parsers.AppendAnyString(&pl.PantherAnyAWSInstanceIds, values...)
}

func (pl *AWSPantherLog) AppendAnyAWSARNPtrs(values ...*string) {
Expand All @@ -82,10 +76,7 @@ func (pl *AWSPantherLog) AppendAnyAWSARNPtrs(values ...*string) {
}

func (pl *AWSPantherLog) AppendAnyAWSARNs(values ...string) {
if pl.PantherAnyAWSARNs == nil { // lazy create
pl.PantherAnyAWSARNs = parsers.NewPantherAnyString()
}
parsers.AppendAnyString(pl.PantherAnyAWSARNs, values...)
parsers.AppendAnyString(&pl.PantherAnyAWSARNs, values...)
}

func (pl *AWSPantherLog) AppendAnyAWSTagPtrs(values ...*string) {
Expand All @@ -98,8 +89,5 @@ func (pl *AWSPantherLog) AppendAnyAWSTagPtrs(values ...*string) {

// NOTE: value should be of the form <key>:<value>
func (pl *AWSPantherLog) AppendAnyAWSTags(values ...string) {
if pl.PantherAnyAWSTags == nil { // lazy create
pl.PantherAnyAWSTags = parsers.NewPantherAnyString()
}
parsers.AppendAnyString(pl.PantherAnyAWSTags, values...)
parsers.AppendAnyString(&pl.PantherAnyAWSTags, values...)
}
12 changes: 4 additions & 8 deletions internal/log_analysis/log_processor/parsers/awslogs/pantherlog_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,7 @@ import (
func TestAppendAnyAWSAccountIds(t *testing.T) {
event := AWSPantherLog{}
value := "012345678912"
expectedAny := parsers.NewPantherAnyString()
parsers.AppendAnyString(expectedAny, value)
expectedAny := parsers.PantherAnyString{value}
event.AppendAnyAWSAccountIds(value)
require.Equal(t, expectedAny, event.PantherAnyAWSAccountIds)

Expand All @@ -55,8 +54,7 @@ func TestAppendAnyAWSAccountIds(t *testing.T) {
func TestAppendAnyAWSInstanceIds(t *testing.T) {
event := AWSPantherLog{}
value := "a"
expectedAny := parsers.NewPantherAnyString()
parsers.AppendAnyString(expectedAny, value)
expectedAny := parsers.PantherAnyString{value}
event.AppendAnyAWSInstanceIds(value)
require.Equal(t, expectedAny, event.PantherAnyAWSInstanceIds)

Expand All @@ -68,8 +66,7 @@ func TestAppendAnyAWSInstanceIds(t *testing.T) {
func TestAppendAnyAWSARNs(t *testing.T) {
event := AWSPantherLog{}
value := "a"
expectedAny := parsers.NewPantherAnyString()
parsers.AppendAnyString(expectedAny, value)
expectedAny := parsers.PantherAnyString{value}
event.AppendAnyAWSARNs(value)
require.Equal(t, expectedAny, event.PantherAnyAWSARNs)

Expand All @@ -81,8 +78,7 @@ func TestAppendAnyAWSARNs(t *testing.T) {
func TestAppendAnyAWSTags(t *testing.T) {
event := AWSPantherLog{}
value := "a"
expectedAny := parsers.NewPantherAnyString()
parsers.AppendAnyString(expectedAny, value)
expectedAny := parsers.PantherAnyString{value}
event.AppendAnyAWSTags(value)
require.Equal(t, expectedAny, event.PantherAnyAWSTags)

Expand Down
90 changes: 20 additions & 70 deletions internal/log_analysis/log_processor/parsers/pantherlog.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,18 +20,17 @@ package parsers

import (
"net"
"reflect"
"regexp"
"sort"
"time"

jsoniter "github.com/json-iterator/go"

"github.com/panther-labs/panther/internal/log_analysis/awsglue/glueschema"
"github.com/panther-labs/panther/internal/log_analysis/log_processor/pantherlog"
"github.com/panther-labs/panther/internal/log_analysis/log_processor/pantherlog/rowid"
"github.com/panther-labs/panther/internal/log_analysis/log_processor/parsers/timestamp"
"github.com/panther-labs/panther/pkg/box"
"github.com/panther-labs/panther/pkg/stringset"
"github.com/panther-labs/panther/pkg/unbox"
)

Expand Down Expand Up @@ -60,53 +59,22 @@ type PantherLog struct {
PantherSourceLabel *string `json:"p_source_label,omitempty" description:"Panther added field with the source label"`

// optional (any)
PantherAnyIPAddresses *PantherAnyString `json:"p_any_ip_addresses,omitempty" description:"Panther added field with collection of ip addresses associated with the row"`
PantherAnyDomainNames *PantherAnyString `json:"p_any_domain_names,omitempty" description:"Panther added field with collection of domain names associated with the row"`
PantherAnySHA1Hashes *PantherAnyString `json:"p_any_sha1_hashes,omitempty" description:"Panther added field with collection of SHA1 hashes associated with the row"`
PantherAnyMD5Hashes *PantherAnyString `json:"p_any_md5_hashes,omitempty" description:"Panther added field with collection of MD5 hashes associated with the row"`
PantherAnySHA256Hashes *PantherAnyString `json:"p_any_sha256_hashes,omitempty" description:"Panther added field with collection of SHA256 hashes of any algorithm associated with the row"`
PantherAnyIPAddresses PantherAnyString `json:"p_any_ip_addresses,omitempty" description:"Panther added field with collection of ip addresses associated with the row"`
PantherAnyDomainNames PantherAnyString `json:"p_any_domain_names,omitempty" description:"Panther added field with collection of domain names associated with the row"`
PantherAnySHA1Hashes PantherAnyString `json:"p_any_sha1_hashes,omitempty" description:"Panther added field with collection of SHA1 hashes associated with the row"`
PantherAnyMD5Hashes PantherAnyString `json:"p_any_md5_hashes,omitempty" description:"Panther added field with collection of MD5 hashes associated with the row"`
PantherAnySHA256Hashes PantherAnyString `json:"p_any_sha256_hashes,omitempty" description:"Panther added field with collection of SHA256 hashes of any algorithm associated with the row"`
}

type PantherAnyString struct { // needed to declare as struct (rather than map) for CF generation
set map[string]struct{} // map is used for uniqueness, serializes as JSON list
}

func init() {
// Register glue mapping for PantherAnyString
glueschema.MustRegisterMapping(reflect.TypeOf(PantherAnyString{}), glueschema.ArrayOf(glueschema.TypeString))
}

func NewPantherAnyString() *PantherAnyString {
return &PantherAnyString{
set: make(map[string]struct{}),
}
}

func (any *PantherAnyString) MarshalJSON() ([]byte, error) {
if any != nil { // copy to slice
values := make([]string, len(any.set))
i := 0
for k := range any.set {
values[i] = k
i++
}
sort.Strings(values) // sort for consistency and to improve compression when stored
return jsoniter.Marshal(values)
}
return []byte{}, nil
}
type PantherAnyString []string

func (any *PantherAnyString) UnmarshalJSON(jsonBytes []byte) error {
var values []string
err := jsoniter.Unmarshal(jsonBytes, &values)
if err != nil {
return err
}
any.set = make(map[string]struct{}, len(values))
for _, entry := range values {
any.set[entry] = struct{}{}
}
return nil
func (any PantherAnyString) MarshalJSON() ([]byte, error) {
// The safe way to do this is to clone.
// MarshalJSON is not supposed to be altering the value.
// If we used stringset.Dedup + sort.Strings we could corrupt the input slice.
values := stringset.New(any...)
sort.Strings(values)
return jsoniter.Marshal(values)
}

// Event returns event data, used when composed
Expand Down Expand Up @@ -188,10 +156,7 @@ func (pl *PantherLog) AppendAnyIPAddressInField(value string) bool {

func (pl *PantherLog) AppendAnyIPAddress(value string) bool {
if net.ParseIP(value) != nil {
if pl.PantherAnyIPAddresses == nil { // lazy create
pl.PantherAnyIPAddresses = NewPantherAnyString()
}
AppendAnyString(pl.PantherAnyIPAddresses, value)
AppendAnyString(&pl.PantherAnyIPAddresses, value)
return true
}
return false
Expand All @@ -206,10 +171,7 @@ func (pl *PantherLog) AppendAnyDomainNamePtrs(values ...*string) {
}

func (pl *PantherLog) AppendAnyDomainNames(values ...string) {
if pl.PantherAnyDomainNames == nil { // lazy create
pl.PantherAnyDomainNames = NewPantherAnyString()
}
AppendAnyString(pl.PantherAnyDomainNames, values...)
AppendAnyString(&pl.PantherAnyDomainNames, values...)
}

func (pl *PantherLog) AppendAnySHA1HashPtrs(values ...*string) {
Expand All @@ -221,10 +183,7 @@ func (pl *PantherLog) AppendAnySHA1HashPtrs(values ...*string) {
}

func (pl *PantherLog) AppendAnySHA1Hashes(values ...string) {
if pl.PantherAnySHA1Hashes == nil { // lazy create
pl.PantherAnySHA1Hashes = NewPantherAnyString()
}
AppendAnyString(pl.PantherAnySHA1Hashes, values...)
AppendAnyString(&pl.PantherAnySHA1Hashes, values...)
}

func (pl *PantherLog) AppendAnyMD5HashPtrs(values ...*string) {
Expand All @@ -236,17 +195,11 @@ func (pl *PantherLog) AppendAnyMD5HashPtrs(values ...*string) {
}

func (pl *PantherLog) AppendAnyMD5Hashes(values ...string) {
if pl.PantherAnyMD5Hashes == nil { // lazy create
pl.PantherAnyMD5Hashes = NewPantherAnyString()
}
AppendAnyString(pl.PantherAnyMD5Hashes, values...)
AppendAnyString(&pl.PantherAnyMD5Hashes, values...)
}

func (pl *PantherLog) AppendAnySHA256Hashes(values ...string) {
if pl.PantherAnySHA256Hashes == nil { // lazy create
pl.PantherAnySHA256Hashes = NewPantherAnyString()
}
AppendAnyString(pl.PantherAnySHA256Hashes, values...)
AppendAnyString(&pl.PantherAnySHA256Hashes, values...)
}

func (pl *PantherLog) AppendAnySHA256HashesPtr(values ...*string) {
Expand All @@ -263,10 +216,7 @@ func AppendAnyString(any *PantherAnyString, values ...string) {
if v == "" { // ignore empty strings
continue
}
if _, exists := any.set[v]; exists {
continue
}
any.set[v] = struct{}{} // new
*any = stringset.Append(*any, v)
}
}

Expand Down
Loading

0 comments on commit e515277

Please sign in to comment.