From c0229c4536a6fbfa296a0f980f97f3f9321697d7 Mon Sep 17 00:00:00 2001 From: Paul Querna Date: Sun, 12 May 2024 17:31:01 -0700 Subject: [PATCH 1/6] use the OCSF value for enums, rather than an offset --- ocsf/mappers/protobuff_v3/enum.go | 7 ++++--- ocsf/mappers/protobuff_v3/enumValue.go | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/ocsf/mappers/protobuff_v3/enum.go b/ocsf/mappers/protobuff_v3/enum.go index c329a82..e639b5c 100644 --- a/ocsf/mappers/protobuff_v3/enum.go +++ b/ocsf/mappers/protobuff_v3/enum.go @@ -30,7 +30,8 @@ func (e *Enum) GetValues() []*EnumValue { // Add UNKNOWN if not present if !e.HasUnknown() { e.AddValue(&EnumValue{ - Name: "UNKNOWN", + Name: "UNKNOWN", + Value: 0, Comment: Comment{ "Type": "NON_OCSF_VALUE", }, @@ -59,8 +60,8 @@ func (e *Enum) Marshal() string { values := e.GetValues() // Marshal values and add to content - for i, v := range values { - content = append(content, "\t"+v.Marshal(i)) + for _, v := range values { + content = append(content, "\t"+v.Marshal()) } // Close Enum diff --git a/ocsf/mappers/protobuff_v3/enumValue.go b/ocsf/mappers/protobuff_v3/enumValue.go index b9e22be..9194cbc 100644 --- a/ocsf/mappers/protobuff_v3/enumValue.go +++ b/ocsf/mappers/protobuff_v3/enumValue.go @@ -7,12 +7,12 @@ import ( "github.com/iancoleman/strcase" ) -func (ev *EnumValue) Marshal(index int) string { +func (ev *EnumValue) Marshal() string { content := []string{} content = append(content, ToEnumValueName(ev.enum.Name+" "+ev.Name)) - content = append(content, fmt.Sprintf("= %d;", index)) + content = append(content, fmt.Sprintf("= %d;", ev.Value)) if len(ev.Comment) > 0 { content = append(content, "//") From 6611495359a196cd4530bddd1a8dc89b9975ff69 Mon Sep 17 00:00:00 2001 From: Paul Querna Date: Mon, 13 May 2024 14:03:50 -0700 Subject: [PATCH 2/6] handle non zero unknowns --- ocsf/mappers/protobuff_v3/enum.go | 2 +- ocsf/mappers/protobuff_v3/enumValue.go | 13 +++++++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/ocsf/mappers/protobuff_v3/enum.go b/ocsf/mappers/protobuff_v3/enum.go index e639b5c..f04bde5 100644 --- a/ocsf/mappers/protobuff_v3/enum.go +++ b/ocsf/mappers/protobuff_v3/enum.go @@ -89,7 +89,7 @@ func (e *Enum) GetPackage() string { // Enum has at least one value ending with UNKNOWN func (e *Enum) HasUnknown() bool { for _, v := range e.values { - if strings.HasSuffix(strings.ToUpper(v.Name), "UNKNOWN") { + if strings.HasSuffix(strings.ToUpper(v.Name), "UNKNOWN") && v.Value == 0 { return true } } diff --git a/ocsf/mappers/protobuff_v3/enumValue.go b/ocsf/mappers/protobuff_v3/enumValue.go index 9194cbc..01220a7 100644 --- a/ocsf/mappers/protobuff_v3/enumValue.go +++ b/ocsf/mappers/protobuff_v3/enumValue.go @@ -2,6 +2,7 @@ package protobuff_v3 import ( "fmt" + "strconv" "strings" "github.com/iancoleman/strcase" @@ -10,7 +11,15 @@ import ( func (ev *EnumValue) Marshal() string { content := []string{} - content = append(content, ToEnumValueName(ev.enum.Name+" "+ev.Name)) + var baseName string + // some OCSF Values have non-Zero UNKNOKWN values, while protos want a zero value unknown. + // class_uid * 100 + activity_id. + if strings.HasSuffix(strings.ToUpper(ev.Name), "UNKNOWN") && ev.Value != 0 { + baseName = ev.enum.Name + " " + ev.Name + " OCSF " + strconv.FormatInt(int64(ev.Value), 10) + } else { + baseName = ev.enum.Name + " " + ev.Name + } + content = append(content, ToEnumValueName(baseName)) content = append(content, fmt.Sprintf("= %d;", ev.Value)) @@ -30,7 +39,7 @@ func ToEnumValueName(input string) string { value, exists := GetMapper().Cache.EnumValues.Get(input) if exists { - return fmt.Sprint(value) + return value.(string) } output := input From 4b30a6599f2f432a450ddb1d5e365268bfdd3d8b Mon Sep 17 00:00:00 2001 From: Paul Querna Date: Mon, 13 May 2024 14:14:09 -0700 Subject: [PATCH 3/6] simplify --- ocsf/mappers/protobuff_v3/enum.go | 17 +------- ocsf/mappers/protobuff_v3/enum_test.go | 56 +++----------------------- 2 files changed, 7 insertions(+), 66 deletions(-) diff --git a/ocsf/mappers/protobuff_v3/enum.go b/ocsf/mappers/protobuff_v3/enum.go index f04bde5..e2a76d7 100644 --- a/ocsf/mappers/protobuff_v3/enum.go +++ b/ocsf/mappers/protobuff_v3/enum.go @@ -42,7 +42,7 @@ func (e *Enum) GetValues() []*EnumValue { // Sort values by name (with UNKNOWN coming first) sort.Slice(values, func(i, j int) bool { - return valueSorter(values, i, j) + return values[i].Value < values[j].Value }) return values @@ -96,21 +96,6 @@ func (e *Enum) HasUnknown() bool { return false } -// Sorts EnumValues by name, with "UNKNOWN" coming first -func valueSorter(values []*EnumValue, i int, j int) bool { - - // if string ends with UNKNOWN, it should be first - if strings.HasSuffix(strings.ToUpper(values[i].Name), "UNKNOWN") { - return true - } - - if strings.HasSuffix(strings.ToUpper(values[j].Name), "UNKNOWN") { - return false - } - - return values[i].Name < values[j].Name -} - // ToEnumName converts a string to a valid Enum Name func ToEnumName(input string) string { diff --git a/ocsf/mappers/protobuff_v3/enum_test.go b/ocsf/mappers/protobuff_v3/enum_test.go index 3a9c1a6..1cd4da8 100644 --- a/ocsf/mappers/protobuff_v3/enum_test.go +++ b/ocsf/mappers/protobuff_v3/enum_test.go @@ -7,37 +7,15 @@ import ( func TestValueSorter(t *testing.T) { values := []*EnumValue{ - {Name: "UNKNOWN"}, - {Name: "B"}, - {Name: "A"}, - {Name: "C"}, - {Name: "D"}, - {Name: "E"}, - {Name: "F"}, - {Name: "G"}, - {Name: "H"}, - {Name: "I"}, - {Name: "J"}, - {Name: "K"}, - {Name: "L"}, - {Name: "M"}, - {Name: "N"}, - {Name: "O"}, - {Name: "P"}, - {Name: "Q"}, - {Name: "R"}, - {Name: "S"}, - {Name: "T"}, - {Name: "U"}, - {Name: "V"}, - {Name: "W"}, - {Name: "X"}, - {Name: "Y"}, - {Name: "Z"}, + {Name: "UNKNOWN", Value: 0}, + {Name: "B", Value: 2}, + {Name: "A", Value: 1}, + {Name: "C", Value: 3}, + {Name: "D", Value: 4}, } sort.Slice(values, func(i, j int) bool { - return valueSorter(values, i, j) + return values[i].Value < values[j].Value }) expected := []string{ @@ -46,28 +24,6 @@ func TestValueSorter(t *testing.T) { "B", "C", "D", - "E", - "F", - "G", - "H", - "I", - "J", - "K", - "L", - "M", - "N", - "O", - "P", - "Q", - "R", - "S", - "T", - "U", - "V", - "W", - "X", - "Y", - "Z", } for i, v := range values { From 46a2378289ad00156c103b3705e3b7470ffac0f4 Mon Sep 17 00:00:00 2001 From: Paul Querna Date: Mon, 13 May 2024 14:19:37 -0700 Subject: [PATCH 4/6] use UNSPECIFIED anyways --- ocsf/mappers/protobuff_v3/enum.go | 12 ++++++------ ocsf/mappers/protobuff_v3/enumValue.go | 13 ++++--------- ocsf/mappers/protobuff_v3/enum_test.go | 4 ++-- 3 files changed, 12 insertions(+), 17 deletions(-) diff --git a/ocsf/mappers/protobuff_v3/enum.go b/ocsf/mappers/protobuff_v3/enum.go index e2a76d7..3697fbc 100644 --- a/ocsf/mappers/protobuff_v3/enum.go +++ b/ocsf/mappers/protobuff_v3/enum.go @@ -27,10 +27,10 @@ func (e *Enum) GetValue(name string) (*EnumValue, bool) { // Get enum values sorted by name (with UNKNOWN coming first) func (e *Enum) GetValues() []*EnumValue { - // Add UNKNOWN if not present - if !e.HasUnknown() { + // Add UNSPECIFIED if not present + if !e.HasUNSPECIFIED() { e.AddValue(&EnumValue{ - Name: "UNKNOWN", + Name: "UNSPECIFIED", Value: 0, Comment: Comment{ "Type": "NON_OCSF_VALUE", @@ -86,10 +86,10 @@ func (e *Enum) GetPackage() string { return e.Package.GetFullName() } -// Enum has at least one value ending with UNKNOWN -func (e *Enum) HasUnknown() bool { +// Enum has at least one value ending with HasUNSPECIFIED +func (e *Enum) HasUNSPECIFIED() bool { for _, v := range e.values { - if strings.HasSuffix(strings.ToUpper(v.Name), "UNKNOWN") && v.Value == 0 { + if v.Value == 0 { return true } } diff --git a/ocsf/mappers/protobuff_v3/enumValue.go b/ocsf/mappers/protobuff_v3/enumValue.go index 01220a7..cbabbf9 100644 --- a/ocsf/mappers/protobuff_v3/enumValue.go +++ b/ocsf/mappers/protobuff_v3/enumValue.go @@ -2,7 +2,6 @@ package protobuff_v3 import ( "fmt" - "strconv" "strings" "github.com/iancoleman/strcase" @@ -11,14 +10,10 @@ import ( func (ev *EnumValue) Marshal() string { content := []string{} - var baseName string - // some OCSF Values have non-Zero UNKNOKWN values, while protos want a zero value unknown. - // class_uid * 100 + activity_id. - if strings.HasSuffix(strings.ToUpper(ev.Name), "UNKNOWN") && ev.Value != 0 { - baseName = ev.enum.Name + " " + ev.Name + " OCSF " + strconv.FormatInt(int64(ev.Value), 10) - } else { - baseName = ev.enum.Name + " " + ev.Name - } + // NOTE: some OCSF Values have non-Zero UNKNOKWN values, while protos want a zero value unknown. + // class_uid * 100 + activity_id. -> These are OCSF values, and we still emit a UNSPECIFRIED=0 enum + // value + baseName := ev.enum.Name + " " + ev.Name content = append(content, ToEnumValueName(baseName)) content = append(content, fmt.Sprintf("= %d;", ev.Value)) diff --git a/ocsf/mappers/protobuff_v3/enum_test.go b/ocsf/mappers/protobuff_v3/enum_test.go index 1cd4da8..408b7cc 100644 --- a/ocsf/mappers/protobuff_v3/enum_test.go +++ b/ocsf/mappers/protobuff_v3/enum_test.go @@ -7,7 +7,7 @@ import ( func TestValueSorter(t *testing.T) { values := []*EnumValue{ - {Name: "UNKNOWN", Value: 0}, + {Name: "UNSPECIFIED", Value: 0}, {Name: "B", Value: 2}, {Name: "A", Value: 1}, {Name: "C", Value: 3}, @@ -19,7 +19,7 @@ func TestValueSorter(t *testing.T) { }) expected := []string{ - "UNKNOWN", + "UNSPECIFIED", "A", "B", "C", From c60540721bf6b4736e17084d9d00b62bcb1f3cb5 Mon Sep 17 00:00:00 2001 From: Paul Querna Date: Mon, 13 May 2024 14:28:08 -0700 Subject: [PATCH 5/6] handle string enums --- ocsf/mappers/protobuff_v3/mapper.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/ocsf/mappers/protobuff_v3/mapper.go b/ocsf/mappers/protobuff_v3/mapper.go index 06165e5..1b79be9 100644 --- a/ocsf/mappers/protobuff_v3/mapper.go +++ b/ocsf/mappers/protobuff_v3/mapper.go @@ -124,6 +124,16 @@ func (mapper *mapper) populateFieldsFromAttributes(message *Message, attributes if len(attr.Enum) > 0 { field.Type = FIELD_TYPE_ENUM + if field.DataType == "string_t" { + // there are a couple string enums that are not really strings + // and not really enums, but we treat them as string primatives + field.Type = FIELD_TYPE_PRIMITIVE + validKeys := make([]string, 0, len(attr.Enum)) + for aek := range attr.Enum { + validKeys = append(validKeys, aek) + } + field.Comment["AllowedValues"] = strings.Join(validKeys, ", ") + } } // Processing Based on Type From ed29478e60b7df423c47e061632b61873ce63fc1 Mon Sep 17 00:00:00 2001 From: Paul Querna Date: Mon, 13 May 2024 14:33:09 -0700 Subject: [PATCH 6/6] cleanup logic --- ocsf/mappers/protobuff_v3/mapper.go | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/ocsf/mappers/protobuff_v3/mapper.go b/ocsf/mappers/protobuff_v3/mapper.go index 1b79be9..9d63442 100644 --- a/ocsf/mappers/protobuff_v3/mapper.go +++ b/ocsf/mappers/protobuff_v3/mapper.go @@ -122,18 +122,16 @@ func (mapper *mapper) populateFieldsFromAttributes(message *Message, attributes field.Type = FIELD_TYPE_OBJECT } - if len(attr.Enum) > 0 { - field.Type = FIELD_TYPE_ENUM - if field.DataType == "string_t" { - // there are a couple string enums that are not really strings - // and not really enums, but we treat them as string primatives - field.Type = FIELD_TYPE_PRIMITIVE - validKeys := make([]string, 0, len(attr.Enum)) - for aek := range attr.Enum { - validKeys = append(validKeys, aek) - } - field.Comment["AllowedValues"] = strings.Join(validKeys, ", ") + if len(attr.Enum) > 0 && attr.Type == "string_t" { + // there are a couple string_t enums that should just be represented as strings + field.Type = FIELD_TYPE_PRIMITIVE + validKeys := make([]string, 0, len(attr.Enum)) + for aek := range attr.Enum { + validKeys = append(validKeys, aek) } + field.Comment["AllowedValues"] = strings.Join(validKeys, ", ") + } else if len(attr.Enum) > 0 { + field.Type = FIELD_TYPE_ENUM } // Processing Based on Type