From ef5c81411e8c9b8a3146a0415bca82c0da5f0607 Mon Sep 17 00:00:00 2001 From: vallabh Date: Wed, 3 Jan 2024 09:14:36 +0000 Subject: [PATCH] 29-provide-json-for-enum-values-for-mapping completed --- Makefile | 2 +- commands/generate_proto.go | 2 + docs/ocsf-tool.md | 2 +- docs/ocsf-tool_config.md | 2 +- docs/ocsf-tool_generate-proto.md | 2 +- docs/ocsf-tool_schema-class-list.md | 2 +- ocsf/mappers/protobuff_v3/mapper.go | 32 +++++- ocsf/mappers/protobuff_v3/package.go | 2 +- ocsf/mappers/protobuff_v3/types.go | 6 +- ocsf/schema/loader-repository.go | 160 +++++++++++++++++++++++++-- ocsf/schema/types.go | 13 ++- 11 files changed, 207 insertions(+), 18 deletions(-) diff --git a/Makefile b/Makefile index e5715a3..0bb16e5 100644 --- a/Makefile +++ b/Makefile @@ -36,6 +36,6 @@ test-compile-proto: clean-output-java clean-output-golang test-create-proto: clean-output-proto # ./bin/ocsf-tool config extensions linux # ./bin/ocsf-tool config profiles cloud container - ./bin/ocsf-tool generate-proto file_activity + ./bin/ocsf-tool generate-proto file_activity security_finding run: build-docs build-project test-create-proto test-compile-proto \ No newline at end of file diff --git a/commands/generate_proto.go b/commands/generate_proto.go index 9f4a7f0..a74a7db 100644 --- a/commands/generate_proto.go +++ b/commands/generate_proto.go @@ -98,6 +98,8 @@ func runGenerateProtoCmd(cmd *cobra.Command, args []string) { // Produce Output mapper.Marshal(events) + mapper.WriteEnumValueMap(protoOutput) + // Print a console message indicating where the output is generated fmt.Printf("Proto files are generated in %s\n", protoOutput) } diff --git a/docs/ocsf-tool.md b/docs/ocsf-tool.md index fb4ef66..53b512f 100644 --- a/docs/ocsf-tool.md +++ b/docs/ocsf-tool.md @@ -14,4 +14,4 @@ * [ocsf-tool generate-proto](ocsf-tool_generate-proto.md) - Generate a Proto file * [ocsf-tool schema-class-list](ocsf-tool_schema-class-list.md) - List all classes in the OCSF schema -###### Auto generated by spf13/cobra on 22-Dec-2023 +###### Auto generated by spf13/cobra on 3-Jan-2024 diff --git a/docs/ocsf-tool_config.md b/docs/ocsf-tool_config.md index 3723433..f90ff17 100644 --- a/docs/ocsf-tool_config.md +++ b/docs/ocsf-tool_config.md @@ -34,4 +34,4 @@ ocsf-tool config values... [flags] * [ocsf-tool](ocsf-tool.md) - -###### Auto generated by spf13/cobra on 22-Dec-2023 +###### Auto generated by spf13/cobra on 3-Jan-2024 diff --git a/docs/ocsf-tool_generate-proto.md b/docs/ocsf-tool_generate-proto.md index 35e3eaf..628759f 100644 --- a/docs/ocsf-tool_generate-proto.md +++ b/docs/ocsf-tool_generate-proto.md @@ -31,4 +31,4 @@ ocsf-tool generate-proto file_activity process_activity * [ocsf-tool](ocsf-tool.md) - -###### Auto generated by spf13/cobra on 22-Dec-2023 +###### Auto generated by spf13/cobra on 3-Jan-2024 diff --git a/docs/ocsf-tool_schema-class-list.md b/docs/ocsf-tool_schema-class-list.md index b85ca25..91818bf 100644 --- a/docs/ocsf-tool_schema-class-list.md +++ b/docs/ocsf-tool_schema-class-list.md @@ -17,4 +17,4 @@ ocsf-tool schema-class-list [flags] * [ocsf-tool](ocsf-tool.md) - -###### Auto generated by spf13/cobra on 22-Dec-2023 +###### Auto generated by spf13/cobra on 3-Jan-2024 diff --git a/ocsf/mappers/protobuff_v3/mapper.go b/ocsf/mappers/protobuff_v3/mapper.go index b3df6e4..06165e5 100644 --- a/ocsf/mappers/protobuff_v3/mapper.go +++ b/ocsf/mappers/protobuff_v3/mapper.go @@ -1,6 +1,7 @@ package protobuff_v3 import ( + "encoding/json" "fmt" "strconv" "strings" @@ -62,6 +63,34 @@ func (mapper *mapper) Marshal(events []schema.Event) { } mapper.RootPackage.Marshal() + +} + +func (mapper *mapper) WriteEnumValueMap(path string) { + + enumValueMap := make(map[string]EnumValue) + + for _, enum := range mapper.Enums { + for _, ev := range enum.values { + enumValueMap[ToEnumValueName(ev.enum.Name+" "+ev.Name)] = EnumValue{ + Name: ev.Name, + Value: ev.Value, + } + } + } + + filePath := commons.PathPrepare(path + "/enum-value-map.json") + + json, jsonError := json.Marshal(enumValueMap) + + if jsonError != nil { + fmt.Println(jsonError.Error()) + return + } + + println("Writing enum value map to " + filePath) + commons.CreateFile(filePath, json) + } func (mapper *mapper) populateFieldsFromAttributes(message *Message, attributes map[string]schema.Attribute) { @@ -138,7 +167,8 @@ func (mapper *mapper) populateFieldsFromAttributes(message *Message, attributes Name: aev.Caption, Value: v, Comment: Comment{ - "Type": "OCSF_VALUE", + "Type": "OCSF_VALUE", + "EnumValue": fmt.Sprintf("%d", v), }, } } diff --git a/ocsf/mappers/protobuff_v3/package.go b/ocsf/mappers/protobuff_v3/package.go index a3f4d94..949a642 100644 --- a/ocsf/mappers/protobuff_v3/package.go +++ b/ocsf/mappers/protobuff_v3/package.go @@ -93,7 +93,7 @@ func (p *Pkg) GetEnums() []*Enum { msgs := maps.Values(GetMapper().Enums) filterFunc := func(e *Enum) bool { - return e.Package.GetName() == p.GetName() + return e.Package.GetFullName() == p.GetFullName() } return commons.Filter(msgs, filterFunc) diff --git a/ocsf/mappers/protobuff_v3/types.go b/ocsf/mappers/protobuff_v3/types.go index 4ae3178..f8ed623 100644 --- a/ocsf/mappers/protobuff_v3/types.go +++ b/ocsf/mappers/protobuff_v3/types.go @@ -69,9 +69,9 @@ type Enum struct { } type EnumValue struct { - Name string - Value int64 - Comment Comment + Name string `json:"name"` + Value int64 `json:"value"` + Comment Comment `json:",omitempty"` enum *Enum } diff --git a/ocsf/schema/loader-repository.go b/ocsf/schema/loader-repository.go index fd91e66..8013a9e 100644 --- a/ocsf/schema/loader-repository.go +++ b/ocsf/schema/loader-repository.go @@ -3,7 +3,9 @@ package schema import ( "encoding/json" "errors" + "fmt" "os" + "strconv" "strings" "github.com/jinzhu/copier" @@ -60,6 +62,9 @@ func (sl *SchemaRepositorySchemaLoader) Load() (*OCSFSchema, error) { sl.processExtensions(ocsfSchema) + jsonOcsfSchema, _ := json.Marshal(ocsfSchema) + commons.CreateFile("./schema.json", jsonOcsfSchema) + return ocsfSchema, nil } @@ -248,6 +253,7 @@ func (sl *SchemaRepositorySchemaLoader) loadSchemaFromDirectory(directory string return nil, schemaVersionError } + println("\nLoading repo dictionary") dictionary := Dictionary{} objectMap := make(map[string]Object) eventMap := make(map[string]Event) @@ -260,6 +266,14 @@ func (sl *SchemaRepositorySchemaLoader) loadSchemaFromDirectory(directory string return nil, dictionaryLoadingError } + println("\nLoading repo categories") + categoriesFile := repoPath("/categories.json") + categories, categoriesLoadingError := sl.loadCategories(categoriesFile) + + if categoriesLoadingError != nil { + return nil, categoriesLoadingError + } + println("\nLoading repo objects") // Load objects defined in json files in the map from /objects directory using schema.Object struct @@ -280,7 +294,7 @@ func (sl *SchemaRepositorySchemaLoader) loadSchemaFromDirectory(directory string // Load events defined in json files in the map from /events directory using schema.Event struct eventsDirectory := repoPath("/events") - eventLoadingError := sl.loadEvents(eventsDirectory, &eventMap, &dictionary) + eventLoadingError := sl.loadEvents(eventsDirectory, &eventMap, &dictionary, &categories) if eventLoadingError != nil { return nil, eventLoadingError @@ -298,6 +312,7 @@ func (sl *SchemaRepositorySchemaLoader) loadSchemaFromDirectory(directory string Version: schemaVersion.Version, Types: dictionary.Types.Attributes, Dictionary: dictionary, + Categories: categories, } return &schema, nil @@ -391,7 +406,7 @@ func (sl *SchemaRepositorySchemaLoader) loadExtensionFromDirectory(path string, // Load events defined in json files in the map from /events directory using schema.Event struct eventsDirectory := commons.CleanPath(path + "/" + "events") - eventLoadingError := sl.loadEvents(eventsDirectory, &schema.Classes, &schema.Dictionary) + eventLoadingError := sl.loadEvents(eventsDirectory, &schema.Classes, &schema.Dictionary, &schema.Categories) if eventLoadingError != nil { return eventLoadingError @@ -433,6 +448,30 @@ func (sl *SchemaRepositorySchemaLoader) loadVersionFromDirectory() (Version, err return version, nil } +// func to load categories from categories.json +func (sl *SchemaRepositorySchemaLoader) loadCategories(path string) (Categories, error) { + + // Declare categories + var categories Categories + + // Categories file path + categoriesFile := commons.CleanPath(path) + + // Load data from file []byte + data, loadDataError := os.ReadFile(categoriesFile) + + if loadDataError != nil { + return categories, loadDataError + } + + // Unmarshal data into categories + if err := json.Unmarshal(data, &categories); err != nil { + return categories, err + } + + return categories, nil +} + // function to load objects from directory func (sl *SchemaRepositorySchemaLoader) loadObjects(directory string, objects *(map[string]Object), dictionary *Dictionary) error { @@ -501,7 +540,7 @@ func (sl *SchemaRepositorySchemaLoader) loadObjects(directory string, objects *( } // function to load events from directory -func (sl *SchemaRepositorySchemaLoader) loadEvents(directory string, events *(map[string]Event), dictionary *Dictionary) error { +func (sl *SchemaRepositorySchemaLoader) loadEvents(directory string, events *(map[string]Event), dictionary *Dictionary, categories *Categories) error { rootDir := commons.Dir(directory) @@ -563,6 +602,58 @@ func (sl *SchemaRepositorySchemaLoader) loadEvents(directory string, events *(ma // extend event sl.extendEvent(&event, events) + // event schema enrichments + category, categoryExists := categories.Attributes[event.Category] + + if categoryExists { + categoryUid, categoryUidExists := event.Attributes["category_uid"] + if categoryUidExists { + + // add category_uid to event + categoryUid.Enum = map[string]EnumAttribute{} + categoryUidString := fmt.Sprintf("%d", category.Uid) + categoryUid.Enum[categoryUidString] = EnumAttribute{ + Caption: category.Caption, + Uid: category.Uid, + Description: category.Description, + } + event.Attributes["category_uid"] = categoryUid + + // add class_uid to event + classUid := fmt.Sprintf("%d%d", (100 * category.Uid), event.Uid) + attributeClassUid := event.Attributes["class_uid"] + attributeClassUid.Enum = Enum{} + attributeClassUid.Enum[classUid] = EnumAttribute{ + Caption: event.Caption, + Description: event.Description, + } + event.Attributes["class_uid"] = attributeClassUid + + // add uid to event + classUidAsInt, classUidConversionError := strconv.Atoi(classUid) + if classUidConversionError == nil { + event.Uid = classUidAsInt + } + + // add type_uid to event + if event.Attributes["activity_id"].Enum != nil { + typeUid, typeUidExists := event.Attributes["type_uid"] + if typeUidExists { + typeUid.Enum = Enum{} + for activityKey, activity := range event.Attributes["activity_id"].Enum { + typeUidKey := fmt.Sprintf("%d%02s", event.Uid, activityKey) + typeUid.Enum[typeUidKey] = EnumAttribute{ + Caption: event.Caption + ": " + activity.Caption, + Description: activity.Description, + } + } + event.Attributes["type_uid"] = typeUid + } + } + + } + } + } return err @@ -629,7 +720,10 @@ func (sl *SchemaRepositorySchemaLoader) extendEvent(item *Event, items *map[stri // set attributes for key, value := range parentItem.Attributes { - item.Attributes[key] = value + _, exists := item.Attributes[key] + if !exists { + item.Attributes[key] = value + } } // if item category is not null then use parent @@ -809,21 +903,52 @@ func (sl *SchemaRepositorySchemaLoader) loadEventFromFile(path string, includeRo event.Extends = e.Extends event.Name = e.Name event.Category = e.Category - event.CategoryName = e.CategoryName + if e.CategoryName != nil { + event.CategoryName = string(e.CategoryName.(string)) + } event.Profiles = e.Profiles event.Uid = e.Uid // iterate over attributes and copy each attribute to object event.Attributes = make(map[string]Attribute) for key, value := range e.Attributes { - event.Attributes[key] = value.(Attribute) + + attr := value.(Attribute) + + if attr.Include != "" { + + enumFilePath := commons.CleanPath(includeRootPath + "/" + attr.Include) + + println("Loading enum from " + enumFilePath) + loadAttributeEnum, loadAttributeEnumError := sl.loadAttributeEnumFromFile(enumFilePath) + + if loadAttributeEnumError != nil { + println(loadAttributeEnumError.Error()) + } + + // Try loading enum from repo if not found in rootPath + if loadAttributeEnumError != nil { + repoEnumFilePath := repoPath(attr.Include) + println("Loading enum from " + repoEnumFilePath) + loadAttributeEnum, loadAttributeEnumError = sl.loadAttributeEnumFromFile(repoEnumFilePath) + } + + if loadAttributeEnumError == nil { + attr.Enum = loadAttributeEnum.Enum + } + } + + attr.Include = "" + + event.Attributes[key] = attr + } return event, nil } -// func detectAndLoadInclude which accepts include file path and loads it using diffrent function based on path. Possible paths are /includes, /profiles +// func detectAndLoadInclude which accepts include file path and loads it using different function based on path. Possible paths are /includes, /profiles func (sl *SchemaRepositorySchemaLoader) includeFile(path string) (map[string]Attribute, error) { // switch based on path prefix @@ -840,6 +965,27 @@ func (sl *SchemaRepositorySchemaLoader) includeFile(path string) (map[string]Att } +// func includeEnum which accepts include file path +func (sl *SchemaRepositorySchemaLoader) loadAttributeEnumFromFile(path string) (Attribute, error) { + + // declare include + var attribute Attribute + + data, loadDataError := os.ReadFile(path) + + if loadDataError != nil { + return attribute, loadDataError + } + + // unmarshal data into include + if err := json.Unmarshal(data, &attribute); err != nil { + return attribute, err + } + + return attribute, nil + +} + // function to load include from file returns Include struct and error. It accepts include file path as string and loads json file from repo Directory and returns Include struct and error func (sl *SchemaRepositorySchemaLoader) loadIncludeFromFile(path string) (Include, error) { diff --git a/ocsf/schema/types.go b/ocsf/schema/types.go index 7911339..0727e11 100644 --- a/ocsf/schema/types.go +++ b/ocsf/schema/types.go @@ -4,6 +4,7 @@ package schema type EnumAttribute struct { Caption string `json:"caption"` Description string `json:"description"` + Uid int `json:"uid"` } // Enum represents a map of string keys to EnumAttribute values. @@ -32,6 +33,8 @@ type Attribute struct { ObjectType string `json:"object_type"` Attributes []string `json:"attributes"` Profile string `json:"profile"` + Uid int `json:"uid"` + Include string `json:"$include"` } // Event represents an event in the schema. @@ -44,7 +47,7 @@ type Event struct { Category string `json:"category"` Caption string `json:"caption"` Profiles []string `json:"profiles"` - CategoryName interface{} `json:"category_name"` + CategoryName string `json:"category_name"` } // Object represents an object in the schema. @@ -94,6 +97,7 @@ type OCSFSchema struct { Types map[string]Type `json:"types"` Version string `json:"version"` Dictionary Dictionary `json:"dictionary"` + Categories Categories `json:"categories"` } type SchemaLoader interface { @@ -167,3 +171,10 @@ type Extension struct { Name string `json:"name"` Uid int `json:"uid"` } + +type Categories struct { + Caption string `json:"caption"` + Name string `json:"name"` + Description string `json:"description"` + Attributes map[string]Attribute `json:"attributes"` +}