From 0bc4608b7f8dbdb58cbd6e8d7f0f067f1a980be1 Mon Sep 17 00:00:00 2001 From: vallabh Date: Sun, 17 Dec 2023 04:14:10 +0000 Subject: [PATCH] added support for repository based schema loading --- .gitignore | 8 +- Makefile | 4 +- commands/config.go | 69 ++- commands/generate_proto.go | 2 +- commands/root.go | 4 + commands/schema_class_list.go | 2 +- commons/arrays.go | 20 + commons/file.go | 101 ++++ commons/file_test.go | 46 ++ commons/fileutils.go | 36 -- commons/git.go | 88 +++ commons/hash.go | 13 + commons/json.go | 16 + commons/maps.go | 13 + config/config.go | 4 + doc-generator/doc-generator.go | 2 +- docs/ocsf-tool.md | 2 +- docs/ocsf-tool_config.md | 3 +- docs/ocsf-tool_generate-proto.md | 2 +- docs/ocsf-tool_schema-class-list.md | 2 +- go.mod | 13 + go.sum | 42 ++ ocsf/mappers/protobuff_v3/mapper.go | 7 +- ocsf/mappers/protobuff_v3/types.go | 2 +- ocsf/schema/OCSFSchema.go | 141 ----- ocsf/schema/loader-repository.go | 836 ++++++++++++++++++++++++++++ ocsf/schema/loader-server.go | 193 +++++++ ocsf/schema/schema.go | 93 ++++ ocsf/schema/types.go | 100 +++- 29 files changed, 1661 insertions(+), 203 deletions(-) create mode 100644 commons/file.go create mode 100644 commons/file_test.go delete mode 100644 commons/fileutils.go create mode 100644 commons/git.go create mode 100644 commons/hash.go create mode 100644 commons/json.go create mode 100644 commons/maps.go delete mode 100644 ocsf/schema/OCSFSchema.go create mode 100644 ocsf/schema/loader-repository.go create mode 100644 ocsf/schema/loader-server.go create mode 100644 ocsf/schema/schema.go diff --git a/.gitignore b/.gitignore index 56e44bf..9b8454c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,8 @@ -output -bin +/output +/bin ocsf-schema.json ocsf-schema-*.json download/ocsf-tool/ -config.yaml \ No newline at end of file +config.yaml +/schema +/schema/* \ No newline at end of file diff --git a/Makefile b/Makefile index 46d8976..e5715a3 100644 --- a/Makefile +++ b/Makefile @@ -34,8 +34,8 @@ test-compile-proto: clean-output-java clean-output-golang find ./output/proto -type f -name "*.proto" | xargs protoc --proto_path=./output/proto --java_out=./output/java --go_opt=paths=source_relative --go_out=./output/golang test-create-proto: clean-output-proto - ./bin/ocsf-tool config extensions linux - ./bin/ocsf-tool config profiles cloud container + # ./bin/ocsf-tool config extensions linux + # ./bin/ocsf-tool config profiles cloud container ./bin/ocsf-tool generate-proto file_activity run: build-docs build-project test-create-proto test-compile-proto \ No newline at end of file diff --git a/commands/config.go b/commands/config.go index 701d69b..f25c814 100644 --- a/commands/config.go +++ b/commands/config.go @@ -1,11 +1,42 @@ package commands import ( + "errors" + "github.com/spf13/cobra" "github.com/spf13/viper" + "github.com/valllabh/ocsf-tool/commons" "github.com/valllabh/ocsf-tool/config" ) +type ConfigVariable struct { + variable string + help string + example string + type_ string +} + +var CONFIGURATIONS map[string]ConfigVariable = map[string]ConfigVariable{ + "extensions": ConfigVariable{ + variable: "extensions", + help: "OCSF Extensions", + example: "linux win", + type_: "[]string", + }, + "profiles": ConfigVariable{ + variable: "profiles", + help: "OCSF Profiles", + example: "cloud linux/linux_users", + type_: "[]string", + }, + "schema.loading.strategy": ConfigVariable{ + variable: "schema.loading.strategy", + help: "Schema Loading Strategy. Possible values: server, repository", + example: "repository", + type_: "string", + }, +} + // Define the setConfig command var configCmd = &cobra.Command{ Use: `config values...`, @@ -13,6 +44,7 @@ var configCmd = &cobra.Command{ Example: ` ocsf-tool config extensions linux win ocsf-tool config profiles cloud linux/linux_users + ocfs-tool config schema.loading.strategy repository `, Long: ` Set configuration values for extensions and profiles @@ -31,20 +63,49 @@ var configCmd = &cobra.Command{ // Extract the variable and values from the args variable, values := args[0], args[1:] + configMeta, configError := getConfig(variable) + // validate variable - if variable != "extensions" && variable != "profiles" { - println("Invalid variable name. Possible values are [extensions, profiles].") + if configError != nil { + println("Invalid config variable:", variable) + // print help + cmd.Help() return } - // set the config value - viper.Set(variable, values) + // switch on the type of the config variable and set the value + switch configMeta.type_ { + case "[]string": + viper.Set(variable, values) + case "string": + viper.Set(variable, values[0]) + } // Write the config file to disk config.WriteConfig() }, } +// getConfig returns ConfigVariable of the given config variable or error if the config variable is invalid +func getConfig(variable string) (ConfigVariable, error) { + // validate variable + if !isValidConfigVariable(variable) { + return ConfigVariable{}, errors.New("Invalid config variable: " + variable) + } + + return CONFIGURATIONS[variable], nil +} + +// getValidConfigVariables returns a list of valid config variables +func getValidConfigVariables() []string { + return commons.GetMapKeys(CONFIGURATIONS) +} + +// isValidConfigVariable checks if the given variable is a valid config variable +func isValidConfigVariable(variable string) bool { + return commons.Contains(getValidConfigVariables(), variable) +} + func init() { // Add the setConfig command to the root command rootCmd.AddCommand(configCmd) diff --git a/commands/generate_proto.go b/commands/generate_proto.go index c9067a6..9f4a7f0 100644 --- a/commands/generate_proto.go +++ b/commands/generate_proto.go @@ -52,7 +52,7 @@ func runGenerateProtoCmd(cmd *cobra.Command, args []string) { golangPackageName, _ := cmd.Flags().GetString("golang-root-package") javaPackageName, _ := cmd.Flags().GetString("java-root-package") - ocsfSchema, _ := schema.LoadOCSFSchema() + ocsfSchema := schema.LoadOCSFSchema() events := []schema.Event{} mapper := protobuff_v3.NewMapper(ocsfSchema) diff --git a/commands/root.go b/commands/root.go index f960cec..0042ae5 100644 --- a/commands/root.go +++ b/commands/root.go @@ -8,10 +8,14 @@ import ( // Define the root command var rootCmd = &cobra.Command{ Use: "ocsf-tool", + PostRun: func(cmd *cobra.Command, args []string) { // Write the config file to disk config.WriteConfig() }, + PersistentPostRun: func(cmd *cobra.Command, args []string) { + println("--") + }, } // Initialize the root command diff --git a/commands/schema_class_list.go b/commands/schema_class_list.go index ffd52f6..a04d408 100644 --- a/commands/schema_class_list.go +++ b/commands/schema_class_list.go @@ -16,7 +16,7 @@ var SchemaClassListCmd = &cobra.Command{ // Define the run function for the SchemaClassListCmd command func runSchemaClassListCmd(cmd *cobra.Command, args []string) { - ocsfSchema, _ := schema.LoadOCSFSchema() + ocsfSchema := schema.LoadOCSFSchema() classes := ocsfSchema.Classes // Group classes by Category diff --git a/commons/arrays.go b/commons/arrays.go index 6afd5d4..6087ef5 100644 --- a/commons/arrays.go +++ b/commons/arrays.go @@ -9,3 +9,23 @@ func Filter[T any](arr []T, filter func(T) bool) []T { } return filtered } + +// function to check if an element exists in a slice +// Example: Contains([]string{"a", "b", "c"}, "b") -> true +func Contains[T comparable](arr []T, element T) bool { + + // check if arr is not nil and empty + if len(arr) == 0 { + return false + } + + // iterate over the array and check if the element exists + for _, a := range arr { + if a == element { + return true + } + } + + // return false if the element does not exist + return false +} diff --git a/commons/file.go b/commons/file.go new file mode 100644 index 0000000..92510a6 --- /dev/null +++ b/commons/file.go @@ -0,0 +1,101 @@ +package commons + +import ( + "os" + "path/filepath" + "strings" +) + +// EnsureDirExists checks if a directory exists at the given path, and creates it if it doesn't exist. +func EnsureDirExists(path string) error { + + path = CleanPath(path) + path = Dir(path) + + if !PathExists(path) { + err := os.MkdirAll(path, 0755) + if err != nil { + return err + } + } + return nil +} + +// Extract Dir path from file path +func Dir(filePath string) string { + return filepath.Dir(filePath) +} + +// Path Exists +func PathExists(path string) bool { + if _, err := os.Stat(path); os.IsNotExist(err) { + return false + } + return true +} + +// CreateFile creates a file at the given path and writes the contents to the file. +// It overwrites the file if it already exists. +func CreateFile(path string, contents []byte) error { + + // Ensure the directory exists + err := EnsureDirExists(path) + if err != nil { + return err + } + + // Write contents to the file + err = os.WriteFile(path, contents, 0644) + if err != nil { + return err + } + + return nil +} + +// PathPrepare replaces $HOME, $TMP, $CWD with their respective paths +func PathPrepare(path string) string { + + // replace $HOME with Home path if path contains $HOME is present + if strings.HasPrefix(path, "$HOME") { + path = strings.Replace(path, "$HOME", os.Getenv("HOME"), 1) + } + + // replace $TMP with Temp path if path contains $TMP is present + if strings.HasPrefix(path, "$TMP") { + path = strings.Replace(path, "$TMP", os.Getenv("TMP"), 1) + } + + // replace $CWD with current working path if path contains $CWD is present + if strings.HasPrefix(path, "$CWD") { + path = strings.Replace(path, "$CWD", os.Getenv("PWD"), 1) + } + + // remove double slashes + path = CleanPath(path) + + return path +} + +// clean Path string +func CleanPath(path string) string { + // remove double slashes + path = strings.Replace(path, "//", "/", 1) + + return path +} + +// Walk walks the file tree rooted at root, calling walkFn for each file or directory in the tree, including root. +func Walk(root string, walkFn filepath.WalkFunc) error { + return filepath.Walk(root, walkFn) +} + +// FileName from path +func FileName(path string) string { + return filepath.Base(path) +} + +// FilenameWithoutExtension from path +func FilenameWithoutExtension(path string) string { + return strings.TrimSuffix(filepath.Base(path), filepath.Ext(path)) +} diff --git a/commons/file_test.go b/commons/file_test.go new file mode 100644 index 0000000..2b99596 --- /dev/null +++ b/commons/file_test.go @@ -0,0 +1,46 @@ +package commons + +import ( + "os" + "testing" +) + +func TestPathPrepare(t *testing.T) { + tests := []struct { + name string + path string + expected string + }{ + { + name: "Replace $HOME", + path: "$HOME/test", + expected: os.Getenv("HOME") + "/test", + }, + { + name: "Replace $TMP", + path: "$TMP/test", + expected: os.Getenv("TMP") + "/test", + }, + { + name: "Replace $CWD", + path: "$CWD/test", + expected: os.Getenv("PWD") + "/test", + }, + { + name: "No replacements", + path: "/test/path", + expected: "/test/path", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + result := PathPrepare(test.path) + if result != test.expected { + t.Errorf("Expected %s, but got %s", test.expected, result) + } else { + t.Logf("Passed Test %s, Input %s, Expected %s, Result %s", test.name, test.path, test.expected, result) + } + }) + } +} diff --git a/commons/fileutils.go b/commons/fileutils.go deleted file mode 100644 index 523b32e..0000000 --- a/commons/fileutils.go +++ /dev/null @@ -1,36 +0,0 @@ -package commons - -import ( - "os" - "path/filepath" -) - -// EnsureDirExists checks if a directory exists at the given path, and creates it if it doesn't exist. -func EnsureDirExists(dirPath string) error { - if _, err := os.Stat(dirPath); os.IsNotExist(err) { - err := os.MkdirAll(dirPath, 0755) - if err != nil { - return err - } - } - return nil -} - -// CreateFile creates a file at the given path and writes the contents to the file. -// It overwrites the file if it already exists. -func CreateFile(filePath string, contents []byte) error { - // Ensure the directory exists - dir := filepath.Dir(filePath) - err := EnsureDirExists(dir) - if err != nil { - return err - } - - // Write contents to the file - err = os.WriteFile(filePath, contents, 0644) - if err != nil { - return err - } - - return nil -} diff --git a/commons/git.go b/commons/git.go new file mode 100644 index 0000000..dc433bb --- /dev/null +++ b/commons/git.go @@ -0,0 +1,88 @@ +package commons + +import ( + "os" + + "gopkg.in/src-d/go-git.v4" + "gopkg.in/src-d/go-git.v4/plumbing" +) + +func GitCloneRepository(url, directory string) error { + _, err := git.PlainClone(directory, false, &git.CloneOptions{ + URL: url, + Progress: os.Stdout, + }) + return err +} + +func GitIsValidGitRepository(directory string) bool { + _, err := git.PlainOpen(directory) + return err == nil +} + +func GitPullRepository(directory string) error { + // Open an existing repository + r, err := git.PlainOpen(directory) + if err != nil { + return err + } + + // Get the working directory for the repository + w, err := r.Worktree() + if err != nil { + return err + } + + // Pull the latest changes from the origin remote and merge into the current branch + err = w.Pull(&git.PullOptions{RemoteName: "origin"}) + if err != nil { + return err + } + + return nil +} + +// func GitCheckoutBranch to checkout branch form name +func GitCheckoutBranch(directory, name string) error { + // Open an existing repository + r, err := git.PlainOpen(directory) + if err != nil { + return err + } + + // Get the working directory for the repository + w, err := r.Worktree() + if err != nil { + return err + } + + // Get the branch reference + branch := plumbing.NewRemoteReferenceName("origin", name) + + // Pull the latest changes from the origin remote and merge into the current branch + err = w.Checkout(&git.CheckoutOptions{Branch: branch}) + if err != nil { + return err + } + + return nil +} + +func GitResetUncommittedChanges(directory string) error { + r, err := git.PlainOpen(directory) + if err != nil { + return err + } + + w, err := r.Worktree() + if err != nil { + return err + } + + err = w.Reset(&git.ResetOptions{Mode: git.HardReset}) + if err != nil { + return err + } + + return nil +} diff --git a/commons/hash.go b/commons/hash.go new file mode 100644 index 0000000..e509d64 --- /dev/null +++ b/commons/hash.go @@ -0,0 +1,13 @@ +package commons + +import ( + "crypto/md5" + "encoding/hex" + "strings" +) + +func Hash(args ...string) string { + hasher := md5.New() + hasher.Write([]byte(strings.Join(args, ""))) + return hex.EncodeToString(hasher.Sum(nil)) +} diff --git a/commons/json.go b/commons/json.go new file mode 100644 index 0000000..c6a5006 --- /dev/null +++ b/commons/json.go @@ -0,0 +1,16 @@ +package commons + +import ( + "encoding/json" + "fmt" +) + +// func to PrintJson is a helper function to print a JSON object +// in a pretty format. +func PrintJson(obj any) { + json, err := json.Marshal(obj) + if err != nil { + panic(err) + } + fmt.Println(string(json)) +} diff --git a/commons/maps.go b/commons/maps.go new file mode 100644 index 0000000..260a8d1 --- /dev/null +++ b/commons/maps.go @@ -0,0 +1,13 @@ +package commons + +// func GetMapKeys is to get keys of map storing any type of values +// Example: GetMapKeys(map[string]int{"a": 1, "b": 2}) -> []string{"a", "b"} +func GetMapKeys[T any](m map[string]T) []string { + keys := make([]string, len(m)) + i := 0 + for k := range m { + keys[i] = k + i++ + } + return keys +} diff --git a/config/config.go b/config/config.go index 450d085..164c7f8 100644 --- a/config/config.go +++ b/config/config.go @@ -43,3 +43,7 @@ func WriteConfig() { println("Config saved.") } + +func LoadConfig() { + +} diff --git a/doc-generator/doc-generator.go b/doc-generator/doc-generator.go index 080916f..44df7a4 100644 --- a/doc-generator/doc-generator.go +++ b/doc-generator/doc-generator.go @@ -13,7 +13,7 @@ func main() { // Declare variables var err error - var docsPath = "./docs" + var docsPath = "./docs/" // Remove docsPath if it exists err = os.RemoveAll(docsPath) diff --git a/docs/ocsf-tool.md b/docs/ocsf-tool.md index fa5c270..c3c7ddd 100644 --- a/docs/ocsf-tool.md +++ b/docs/ocsf-tool.md @@ -14,4 +14,4 @@ * [ocsf-tool generate-proto](ocsf-tool_generate-proto.md) - Generate a Proto file * [ocsf-tool schema-class-list](ocsf-tool_schema-class-list.md) - List all classes in the OCSF schema -###### Auto generated by spf13/cobra on 24-Nov-2023 +###### Auto generated by spf13/cobra on 17-Dec-2023 diff --git a/docs/ocsf-tool_config.md b/docs/ocsf-tool_config.md index e07b6d1..abcf3bd 100644 --- a/docs/ocsf-tool_config.md +++ b/docs/ocsf-tool_config.md @@ -20,6 +20,7 @@ ocsf-tool config values... [flags] ocsf-tool config extensions linux win ocsf-tool config profiles cloud linux/linux_users + ocfs-tool config schema.loading.strategy repository ``` @@ -33,4 +34,4 @@ ocsf-tool config values... [flags] * [ocsf-tool](ocsf-tool.md) - -###### Auto generated by spf13/cobra on 24-Nov-2023 +###### Auto generated by spf13/cobra on 17-Dec-2023 diff --git a/docs/ocsf-tool_generate-proto.md b/docs/ocsf-tool_generate-proto.md index 4336717..7721a70 100644 --- a/docs/ocsf-tool_generate-proto.md +++ b/docs/ocsf-tool_generate-proto.md @@ -31,4 +31,4 @@ ocsf-tool generate-proto file_activity process_activity * [ocsf-tool](ocsf-tool.md) - -###### Auto generated by spf13/cobra on 24-Nov-2023 +###### Auto generated by spf13/cobra on 17-Dec-2023 diff --git a/docs/ocsf-tool_schema-class-list.md b/docs/ocsf-tool_schema-class-list.md index 80aec15..c361e7e 100644 --- a/docs/ocsf-tool_schema-class-list.md +++ b/docs/ocsf-tool_schema-class-list.md @@ -16,4 +16,4 @@ ocsf-tool schema-class-list [flags] * [ocsf-tool](ocsf-tool.md) - -###### Auto generated by spf13/cobra on 24-Nov-2023 +###### Auto generated by spf13/cobra on 17-Dec-2023 diff --git a/go.mod b/go.mod index 3813016..fd2d93c 100644 --- a/go.mod +++ b/go.mod @@ -11,28 +11,41 @@ require ( require ( github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect + github.com/emirpasic/gods v1.12.0 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/hashicorp/errwrap v1.0.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect + github.com/jinzhu/copier v0.4.0 // indirect + github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd // indirect github.com/kr/text v0.2.0 // indirect github.com/magiconair/properties v1.8.7 // indirect + github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/pelletier/go-toml/v2 v2.1.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/sagikazarmark/locafero v0.3.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect + github.com/sergi/go-diff v1.0.0 // indirect github.com/sourcegraph/conc v0.3.0 // indirect github.com/spf13/cast v1.5.1 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/spf13/viper v1.17.0 // indirect + github.com/src-d/gcfg v1.4.0 // indirect github.com/subosito/gotenv v1.6.0 // indirect + github.com/xanzy/ssh-agent v0.2.1 // indirect go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.9.0 // indirect + golang.org/x/crypto v0.13.0 // indirect + golang.org/x/net v0.15.0 // indirect golang.org/x/sys v0.13.0 // indirect golang.org/x/text v0.13.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/ini.v1 v1.67.0 // indirect + gopkg.in/src-d/go-billy.v4 v4.3.2 // indirect + gopkg.in/src-d/go-git.v4 v4.13.1 // indirect + gopkg.in/warnings.v0 v0.1.2 // indirect ) require ( diff --git a/go.sum b/go.sum index fff69e7..2a7b132 100644 --- a/go.sum +++ b/go.sum @@ -38,6 +38,9 @@ cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3f dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs= +github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= @@ -48,17 +51,22 @@ github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnht github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cpuguy83/go-md2man/v2 v2.0.3 h1:qMCsGGgs+MAzDFyp9LpAe1Lqy/fY/qCovCm0qnXZOBM= github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg= +github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= +github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -130,23 +138,35 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= +github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/jinzhu/copier v0.4.0 h1:w3ciUoD19shMCRargcpm0cm91ytaBhDvuRpz1ODO/U8= +github.com/jinzhu/copier v0.4.0/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd h1:Coekwdh0v2wtGp9Gmz1Ze3eVRAWJMLokvN3QjdzCHLY= +github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/pelletier/go-buffruneio v0.2.0/go.mod h1:JkE26KsDizTr40EUHkXVtNPvgGtbSNq5BcowyYOWdKo= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -158,6 +178,8 @@ github.com/sagikazarmark/locafero v0.3.0 h1:zT7VEGWC2DTflmccN/5T1etyKvxSxpHsjb9c github.com/sagikazarmark/locafero v0.3.0/go.mod h1:w+v7UsPNFwzF1cHuOajOOzoq4U7v/ig1mpRjqV+Bu1U= github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= +github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ= +github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= github.com/spf13/afero v1.10.0 h1:EaGW2JJh15aKOejeuJ+wpFSHnbd7GE6Wvp3TsNhb6LY= @@ -170,7 +192,10 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.17.0 h1:I5txKw7MJasPL/BrfkbA0Jyo/oELqVmux4pR/UxOMfI= github.com/spf13/viper v1.17.0/go.mod h1:BmMMMLQXSbcHK6KAOiFLz0l5JHrU89OdIRHvsk0+yVI= +github.com/src-d/gcfg v1.4.0 h1:xXbNR5AlLSA315x2UO+fTSSAXCDf+Ar38/6oyGbDKQ4= +github.com/src-d/gcfg v1.4.0/go.mod h1:p/UMsR43ujA89BJY9duynAwIpvqEujIH/jFlfL7jWoI= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -182,6 +207,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= +github.com/xanzy/ssh-agent v0.2.1 h1:TCbipTQL2JiiCprBWx9frJ2eJlCYT00NmctrHxVAr70= +github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -196,13 +223,17 @@ go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= +golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -270,6 +301,8 @@ golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -291,6 +324,7 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190221075227-b4e8571b14e0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -355,6 +389,7 @@ golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBn golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190729092621-ff9f1409240a/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -489,6 +524,13 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EV gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/src-d/go-billy.v4 v4.3.2 h1:0SQA1pRztfTFx2miS8sA97XvooFeNOmvUenF4o0EcVg= +gopkg.in/src-d/go-billy.v4 v4.3.2/go.mod h1:nDjArDMp+XMs1aFAESLRjfGSgfvoYN0hDfzEk0GjC98= +gopkg.in/src-d/go-git-fixtures.v3 v3.5.0/go.mod h1:dLBcvytrw/TYZsNTWCnkNF2DSIlzWYqTe3rJR56Ac7g= +gopkg.in/src-d/go-git.v4 v4.13.1 h1:SRtFyV8Kxc0UP7aCHcijOMQGPxHSmMOPrzulQWolkYE= +gopkg.in/src-d/go-git.v4 v4.13.1/go.mod h1:nx5NYcxdKxq5fpltdHnPa2Exj4Sx0EclMWZQbYDu2z8= +gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= +gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/ocsf/mappers/protobuff_v3/mapper.go b/ocsf/mappers/protobuff_v3/mapper.go index 404d1f1..2d17a56 100644 --- a/ocsf/mappers/protobuff_v3/mapper.go +++ b/ocsf/mappers/protobuff_v3/mapper.go @@ -12,7 +12,7 @@ import ( var _mapper *mapper -func NewMapper(schema schema.OCSFSchema) *mapper { +func NewMapper(schema *schema.OCSFSchema) *mapper { _mapper = &mapper{ Schema: schema, Preprocessor: Preprocessor{ @@ -59,7 +59,6 @@ func (mapper *mapper) Marshal(events []schema.Event) { mapper.populateFieldsFromAttributes(&m, event.Attributes) AddMessage(&m) - } mapper.RootPackage.Marshal() @@ -103,7 +102,6 @@ func (mapper *mapper) populateFieldsFromAttributes(message *Message, attributes field.DataType = attr.ObjectType attributeIsSelfReferencing := field.DataType == message.Name _, isObjectMapped := GetMessage(field.DataType) - if !isObjectMapped && !attributeIsSelfReferencing { object, schemaForObjectExists := mapper.getObject(field.DataType) if schemaForObjectExists { @@ -112,9 +110,10 @@ func (mapper *mapper) populateFieldsFromAttributes(message *Message, attributes GroupKey: "Object", Package: mapper.PackageRef("objects"), } - mapper.populateFieldsFromAttributes(m, object.Attributes) AddMessage(m) + + mapper.populateFieldsFromAttributes(m, object.Attributes) } } diff --git a/ocsf/mappers/protobuff_v3/types.go b/ocsf/mappers/protobuff_v3/types.go index a19147a..f3e47c1 100644 --- a/ocsf/mappers/protobuff_v3/types.go +++ b/ocsf/mappers/protobuff_v3/types.go @@ -90,7 +90,7 @@ type CacheMap struct { } type mapper struct { - Schema schema.OCSFSchema + Schema *schema.OCSFSchema Preprocessor Preprocessor Messages Messages Enums Enums diff --git a/ocsf/schema/OCSFSchema.go b/ocsf/schema/OCSFSchema.go deleted file mode 100644 index 72583fa..0000000 --- a/ocsf/schema/OCSFSchema.go +++ /dev/null @@ -1,141 +0,0 @@ -package schema - -import ( - "crypto/md5" - "encoding/hex" - "encoding/json" - "fmt" - "io" - "log" - "net/http" - "os" - "strings" - - "github.com/spf13/viper" -) - -const ( - apiURL = "https://schema.ocsf.io/export/schema" -) - -var schemaJSON string - -func LoadOCSFSchema() (OCSFSchema, error) { - // Get the extensions and profiles using viper - extensions := viper.GetStringSlice("extensions") - profiles := viper.GetStringSlice("profiles") - - // if extensions are empty then print message - if len(extensions) == 0 { - fmt.Println("No extensions specified. Using All by default.") - } - println("Extensions: ", strings.Join(extensions, ",")) - - // if profiles are empty then print message - if len(profiles) == 0 { - fmt.Println("No profiles specified. Using All by default.") - } - - println("Profiles: ", strings.Join(profiles, ",")) - - schemaJSON = getSchemaJsonFileName(extensions, profiles) - - // Download the schema and save it to disk - err := downloadSchemaAndSave(extensions, profiles) - if err != nil { - fmt.Println("Error downloading schema:", err) - os.Exit(1) - } - - // Read the JSON file - data, err := os.ReadFile(schemaJSON) - if err != nil { - log.Fatalf("Error reading JSON file: %v", err) - } - - // Define a variable of the struct type - var schema OCSFSchema - - // Unmarshal the JSON data into the struct - if err := json.Unmarshal(data, &schema); err != nil { - log.Fatalf("Error unmarshalling JSON data: %v", err) - } - - return schema, err -} - -// downloadSchemaAndSave downloads the schema JSON from the API and saves it to a file. -func downloadSchemaAndSave(extensions []string, profiles []string) error { - - // Return if the schema JSON file already exists - if _, err := os.Stat(schemaJSON); err == nil { - fmt.Printf("Using schema file: %s\n", schemaJSON) - return nil - } - - // Build the query string for extensions and profiles - queryParams := []string{} - if len(extensions) > 0 { - queryParams = append(queryParams, "extensions="+strings.Join(extensions, ",")) - } - if len(profiles) > 0 { - queryParams = append(queryParams, "profiles="+strings.Join(profiles, ",")) - } - queryString := "" - if len(queryParams) > 0 { - queryString = "?" + strings.Join(queryParams, "&") - } - - // Construct the full API URL with query parameters - fullURL := apiURL + queryString - - // Send a GET request to the API endpoint - fmt.Println("Sending GET request to API endpoint...") - fmt.Printf("API URL: %s\n", fullURL) - resp, err := http.Get(fullURL) - if err != nil { - fmt.Println("Error sending GET request:", err) - return err - } - defer resp.Body.Close() - - // Check if the response status code is 200 (OK) - if resp.StatusCode != http.StatusOK { - fmt.Printf("API returned status code: %d\n", resp.StatusCode) - return fmt.Errorf("API returned status code: %d", resp.StatusCode) - } - - // Read the response body - fmt.Println("Reading response body...") - body, err := io.ReadAll(resp.Body) - if err != nil { - fmt.Println("Error reading response body:", err) - return err - } - - // Save the response body as a JSON file - fmt.Printf("Saving schema to file: %s\n", schemaJSON) - err = os.WriteFile(schemaJSON, body, 0644) - if err != nil { - fmt.Println("Error writing JSON file:", err) - return err - } - - fmt.Printf("Schema saved to %s\n", schemaJSON) - return nil -} - -func getSchemaJsonFileName(extensions []string, profiles []string) string { - hash := getSchemaHash(extensions, profiles) - return fmt.Sprintf("ocsf-schema-%s.json", hash) -} - -func getSchemaHash(extensions []string, profiles []string) string { - extensionsStr := strings.Join(extensions, "") - profilesStr := strings.Join(profiles, "") - - hasher := md5.New() - hasher.Write([]byte(extensionsStr + profilesStr)) - - return hex.EncodeToString(hasher.Sum(nil)) -} diff --git a/ocsf/schema/loader-repository.go b/ocsf/schema/loader-repository.go new file mode 100644 index 0000000..13e0874 --- /dev/null +++ b/ocsf/schema/loader-repository.go @@ -0,0 +1,836 @@ +package schema + +import ( + "encoding/json" + "errors" + "os" + "strings" + + "github.com/jinzhu/copier" + "github.com/spf13/viper" + "github.com/valllabh/ocsf-tool/commons" + "gopkg.in/src-d/go-git.v4" +) + +func init() { + RegisterSchemaLoader("repository", &SchemaRepositorySchemaLoader{}) +} + +func (sl *SchemaRepositorySchemaLoader) Init() { + +} + +func (sl *SchemaRepositorySchemaLoader) Config() { + + // Schema Repository Strategy Default Options + + // config for git repository url + viper.SetDefault("schema.loading.strategies.repository.url", "https://github.com/ocsf/ocsf-schema") + + // config for directory to clone git repository + viper.SetDefault("schema.loading.strategies.repository.directory.path", "$CWD/schema/git") + + // config for branch to checkout + viper.SetDefault("schema.loading.strategies.repository.branch.name", "main") +} + +// Downloads ocsf schema from and write schema to file +func (sl *SchemaRepositorySchemaLoader) Load() (*OCSFSchema, error) { + + // Load common options + LoadCommonOptions(sl) + + // Download git repository + downloadRepoError := sl.downloadRepo() + + // Check for error while downloading git repository + if downloadRepoError != nil { + return nil, downloadRepoError + } + + // Process git repository + println("Processing git repository") + return sl.processRepo() + +} + +// SetExtensions sets extensions +func (sl *SchemaRepositorySchemaLoader) SetExtensions(extensions []string) { + sl.extensions = extensions +} + +// GetExtensions returns extensions +func (sl *SchemaRepositorySchemaLoader) GetExtensions() []string { + return sl.extensions +} + +// SetProfiles sets profiles +func (sl *SchemaRepositorySchemaLoader) SetProfiles(profiles []string) { + sl.profiles = profiles +} + +// GetProfiles returns profiles +func (sl *SchemaRepositorySchemaLoader) GetProfiles() []string { + return sl.profiles +} + +// ProfileExists returns true if profile exists +func (sl *SchemaRepositorySchemaLoader) ProfileExists(profile string) bool { + + // return true if no profile is configured + if len(sl.GetProfiles()) == 0 { + return true + } + + return commons.Contains(sl.GetProfiles(), profile) +} + +// ExtensionExists returns true if extension exists +func (sl *SchemaRepositorySchemaLoader) ExtensionExists(extension string) bool { + + // return true if no extension is configured + if len(sl.GetExtensions()) == 0 { + return true + } + + return commons.Contains(sl.GetExtensions(), extension) +} + +func repoPath(path string) string { + + directory := viper.GetString("schema.loading.strategies.repository.directory.path") + + directory = commons.PathPrepare(directory) + + return commons.CleanPath(directory + "/" + path) +} + +// function to load repository path from config +func (sl *SchemaRepositorySchemaLoader) loadRepoPath() string { + // Load directory from config + directory := viper.GetString("schema.loading.strategies.repository.directory.path") + + // Prepare Directory Path + directory = commons.PathPrepare(directory) + + return directory +} + +// function to download git repository for given URL and save it to disk +func (sl *SchemaRepositorySchemaLoader) downloadRepo() error { + + // Load directory from config + directory := sl.loadRepoPath() + + // Load URL from config + url := viper.GetString("schema.loading.strategies.repository.url") + + // Check if directory exists using commons + if commons.PathExists(directory) { + // check if directory is a git repository + if commons.GitIsValidGitRepository(directory) { + + // reset uncommitted changes + println("Resetting uncommitted changes") + errGitResetUncommittedChanges := commons.GitResetUncommittedChanges(directory) + if errGitResetUncommittedChanges != nil { + return errGitResetUncommittedChanges + } + + // checkout branch + branch := viper.GetString("schema.loading.strategies.repository.branch.name") + println("Checking out branch: " + branch) + errGitCheckoutBranch := commons.GitCheckoutBranch(directory, branch) + + if errGitCheckoutBranch != nil { + return errGitCheckoutBranch + } + + // pull latest changes + println("Pulling latest changes") + errGitPullRepository := commons.GitPullRepository(directory) + + // if schema is already up to date + if errGitPullRepository == git.NoErrAlreadyUpToDate { + println("Schema is already up to date") + return nil + } + + } + } else { + + // Clone git repository + println("Cloning git repository from " + url + " to " + directory) + errGitCloneRepository := commons.GitCloneRepository(url, directory) + if errGitCloneRepository != nil { + return errGitCloneRepository + } + + } + + return nil +} + +// func to process repository +func (sl *SchemaRepositorySchemaLoader) processRepo() (*OCSFSchema, error) { + + // Load directory from config + directory := sl.loadRepoPath() + + // Load schema from directory + println("Loading schema from " + directory) + schema, loadSchemaError := sl.loadSchemaFromDirectory(directory) + + // Check for error while loading schema + if loadSchemaError != nil { + return nil, loadSchemaError + } + + // Return schema + return schema, nil +} + +// function to load schema from directory +func (sl *SchemaRepositorySchemaLoader) loadSchemaFromDirectory(directory string) (*OCSFSchema, error) { + + // Declare schema + var schema OCSFSchema + + // Load version from directory + schemaVersion, schemaVersionError := sl.loadVersionFromDirectory() + + // Check for error while loading version from directory + if schemaVersionError != nil { + return nil, schemaVersionError + } + + dictionary := Dictionary{} + objectMap := make(map[string]Object) + eventMap := make(map[string]Event) + + // Load dictionary from dictionary.json file + dictionaryFile := repoPath("/dictionary.json") + dictionaryLoadingError := sl.loadDictionary(dictionaryFile, &dictionary) + + if dictionaryLoadingError != nil { + return nil, dictionaryLoadingError + } + + // Load objects defined in json files in the map from /objects directory using schema.Object struct + objectsDirectory := repoPath("/objects") + objectLoadingError := sl.loadObjects(objectsDirectory, &objectMap, &dictionary) + + if objectLoadingError != nil { + return nil, objectLoadingError + } + + // extend object attributes + for _, object := range objectMap { + sl.extendAttribute(&object.Attributes, &dictionary, &objectMap) + } + + // Load events defined in json files in the map from /events directory using schema.Event struct + eventsDirectory := repoPath("/events") + + eventLoadingError := sl.loadEvents(eventsDirectory, &eventMap, &dictionary) + + if eventLoadingError != nil { + return nil, eventLoadingError + } + + // extend event attributes + for _, event := range eventMap { + sl.extendAttribute(&event.Attributes, &dictionary, &objectMap) + } + + // Build schema + schema = OCSFSchema{ + Objects: objectMap, + Classes: eventMap, + Version: schemaVersion.Version, + Types: dictionary.Types.Attributes, + Dictionary: dictionary, + } + + // Load extensions defined in json files in the map from /extensions directory using schema.Extension struct + extensionsDirectory := repoPath("/extensions") + extensionsLoadingError := sl.loadExtensions(extensionsDirectory, &schema) + + if extensionsLoadingError != nil { + return nil, extensionsLoadingError + } + + return &schema, nil +} + +// GetSchemaHash returns the hash of the schema +func (sl *SchemaRepositorySchemaLoader) GetSchemaHash() string { + return commons.Hash( + strings.Join(sl.GetExtensions(), " "), + strings.Join(sl.GetProfiles(), " "), + ) +} + +// function to load extensions from directory +func (sl *SchemaRepositorySchemaLoader) loadExtensions(directory string, ocsfSchema *OCSFSchema) error { + + // recursively load each file in extensions directory + err := commons.Walk(directory, func(path string, info os.FileInfo, err error) error { + + // Check if file is a json file + if strings.HasSuffix(path, "extension.json") { + + // Load extension from file + err := sl.loadExtensionFromDirectory(commons.Dir(path), ocsfSchema) + + // Check for error while loading extension from file + if err != nil { + return err + } + + } + + return nil + }) + + return err +} + +func (sl *SchemaRepositorySchemaLoader) loadExtension(path string) (Extension, error) { + + // Declare extension + var extension Extension + + // load data from file []byte + data, loadDataError := os.ReadFile(path) + + if loadDataError != nil { + return extension, loadDataError + } + + // unmarshal data into extension + if err := json.Unmarshal(data, &extension); err != nil { + return extension, err + } + + return extension, nil +} + +// function to load extension from file +func (sl *SchemaRepositorySchemaLoader) loadExtensionFromDirectory(path string, ocsfSchema *OCSFSchema) error { + + println("Loading extension from " + path) + + extensionFile := path + "/" + "extension.json" + extension, extensionLoadingError := sl.loadExtension(extensionFile) + + if extensionLoadingError != nil { + return extensionLoadingError + } + + // check if extension is configured to be loaded else ignore + if !sl.ExtensionExists(extension.Name) { + println("Ignoring extension " + extension.Caption + " as it is not configured to be loaded.") + return nil + } + + // Load dictionary from dictionary.json file + dictionaryFile := commons.CleanPath(path + "/" + "dictionary.json") + dictionaryLoadingError := sl.loadDictionary(dictionaryFile, &ocsfSchema.Dictionary) + + if dictionaryLoadingError != nil { + return dictionaryLoadingError + } + + // Load objects defined in json files in the map from /objects directory using schema.Object struct + objectsDirectory := commons.CleanPath(path + "/" + "objects") + objectLoadingError := sl.loadObjects(objectsDirectory, &ocsfSchema.Objects, &ocsfSchema.Dictionary) + + if objectLoadingError != nil { + return objectLoadingError + } + + // extend object attributes + for _, object := range ocsfSchema.Objects { + sl.extendAttribute(&object.Attributes, &ocsfSchema.Dictionary, &ocsfSchema.Objects) + } + + // Load events defined in json files in the map from /events directory using schema.Event struct + eventsDirectory := commons.CleanPath(path + "/" + "events") + eventLoadingError := sl.loadEvents(eventsDirectory, &ocsfSchema.Classes, &ocsfSchema.Dictionary) + + if eventLoadingError != nil { + return eventLoadingError + } + + return nil +} + +// func to load version from directory returns version and error. version.json from repo directory is loaded and version is returned +func (sl *SchemaRepositorySchemaLoader) loadVersionFromDirectory() (Version, error) { + + // Declare version + var version Version + + // Version file path + versionFile := repoPath("/version.json") + + // Load data from file []byte + data, loadDataError := os.ReadFile(versionFile) + + if loadDataError != nil { + return version, loadDataError + } + + // Unmarshal data into version + if err := json.Unmarshal(data, &version); err != nil { + return version, err + } + + return version, nil +} + +// function to load objects from directory +func (sl *SchemaRepositorySchemaLoader) loadObjects(directory string, objects *(map[string]Object), dictionary *Dictionary) error { + + rootDir := commons.Dir(directory) + + // recursively load each file in objects directory + err := commons.Walk(directory, func(path string, info os.FileInfo, err error) error { + + // Check if file is a json file + if strings.HasSuffix(path, ".json") { + + println("Loading object from " + path) + + // Load object from file + object, err := sl.loadObjectFromFile(path, rootDir) + + // Check for error while loading object from file + if err != nil { + return err + } + + // if object.name is blank and object.extend exists and objects contain object.extend then merge object.profiles and merge object.attributes + if object.Name == "" && object.Extends != "" { + + println("Extending object " + object.Extends) + + originalObject, objectExists := (*objects)[object.Extends] + + if objectExists { + // if originalObject.Profiles does not exist then create it + if originalObject.Profiles == nil { + originalObject.Profiles = make([]string, 0) + } + + originalObject.Profiles = append(originalObject.Profiles, object.Profiles...) + + // iterate over attributes and add them to originalObject + for key, value := range object.Attributes { + originalObject.Attributes[key] = value + } + + (*objects)[object.Extends] = originalObject + } else { + println("Object " + object.Extends + " does not exist") + } + } else { + + // Add object to schema + (*objects)[object.Name] = object + + } + } + + return nil + }) + + // iterate over objects and resolve extends + for _, object := range *objects { + + // extend object + sl.extendObject(&object, objects) + + } + + return err +} + +// function to load events from directory +func (sl *SchemaRepositorySchemaLoader) loadEvents(directory string, events *(map[string]Event), dictionary *Dictionary) error { + + rootDir := commons.Dir(directory) + + // recursively load each file in events directory + err := commons.Walk(directory, func(path string, info os.FileInfo, err error) error { + + // Check if file is a json file + if strings.HasSuffix(path, ".json") { + + println("Loading event from " + path) + + // Load event from file + event, err := sl.loadEventFromFile(path, rootDir) + + // Check for error while loading event from file + if err != nil { + return err + } + + // Add event to schema + (*events)[event.Name] = event + } + + return nil + }) + + // iterate over events and resolve extends + for _, event := range *events { + + // extend event + sl.extendEvent(&event, events) + + } + + return err +} + +// function to load Dictionary from dictionary.json file +func (sl *SchemaRepositorySchemaLoader) loadDictionary(path string, dictionary *Dictionary) error { + + // Load data from file []byte + data, loadDataError := os.ReadFile(path) + + if loadDataError != nil { + return loadDataError + } + + _dictionary := Dictionary{} + + // Unmarshal data into dictionary + if err := json.Unmarshal(data, &_dictionary); err != nil { + return err + } + + // set map if nil + if (*dictionary).Attributes == nil { + (*dictionary).Attributes = make(map[string]Attribute) + } + + // merge _dictionary attributes into dictionary attributes + for key, value := range _dictionary.Attributes { + (*dictionary).Attributes[key] = value + } + + return nil +} + +// function to get all parent attributes recursively of Object using Extends from given map of items +func (sl *SchemaRepositorySchemaLoader) extendObject(item *Object, items *map[string]Object) { + + // check if item has parent + if item.Extends != "" { + // get parent item + parentItem := (*items)[item.Extends] + + // iterate over parent attributes and add them to attributes + for key, value := range parentItem.Attributes { + item.Attributes[key] = value + } + + // extend parent + sl.extendObject(&parentItem, items) + + (*items)[item.Name] = *item + } + +} + +// function to get all parent attributes recursively of Event using Extends from given map of items +func (sl *SchemaRepositorySchemaLoader) extendEvent(item *Event, items *map[string]Event) { + + // check if item has parent + if item.Extends != "" { + // get parent item + parentItem := (*items)[item.Extends] + + // set attributes + for key, value := range parentItem.Attributes { + item.Attributes[key] = value + } + + // if item category is not null then use parent + if item.Category == "" { + item.Category = parentItem.Category + } + + // get parent attributes + sl.extendEvent(&parentItem, items) + + (*items)[item.Name] = *item + } + +} + +// function to extend attribute from dictionary +func (sl *SchemaRepositorySchemaLoader) extendAttribute(attributes *(map[string]Attribute), dictionary *Dictionary, objects *(map[string]Object)) { + + // iterate over attributes + for key, attribute := range *attributes { + dictionaryAttribute, dictionaryAttributeExists := dictionary.Attributes[key] + // if attribute exists in dictionary then copy it to attribute + if dictionaryAttributeExists { + + copyError := copier.CopyWithOption(&attribute, &dictionaryAttribute, copier.Option{IgnoreEmpty: true, DeepCopy: true}) + + // if attribute.Type does not end with _t + if !strings.HasSuffix(attribute.Type, "_t") { + // check if attribute.Type exists in objects + object, objectExists := (*objects)[attribute.Type] + if objectExists { + attribute.Type = "object_t" + attribute.ObjectType = object.Name + attribute.ObjectName = object.Caption + } + } + + (*attributes)[key] = attribute + + if copyError != nil { + println("Error while extending attribute " + key) + println(copyError.Error()) + } + } + } +} + +// function to load object from file +func (sl *SchemaRepositorySchemaLoader) loadObjectFromFile(path string, includeRootPath string) (Object, error) { + // Declare object + var object Object + + // load data from file []byte + data, loadDataError := os.ReadFile(path) + + if loadDataError != nil { + return object, loadDataError + } + + type Alias RepositoryObject + o := RepositoryObject{} + aux := &struct { + *Alias + }{ + Alias: (*Alias)(&o), + } + if err := json.Unmarshal(data, &aux); err != nil { + return object, err + } + + for key, value := range o.Attributes { + if key == "$include" { + includes := value.([]interface{}) + for _, include := range includes { + + // parse include file path + includeFilePath := commons.CleanPath(includeRootPath + "/" + include.(string)) + + // include file path + attributes, includeError := sl.includeFile(includeFilePath) + + if includeError != nil { + return object, includeError + } + + if len(attributes) != 0 { + // append attributes to event + for key, value := range attributes { + o.Attributes[key] = value + } + } + + } + delete(o.Attributes, "$include") + } else { + attr := Attribute{} + jsonAttr, _ := json.Marshal(value) + json.Unmarshal(jsonAttr, &attr) + o.Attributes[key] = attr + } + } + + // copy each field from RepositoryObject to Object + object.Caption = o.Caption + object.Constraints = o.Constraints + object.Description = o.Description + object.Extends = o.Extends + object.Name = o.Name + + // iterate over attributes and copy each attribute to object + object.Attributes = make(map[string]Attribute) + for key, value := range o.Attributes { + object.Attributes[key] = value.(Attribute) + } + + return object, nil +} + +// function to load event from file +func (sl *SchemaRepositorySchemaLoader) loadEventFromFile(path string, includeRootPath string) (Event, error) { + // Declare event + var event Event + + // load data from file []byte + data, loadDataError := os.ReadFile(path) + + if loadDataError != nil { + return event, loadDataError + } + + type Alias RepositoryEvent + e := RepositoryEvent{} + aux := &struct { + *Alias + }{ + Alias: (*Alias)(&e), + } + if err := json.Unmarshal(data, &aux); err != nil { + return event, err + } + + for key, value := range e.Attributes { + if key == "$include" { + includes := value.([]interface{}) + for _, include := range includes { + + // parse include file path + includeFilePath := commons.CleanPath(includeRootPath + "/" + include.(string)) + + // include file path + attributes, includeError := sl.includeFile(includeFilePath) + + if includeError != nil { + return event, includeError + } + + if len(attributes) != 0 { + // append attributes to event + for key, value := range attributes { + e.Attributes[key] = value + } + } + + } + delete(e.Attributes, "$include") + } else { + attr := Attribute{} + jsonAttr, _ := json.Marshal(value) + json.Unmarshal(jsonAttr, &attr) + e.Attributes[key] = attr + } + } + + // copy each field from RepositoryEvent to Event + event.Caption = e.Caption + event.Description = e.Description + event.Extends = e.Extends + event.Name = e.Name + event.Category = e.Category + event.CategoryName = e.CategoryName + event.Profiles = e.Profiles + event.Uid = e.Uid + + // iterate over attributes and copy each attribute to object + event.Attributes = make(map[string]Attribute) + for key, value := range e.Attributes { + event.Attributes[key] = value.(Attribute) + } + + return event, nil + +} + +// func detectAndLoadInclude which accepts include file path and loads it using diffrent function based on path. Possible paths are /includes, /profiles +func (sl *SchemaRepositorySchemaLoader) includeFile(path string) (map[string]Attribute, error) { + + // switch based on path prefix + switch { + case strings.Contains(path, "includes/"): + data, err := sl.loadIncludeFromFile(path) + return data.Attributes, err + case strings.Contains(path, "profiles/"): + data, err := sl.loadProfileFromFile(path) + return data.Attributes, err + default: + return nil, errors.New("Invalid include path " + path) + } + +} + +// function to load include from file returns Include struct and error. It accepts include file path as string and loads json file from repo Directory and returns Include struct and error +func (sl *SchemaRepositorySchemaLoader) loadIncludeFromFile(path string) (Include, error) { + + // declare include + var include Include + + // load data from file []byte + data, loadDataError := os.ReadFile(path) + + if loadDataError != nil { + return include, loadDataError + } + + // unmarshal data into include + if err := json.Unmarshal(data, &include); err != nil { + return include, err + } + + return include, nil +} + +// function to load profile from file returns Profile struct and error. It accepts profile file path as string and loads json file from repo Directory and returns Profile struct and error +func (sl *SchemaRepositorySchemaLoader) loadProfileFromFile(path string) (Profile, error) { + + println("Loading profile from " + path) + + extensionFile := commons.CleanPath(commons.Dir(commons.Dir(path)) + "/" + "extension.json") + extension, extensionLoadingError := sl.loadExtension(extensionFile) + + // declare profile + var profile Profile + + // load data from file []byte + data, loadDataError := os.ReadFile(path) + + if loadDataError != nil { + return profile, loadDataError + } + + // unmarshal data into profile + profileUnmarshalError := json.Unmarshal(data, &profile) + + if profileUnmarshalError != nil { + return profile, profileUnmarshalError + } + + profileName := profile.Name + + // check if profile belongs to extension + if extensionLoadingError == nil { + profileName = extension.Name + "/" + profile.Name + } + + if !sl.ProfileExists(profileName) { + println("Ignoring profile " + profileName + " as it is not configured to be loaded.") + return Profile{}, nil + } + + // iterate over attributes and add attribute.profile = profileName + for key, attribute := range profile.Attributes { + attribute.Profile = profileName + profile.Attributes[key] = attribute + } + + return profile, nil +} diff --git a/ocsf/schema/loader-server.go b/ocsf/schema/loader-server.go new file mode 100644 index 0000000..2fe6c0d --- /dev/null +++ b/ocsf/schema/loader-server.go @@ -0,0 +1,193 @@ +package schema + +import ( + "encoding/json" + "fmt" + "io" + "log" + "net/http" + "net/url" + "os" + "strings" + + "github.com/spf13/viper" + "github.com/valllabh/ocsf-tool/commons" +) + +func init() { + // Discontinue server strategy + // RegisterSchemaLoader("server", &SchemaServerSchemaLoader{}) +} + +func (sl *SchemaServerSchemaLoader) Init() { + + // print extensions + fmt.Println("Extensions:", sl.extensions) + + // print profiles + fmt.Println("Profiles:", sl.profiles) + +} + +func (sl *SchemaServerSchemaLoader) Config() { + + // Schema Server Strategy Default Options + viper.SetDefault("schema.loading.strategies.server.url", "https://schema.ocsf.io") + viper.SetDefault("schema.loading.strategies.server.api.version", "1.0.0") + +} + +// Downloads ocsf schema from and write schema to file +func (sl *SchemaServerSchemaLoader) Load() (*OCSFSchema, error) { + + // Load common options + LoadCommonOptions(sl) + + schemaJSON := GetSchemaJsonFilePath(sl) + + // Check if the schema JSON file doe not exists + if !commons.PathExists(schemaJSON) { + // Download the schema and save it to disk + err := sl.downloadSchemaAndSave() + if err != nil { + fmt.Println("Error downloading schema:", err) + os.Exit(1) + } + } + + // Read the JSON file + data, err := os.ReadFile(schemaJSON) + if err != nil { + log.Fatalf("Error reading JSON file: %v", err) + } + + // Define a variable of the struct type + var schema OCSFSchema + + // Unmarshal the JSON data into the struct + if err := json.Unmarshal(data, &schema); err != nil { + log.Fatalf("Error unmarshalling JSON data: %v", err) + } + + return &schema, err +} + +// GetSchemaHash returns the hash of the schema +func (sl *SchemaServerSchemaLoader) GetSchemaHash() string { + return commons.Hash( + strings.Join(sl.GetExtensions(), " "), + strings.Join(sl.GetProfiles(), " "), + viper.GetString("schema.loading.strategies.server.api.version"), + ) +} + +// SetExtensions sets extensions +func (sl *SchemaServerSchemaLoader) SetExtensions(extensions []string) { + sl.extensions = extensions +} + +// GetExtensions returns extensions +func (sl *SchemaServerSchemaLoader) GetExtensions() []string { + return sl.extensions +} + +// SetProfiles sets profiles +func (sl *SchemaServerSchemaLoader) SetProfiles(profiles []string) { + sl.profiles = profiles +} + +// GetProfiles returns profiles +func (sl *SchemaServerSchemaLoader) GetProfiles() []string { + return sl.profiles +} + +// ProfileExists returns true if profile exists +func (sl *SchemaServerSchemaLoader) ProfileExists(profile string) bool { + return commons.Contains(sl.GetProfiles(), profile) +} + +// ExtensionExists returns true if extension exists +func (sl *SchemaServerSchemaLoader) ExtensionExists(extension string) bool { + return commons.Contains(sl.GetExtensions(), extension) +} + +// func to get schema api base url from config +func getSchemaServerURL(path string, queryParams map[string]string) (string, error) { + + url, urlError := url.Parse(viper.GetString("schema.loading.strategies.server.url")) + + url.Path = commons.CleanPath(url.Path + "/" + path) + + // add query param to url from queryParams + q := url.Query() + for key, value := range queryParams { + q.Add(key, value) + } + url.RawQuery = q.Encode() + + return url.String(), urlError +} + +// downloadSchemaAndSave downloads the schema JSON from the API and saves it to a file. +func (sl *SchemaServerSchemaLoader) downloadSchemaAndSave() error { + + // TODO: version specific API calls + + // Build the query string for extensions and profiles + queryParams := map[string]string{} + if len(sl.extensions) > 0 { + queryParams["extensions"] = strings.Join(sl.extensions, ",") + } + if len(sl.profiles) > 0 { + queryParams["profiles"] = strings.Join(sl.profiles, ",") + } + + // Construct the full API URL with query parameters + url, urlError := getSchemaServerURL("export/schema", queryParams) + + if urlError != nil { + fmt.Println("Error constructing API URL:", urlError) + return urlError + } + + // Send a GET request to the API endpoint + fmt.Println("Sending GET request to API endpoint...") + fmt.Printf("API URL: %s\n", url) + resp, err := http.Get(url) + if err != nil { + fmt.Println("Error sending GET request:", err) + return err + } + defer resp.Body.Close() + + // Check if the response status code is 200 (OK) + if resp.StatusCode != http.StatusOK { + fmt.Printf("API returned status code: %d\n", resp.StatusCode) + return fmt.Errorf("API returned status code: %d", resp.StatusCode) + } + + // Read the response body + fmt.Println("Reading response body...") + body, err := io.ReadAll(resp.Body) + if err != nil { + fmt.Println("Error reading response body:", err) + return err + } + + schemaJSON := GetSchemaJsonFilePath(sl) + + // Save the response body as a JSON file + fmt.Printf("Saving schema to file: %s\n", schemaJSON) + + // Create the directory if it does not exist + commons.EnsureDirExists(schemaJSON) + + err = os.WriteFile(schemaJSON, body, 0644) + if err != nil { + fmt.Println("Error writing JSON file:", err) + return err + } + + fmt.Printf("Schema saved to %s\n", schemaJSON) + return nil +} diff --git a/ocsf/schema/schema.go b/ocsf/schema/schema.go new file mode 100644 index 0000000..78ac4d1 --- /dev/null +++ b/ocsf/schema/schema.go @@ -0,0 +1,93 @@ +package schema + +import ( + "fmt" + "os" + + "github.com/spf13/viper" + "github.com/valllabh/ocsf-tool/commons" +) + +var ocsfSchema *OCSFSchema +var schemaLoaders map[string]SchemaLoader = map[string]SchemaLoader{} + +func LoadOCSFSchema() *OCSFSchema { + + // if ocsfSchema is already loaded, return it + if ocsfSchema != nil { + return ocsfSchema + } + + // load schema.loading.strategy from config + strategy := viper.GetString("schema.loading.strategy") + + println("Loading schema using strategy:", strategy) + + // get schema loader for the strategy + schemaLoader, schemaLoaderExists := GetSchemaLoader(strategy) + + if !schemaLoaderExists { + fmt.Println("Invalid schema loading strategy:", strategy) + os.Exit(1) + } + + // initialize the schema loader + schemaLoader.Init() + + // load the schema + ocsfSchema, ocsfSchemaLoadingError := schemaLoader.Load() + if ocsfSchemaLoadingError != nil { + fmt.Println("Error loading schema:", ocsfSchemaLoadingError) + os.Exit(1) + } + + return ocsfSchema +} + +// InitOCSFSchemaLoader initializes all the schema loaders +func init() { + + viper.SetDefault("profiles", []string{}) + viper.SetDefault("extensions", []string{}) + + // set schema path + viper.SetDefault("schema.path", "$CWD/schema") + + // set default schema loading strategy + viper.SetDefault("schema.loading.strategy", "repository") + +} + +// get schema loader by name +func GetSchemaLoader(name string) (SchemaLoader, bool) { + schemaLoader, schemaLoaderExists := schemaLoaders[name] + return schemaLoader, schemaLoaderExists +} + +func RegisterSchemaLoader(name string, schemaLoader SchemaLoader) { + println("Registering schema loader:", name) + schemaLoaders[name] = schemaLoader + + schemaLoaders[name].Config() +} + +func LoadCommonOptions(sl SchemaLoader) { + sl.SetExtensions(viper.GetStringSlice("extensions")) + sl.SetProfiles(viper.GetStringSlice("profiles")) +} + +func GetSchemaJsonFilePath(sl SchemaLoader) string { + + fileNameHash := sl.GetSchemaHash() + + // Load directory from config + path := viper.GetString("schema.path") + + // Prepare file name + path += fmt.Sprintf("/ocsf-schema-%s.json", fileNameHash) + + // Prepare Directory Path + path = commons.PathPrepare(path) + + return path +} diff --git a/ocsf/schema/types.go b/ocsf/schema/types.go index a6c3dee..8068562 100644 --- a/ocsf/schema/types.go +++ b/ocsf/schema/types.go @@ -36,6 +36,7 @@ type Attribute struct { // Event represents an event in the schema. type Event struct { Attributes map[string]Attribute `json:"attributes"` + Extends string `json:"extends"` Name string `json:"name"` Description string `json:"description"` Uid int `json:"uid"` @@ -53,6 +54,7 @@ type Object struct { Description string `json:"description"` Extends string `json:"extends"` Name string `json:"name"` + Profiles []string `json:"profiles"` } // Type represents a type in the schema. @@ -68,11 +70,99 @@ type Type struct { Range []int `json:"range"` } +type Profile struct { + Attributes map[string]Attribute `json:"attributes"` + Name string `json:"name"` + Caption string `json:"caption"` + Meta string `json:"meta"` + Description string `json:"description"` +} + +type Include struct { + Attributes map[string]Attribute `json:"attributes"` + Name string `json:"name"` + Caption string `json:"caption"` + Description string `json:"description"` + Annotations map[string]string `json:"annotations"` +} + // OCSFSchema represents the entire schema. type OCSFSchema struct { - BaseEvent Event `json:"base_event"` - Classes map[string]Event `json:"classes"` - Objects map[string]Object `json:"objects"` - Types map[string]Type `json:"types"` - Version string `json:"version"` + Classes map[string]Event `json:"classes"` + Objects map[string]Object `json:"objects"` + Types map[string]Type `json:"types"` + Version string `json:"version"` + Dictionary Dictionary `json:"dictionary"` +} + +type SchemaLoader interface { + Config() + Init() + Load() (*OCSFSchema, error) + + SetExtensions([]string) + SetProfiles([]string) + GetExtensions() []string + GetProfiles() []string + + ProfileExists(string) bool + ExtensionExists(string) bool + + GetSchemaHash() string +} + +type SchemaRepositorySchemaLoader struct { + extensions []string + profiles []string +} + +type SchemaServerSchemaLoader struct { + extensions []string + profiles []string +} + +type Version struct { + Version string `json:"version"` +} + +type RepositoryObject struct { + Attributes map[string]interface{} `json:"attributes"` + Caption string `json:"caption"` + Constraints Constraints `json:"constraints"` + Description string `json:"description"` + Extends string `json:"extends"` + Name string `json:"name"` +} + +type RepositoryEvent struct { + Attributes map[string]interface{} `json:"attributes"` + Extends string `json:"extends"` + Name string `json:"name"` + Description string `json:"description"` + Uid int `json:"uid"` + Category string `json:"category"` + Caption string `json:"caption"` + Profiles []string `json:"profiles"` + CategoryName interface{} `json:"category_name"` +} + +type RepositoryTypes struct { + Caption string `json:"caption"` + Description string `json:"description"` + Attributes map[string]Type `json:"attributes"` +} + +type Dictionary struct { + Caption string `json:"caption"` + Description string `json:"description"` + Name string `json:"name"` + Attributes map[string]Attribute `json:"attributes"` + Types RepositoryTypes `json:"types"` +} + +type Extension struct { + Caption string `json:"caption"` + Description string `json:"description"` + Name string `json:"name"` + Uid int `json:"uid"` }