From 48adfd8fb3e883e76b855d66d27b7324cd31d8ba Mon Sep 17 00:00:00 2001 From: cx-monicac Date: Wed, 17 May 2023 15:42:03 +0100 Subject: [PATCH] feat: output report to file + different formats (#71) close #40 --- cmd/main.go | 45 ++++++++++-- config/config.go | 10 +++ go.mod | 1 + go.sum | 2 + reporting/json.go | 15 ++++ reporting/report.go | 85 +++++++++++---------- reporting/report_test.go | 2 +- reporting/sarif.go | 154 +++++++++++++++++++++++++++++++++++++++ reporting/yaml.go | 15 ++++ secrets/secrets.go | 20 ++++- 10 files changed, 303 insertions(+), 46 deletions(-) create mode 100644 config/config.go create mode 100644 reporting/json.go create mode 100644 reporting/sarif.go create mode 100644 reporting/yaml.go diff --git a/cmd/main.go b/cmd/main.go index 55828449..4aba7cd3 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -2,7 +2,9 @@ package cmd import ( "fmt" + "github.com/checkmarx/2ms/config" "os" + "path/filepath" "strings" "sync" @@ -17,13 +19,17 @@ import ( "github.com/spf13/cobra" ) -const timeSleepInterval = 50 - var Version = "0.0.0" const ( - tagsFlagName = "tags" - logLevelFlagName = "log-level" + timeSleepInterval = 50 + tagsFlagName = "tags" + logLevelFlagName = "log-level" + reportPath = "report-path" + stdoutFormat = "stdout-format" + jsonFormat = "json" + yamlFormat = "yaml" + sarifFormat = "sarif" ) var rootCmd = &cobra.Command{ @@ -76,6 +82,8 @@ func Execute() { cobra.OnInitialize(initLog) rootCmd.PersistentFlags().StringSlice(tagsFlagName, []string{"all"}, "select rules to be applied") rootCmd.PersistentFlags().String(logLevelFlagName, "info", "log level (trace, debug, info, warn, error, fatal)") + rootCmd.PersistentFlags().StringSlice(reportPath, []string{""}, "path to generate report files. The output format will be determined by the file extension (.json, .yaml, .sarif)") + rootCmd.PersistentFlags().String(stdoutFormat, "yaml", "stdout output format, available formats are: json, yaml, sarif") rootCmd.PersistentPreRun = preRun rootCmd.PersistentPostRun = postRun @@ -111,6 +119,20 @@ func validateTags(tags []string) { } } +func validateFormat(stdout string, reportPath []string) { + if !(strings.EqualFold(stdout, yamlFormat) || strings.EqualFold(stdout, jsonFormat) || strings.EqualFold(stdout, sarifFormat)) { + log.Fatal().Msgf(`invalid output format: %s, available formats are: json, yaml and sarif`, stdout) + } + for _, path := range reportPath { + + fileExtension := filepath.Ext(path) + format := strings.TrimPrefix(fileExtension, ".") + if !(strings.EqualFold(format, yamlFormat) || strings.EqualFold(format, jsonFormat) || strings.EqualFold(format, sarifFormat)) { + log.Fatal().Msgf(`invalid report extension: %s, available extensions are: json, yaml and sarif`, format) + } + } +} + func preRun(cmd *cobra.Command, args []string) { tags, err := cmd.Flags().GetStringSlice(tagsFlagName) if err != nil { @@ -144,13 +166,26 @@ func preRun(cmd *cobra.Command, args []string) { func postRun(cmd *cobra.Command, args []string) { channels.WaitGroup.Wait() + reportPath, _ := cmd.Flags().GetStringSlice(reportPath) + stdoutFormat, _ := cmd.Flags().GetString(stdoutFormat) + + validateFormat(stdoutFormat, reportPath) + + cfg := config.LoadConfig("2ms", Version) + // Wait for last secret to be added to report time.Sleep(time.Millisecond * timeSleepInterval) // ------------------------------------- // Show Report if report.TotalItemsScanned > 0 { - report.ShowReport() + report.ShowReport(stdoutFormat, cfg) + if len(reportPath) > 0 { + err := report.WriteFile(reportPath, cfg) + if err != nil { + log.Error().Msgf("Failed to create report file with error: %s", err) + } + } } else { log.Error().Msg("Scan completed with empty content") os.Exit(0) diff --git a/config/config.go b/config/config.go new file mode 100644 index 00000000..bebe909b --- /dev/null +++ b/config/config.go @@ -0,0 +1,10 @@ +package config + +type Config struct { + Name string + Version string +} + +func LoadConfig(name string, version string) *Config { + return &Config{Name: name, Version: version} +} diff --git a/go.mod b/go.mod index 3fa40718..5b18c6bb 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/spf13/cobra v1.6.1 github.com/stretchr/testify v1.8.1 github.com/zricethezav/gitleaks/v8 v8.16.1 + gopkg.in/yaml.v2 v2.4.0 ) require ( diff --git a/go.sum b/go.sum index b1ea402f..7503ea95 100644 --- a/go.sum +++ b/go.sum @@ -524,6 +524,8 @@ gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/reporting/json.go b/reporting/json.go new file mode 100644 index 00000000..14a8df1a --- /dev/null +++ b/reporting/json.go @@ -0,0 +1,15 @@ +package reporting + +import ( + "encoding/json" + "log" +) + +func writeJson(report Report) string { + jsonReport, err := json.MarshalIndent(report, "", " ") + if err != nil { + log.Fatalf("failed to create Json report with error: %v", err) + } + + return string(jsonReport) +} diff --git a/reporting/report.go b/reporting/report.go index a521245a..1d4771c2 100644 --- a/reporting/report.go +++ b/reporting/report.go @@ -2,24 +2,33 @@ package reporting import ( "fmt" + "github.com/checkmarx/2ms/config" + "os" "path/filepath" "strings" ) +const ( + jsonFormat = "json" + yamlFormat = "yaml" + sarifFormat = "sarif" +) + type Report struct { - Results map[string][]Secret - TotalItemsScanned int - TotalSecretsFound int + TotalItemsScanned int `json:"totalItemsScanned"` + TotalSecretsFound int `json:"totalSecretsFound"` + Results map[string][]Secret `json:"results"` } type Secret struct { - ID string - Description string - StartLine int - EndLine int - StartColumn int - EndColumn int - Value string + ID string `json:"id"` + Source string `json:"source"` + Description string `json:"description"` + StartLine int `json:"startLine"` + EndLine int `json:"endLine"` + StartColumn int `json:"startColumn"` + EndColumn int `json:"endColumn"` + Value string `json:"value"` } func Init() *Report { @@ -28,39 +37,41 @@ func Init() *Report { } } -func (r *Report) ShowReport() { - fmt.Println("Summary:") - fmt.Printf("- Total items scanned: %d\n", r.TotalItemsScanned) - fmt.Printf("- Total items with secrets: %d\n", len(r.Results)) - if len(r.Results) > 0 { - fmt.Printf("- Total secrets found: %d\n", r.TotalSecretsFound) - fmt.Println("Detailed Report:") - r.generateResultsReport() - } +func (r *Report) ShowReport(format string, cfg *config.Config) { + output := r.getOutput(format, cfg) + fmt.Println("Summary:") + fmt.Print(output) } -func (r *Report) generateResultsReport() { - for source, secrets := range r.Results { - itemId := getItemId(source) - fmt.Printf("- Item ID: %s\n", itemId) - fmt.Printf(" - Item Full Path: %s\n", source) - fmt.Println(" - Secrets:") - for _, secret := range secrets { - fmt.Printf(" - Type: %s\n", secret.Description) - fmt.Printf(" - Value: %.40s\n", secret.Value) +func (r *Report) WriteFile(reportPath []string, cfg *config.Config) error { + for _, path := range reportPath { + file, err := os.Create(path) + if err != nil { + return err + } + + fileExtension := filepath.Ext(path) + format := strings.TrimPrefix(fileExtension, ".") + output := r.getOutput(format, cfg) + + _, err = file.WriteString(output) + if err != nil { + return err } } + return nil } -func getItemId(fullPath string) string { - var itemId string - if strings.Contains(fullPath, "/") { - itemLinkStrings := strings.Split(fullPath, "/") - itemId = itemLinkStrings[len(itemLinkStrings)-1] - } - if strings.Contains(fullPath, "\\") { - itemId = filepath.Base(fullPath) +func (r *Report) getOutput(format string, cfg *config.Config) string { + var output string + switch format { + case jsonFormat: + output = writeJson(*r) + case yamlFormat: + output = writeYaml(*r) + case sarifFormat: + output = writeSarif(*r, cfg) } - return itemId + return output } diff --git a/reporting/report_test.go b/reporting/report_test.go index 6ed251c6..c05cdbe4 100644 --- a/reporting/report_test.go +++ b/reporting/report_test.go @@ -24,7 +24,7 @@ JPcHeO7M6FohKgcEHX84koQDN98J/L7pFlSoU7WOl6f8BKavIdeSTPS9qQYWdQuT -----END RSA PRIVATE KEY-----`) results := map[string][]Secret{} - report := Report{results, 1, 1} + report := Report{len(results), 1, results} secret := Secret{Description: "bla", StartLine: 0, StartColumn: 0, EndLine: 0, EndColumn: 0, Value: secretValue} source := "directory\\rawStringAsFile.txt" diff --git a/reporting/sarif.go b/reporting/sarif.go new file mode 100644 index 00000000..e2357a91 --- /dev/null +++ b/reporting/sarif.go @@ -0,0 +1,154 @@ +package reporting + +import ( + "encoding/json" + "fmt" + "github.com/checkmarx/2ms/config" + "log" +) + +func writeSarif(report Report, cfg *config.Config) string { + sarif := Sarif{ + Schema: "https://schemastore.azurewebsites.net/schemas/json/sarif-2.1.0-rtm.5.json", + Version: "2.1.0", + Runs: getRuns(report, cfg), + } + + sarifReport, err := json.MarshalIndent(sarif, "", " ") + if err != nil { + log.Fatalf("failed to create Sarif report with error: %v", err) + } + + return string(sarifReport) +} + +func getRuns(report Report, cfg *config.Config) []Runs { + return []Runs{ + { + Tool: getTool(cfg), + Results: getResults(report), + }, + } +} + +func getTool(cfg *config.Config) Tool { + tool := Tool{ + Driver: Driver{ + Name: cfg.Name, + SemanticVersion: cfg.Version, + }, + } + + return tool +} + +func hasNoResults(report Report) bool { + return len(report.Results) == 0 +} + +func messageText(secret Secret) string { + return fmt.Sprintf("%s has detected secret for file %s.", secret.Description, secret.ID) +} + +func getResults(report Report) []Results { + var results []Results + + // if this report has no results, ensure that it is represented as [] instead of null/nil + if hasNoResults(report) { + results = make([]Results, 0) + return results + } + + for _, secrets := range report.Results { + for _, secret := range secrets { + r := Results{ + Message: Message{ + Text: messageText(secret), + }, + RuleId: secret.Description, + Locations: getLocation(secret), + } + results = append(results, r) + } + } + return results +} + +func getLocation(secret Secret) []Locations { + return []Locations{ + { + PhysicalLocation: PhysicalLocation{ + ArtifactLocation: ArtifactLocation{ + URI: secret.ID, + }, + Region: Region{ + StartLine: secret.StartLine, + EndLine: secret.EndLine, + StartColumn: secret.StartColumn, + EndColumn: secret.EndColumn, + Snippet: Snippet{ + Text: secret.Value, + }, + }, + }, + }, + } +} + +type Sarif struct { + Schema string `json:"$schema"` + Version string `json:"version"` + Runs []Runs `json:"runs"` +} +type ShortDescription struct { + Text string `json:"text"` +} + +type Driver struct { + Name string `json:"name"` + SemanticVersion string `json:"semanticVersion"` +} + +type Tool struct { + Driver Driver `json:"driver"` +} + +type Message struct { + Text string `json:"text"` +} + +type ArtifactLocation struct { + URI string `json:"uri"` +} + +type Region struct { + StartLine int `json:"startLine"` + StartColumn int `json:"startColumn"` + EndLine int `json:"endLine"` + EndColumn int `json:"endColumn"` + Snippet Snippet `json:"snippet"` +} + +type Snippet struct { + Text string `json:"text"` +} + +type PhysicalLocation struct { + ArtifactLocation ArtifactLocation `json:"artifactLocation"` + Region Region `json:"region"` +} + +type Locations struct { + PhysicalLocation PhysicalLocation `json:"physicalLocation"` +} + +type Results struct { + Message Message `json:"message"` + RuleId string `json:"ruleId"` + Locations []Locations `json:"locations"` +} + +type Runs struct { + Tool Tool `json:"tool"` + Results []Results `json:"results"` +} diff --git a/reporting/yaml.go b/reporting/yaml.go new file mode 100644 index 00000000..f91bf7d1 --- /dev/null +++ b/reporting/yaml.go @@ -0,0 +1,15 @@ +package reporting + +import ( + "gopkg.in/yaml.v2" + "log" +) + +func writeYaml(report Report) string { + yamlReport, err := yaml.Marshal(&report) + if err != nil { + log.Fatalf("failed to create Yaml report with error: %v", err) + } + + return string(yamlReport) +} diff --git a/secrets/secrets.go b/secrets/secrets.go index 6d109a54..b62fcb33 100644 --- a/secrets/secrets.go +++ b/secrets/secrets.go @@ -6,6 +6,7 @@ import ( "github.com/zricethezav/gitleaks/v8/cmd/generate/config/rules" "github.com/zricethezav/gitleaks/v8/config" "github.com/zricethezav/gitleaks/v8/detect" + "path/filepath" "strings" "sync" ) @@ -45,11 +46,11 @@ func Init(tags []string) *Secrets { allRules, _ := loadAllRules() rulesToBeApplied := getRules(allRules, tags) - cfg := config.Config{ + config := config.Config{ Rules: rulesToBeApplied, } - detector := detect.NewDetector(cfg) + detector := detect.NewDetector(config) return &Secrets{ rules: rulesToBeApplied, @@ -64,10 +65,23 @@ func (s *Secrets) Detect(secretsChannel chan reporting.Secret, item plugins.Item Raw: item.Content, } for _, value := range s.detector.Detect(fragment) { - secretsChannel <- reporting.Secret{ID: item.ID, Description: value.Description, StartLine: value.StartLine, StartColumn: value.StartColumn, EndLine: value.EndLine, EndColumn: value.EndColumn, Value: value.Secret} + itemId := getItemId(item.ID) + secretsChannel <- reporting.Secret{ID: itemId, Source: item.ID, Description: value.Description, StartLine: value.StartLine, StartColumn: value.StartColumn, EndLine: value.EndLine, EndColumn: value.EndColumn, Value: value.Secret} } } +func getItemId(fullPath string) string { + var itemId string + if strings.Contains(fullPath, "/") { + itemLinkStrings := strings.Split(fullPath, "/") + itemId = itemLinkStrings[len(itemLinkStrings)-1] + } + if strings.Contains(fullPath, "\\") { + itemId = filepath.Base(fullPath) + } + return itemId +} + func getRules(allRules []Rule, tags []string) map[string]config.Rule { rulesToBeApplied := make(map[string]config.Rule)