diff --git a/cmd/main.go b/cmd/main.go index e9e0624a..4ff06db3 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -70,7 +70,7 @@ var channels = plugins.Channels{ } var report = reporting.Init() -var secretsChan = make(chan reporting.Secret) +var secretsChan = make(chan *reporting.Secret) func Execute() (int, error) { vConfig.SetEnvPrefix(envPrefix) @@ -120,38 +120,20 @@ func preRun(cmd *cobra.Command, args []string) error { return err } - secrets, err := secrets.Init(secretsConfigVar) + engine, err := secrets.Init(secretsConfigVar) if err != nil { return err } - if err := secrets.AddRegexRules(customRegexRuleVar); err != nil { + if err := engine.AddRegexRules(customRegexRuleVar); err != nil { return err } channels.WaitGroup.Add(1) - go func() { - defer channels.WaitGroup.Done() - - wgItems := &sync.WaitGroup{} - for item := range channels.Items { - report.TotalItemsScanned++ - wgItems.Add(1) - go secrets.Detect(item, secretsChan, wgItems, ignoreVar) - } - wgItems.Wait() - close(secretsChan) - }() + go processItems(engine) channels.WaitGroup.Add(1) - go func() { - defer channels.WaitGroup.Done() - for secret := range secretsChan { - report.TotalSecretsFound++ - report.Results[secret.ID] = append(report.Results[secret.ID], secret) - - } - }() + go processSecrets() return nil } diff --git a/cmd/workers.go b/cmd/workers.go new file mode 100644 index 00000000..2246949b --- /dev/null +++ b/cmd/workers.go @@ -0,0 +1,29 @@ +package cmd + +import ( + "sync" + + "github.com/checkmarx/2ms/secrets" +) + +func processItems(engine *secrets.Engine) { + defer channels.WaitGroup.Done() + + wgItems := &sync.WaitGroup{} + for item := range channels.Items { + report.TotalItemsScanned++ + wgItems.Add(1) + go engine.Detect(item, secretsChan, wgItems, ignoreVar) + } + wgItems.Wait() + close(secretsChan) +} + +func processSecrets() { + defer channels.WaitGroup.Done() + + for secret := range secretsChan { + report.TotalSecretsFound++ + report.Results[secret.ID] = append(report.Results[secret.ID], secret) + } +} diff --git a/reporting/report.go b/reporting/report.go index e2a75ff7..1be8437e 100644 --- a/reporting/report.go +++ b/reporting/report.go @@ -17,9 +17,9 @@ const ( ) type Report struct { - TotalItemsScanned int `json:"totalItemsScanned"` - TotalSecretsFound int `json:"totalSecretsFound"` - Results map[string][]Secret `json:"results"` + TotalItemsScanned int `json:"totalItemsScanned"` + TotalSecretsFound int `json:"totalSecretsFound"` + Results map[string][]*Secret `json:"results"` } type Secret struct { @@ -35,7 +35,7 @@ type Secret struct { func Init() *Report { return &Report{ - Results: make(map[string][]Secret), + Results: make(map[string][]*Secret), } } diff --git a/reporting/report_test.go b/reporting/report_test.go index 44a08423..27576b4d 100644 --- a/reporting/report_test.go +++ b/reporting/report_test.go @@ -23,9 +23,9 @@ JPcHeO7M6FohKgcEHX84koQDN98J/L7pFlSoU7WOl6f8BKavIdeSTPS9qQYWdQuT 4Xgur9w/aLZrLM3DSatR+kL+cVTyDTtgCt9Dc8k48Q== -----END RSA PRIVATE KEY-----`) - results := map[string][]Secret{} + results := map[string][]*Secret{} report := Report{len(results), 1, results} - secret := Secret{Source: "bla", StartLine: 0, StartColumn: 0, EndLine: 0, EndColumn: 0, Value: secretValue} + secret := &Secret{Source: "bla", StartLine: 0, StartColumn: 0, EndLine: 0, EndColumn: 0, Value: secretValue} source := "directory\\rawStringAsFile.txt" report.Results[source] = append(report.Results[source], secret) @@ -36,6 +36,6 @@ JPcHeO7M6FohKgcEHX84koQDN98J/L7pFlSoU7WOl6f8BKavIdeSTPS9qQYWdQuT } if !reflect.DeepEqual(report.Results, results) { - t.Errorf("got %q want %q", key, results) + t.Errorf("got %+v want %+v", key, results) } } diff --git a/reporting/sarif.go b/reporting/sarif.go index a26c3d60..db2f13f3 100644 --- a/reporting/sarif.go +++ b/reporting/sarif.go @@ -74,7 +74,7 @@ func getResults(report Report) []Results { return results } -func getLocation(secret Secret) []Locations { +func getLocation(secret *Secret) []Locations { return []Locations{ { PhysicalLocation: PhysicalLocation{ diff --git a/secrets/secrets.go b/secrets/engine.go similarity index 91% rename from secrets/secrets.go rename to secrets/engine.go index 5809eb3c..4f2f05a2 100644 --- a/secrets/secrets.go +++ b/secrets/engine.go @@ -19,7 +19,7 @@ import ( "github.com/zricethezav/gitleaks/v8/report" ) -type Secrets struct { +type Engine struct { rules map[string]config.Rule detector detect.Detector } @@ -34,7 +34,7 @@ type SecretsConfig struct { MaxTargetMegabytes int } -func Init(secretsConfig SecretsConfig) (*Secrets, error) { +func Init(secretsConfig SecretsConfig) (*Engine, error) { selectedRules := rules.FilterRules(secretsConfig.SelectedList, secretsConfig.IgnoreList, secretsConfig.SpecialList) if len(*selectedRules) == 0 { return nil, fmt.Errorf("no rules were selected") @@ -52,13 +52,13 @@ func Init(secretsConfig SecretsConfig) (*Secrets, error) { }) detector.MaxTargetMegaBytes = secretsConfig.MaxTargetMegabytes - return &Secrets{ + return &Engine{ rules: rulesToBeApplied, detector: *detector, }, nil } -func (s *Secrets) Detect(item plugins.Item, secretsChannel chan reporting.Secret, wg *sync.WaitGroup, ignoredIds []string) { +func (s *Engine) Detect(item plugins.Item, secretsChannel chan *reporting.Secret, wg *sync.WaitGroup, ignoredIds []string) { defer wg.Done() fragment := detect.Fragment{ @@ -66,7 +66,7 @@ func (s *Secrets) Detect(item plugins.Item, secretsChannel chan reporting.Secret } for _, value := range s.detector.Detect(fragment) { itemId := getFindingId(item, value) - secret := reporting.Secret{ + secret := &reporting.Secret{ ID: itemId, Source: item.Source, RuleID: value.RuleID, @@ -76,7 +76,7 @@ func (s *Secrets) Detect(item plugins.Item, secretsChannel chan reporting.Secret EndColumn: value.EndColumn, Value: value.Secret, } - if !isSecretIgnored(&secret, &ignoredIds) { + if !isSecretIgnored(secret, &ignoredIds) { secretsChannel <- secret } else { log.Debug().Msgf("Secret %s was ignored", secret.ID) @@ -84,7 +84,7 @@ func (s *Secrets) Detect(item plugins.Item, secretsChannel chan reporting.Secret } } -func (s *Secrets) AddRegexRules(patterns []string) error { +func (s *Engine) AddRegexRules(patterns []string) error { for idx, pattern := range patterns { regex, err := regexp.Compile(pattern) if err != nil { diff --git a/secrets/secrets_test.go b/secrets/engine_test.go similarity index 96% rename from secrets/secrets_test.go rename to secrets/engine_test.go index 44bdccb6..3df10a8c 100644 --- a/secrets/secrets_test.go +++ b/secrets/engine_test.go @@ -123,17 +123,18 @@ func TestSecrets(t *testing.T) { } t.Run(name, func(t *testing.T) { fmt.Printf("Start test %s", name) - secretsChan := make(chan reporting.Secret, 1) + secretsChan := make(chan *reporting.Secret, 1) wg := &sync.WaitGroup{} wg.Add(1) detector.Detect(plugins.Item{Content: secret.Content}, secretsChan, wg, nil) close(secretsChan) s := <-secretsChan - if s.Value == "" && secret.ShouldFind { + + if s == nil && secret.ShouldFind { t.Errorf("secret \"%s\" not found", secret.Name) } - if s.Value != "" && !secret.ShouldFind { + if s != nil && !secret.ShouldFind { t.Errorf("should not find") } })