diff --git a/cmd/main.go b/cmd/main.go index da8f38a2..f92c4855 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -2,10 +2,10 @@ package cmd import ( "fmt" + "log" "path/filepath" "regexp" "strings" - "time" "github.com/checkmarx/2ms/config" "github.com/checkmarx/2ms/lib" @@ -131,7 +131,7 @@ func Execute() error { rootCmd.AddGroup(&cobra.Group{Title: group, ID: group}) for _, plugin := range allPlugins { - subCommand, err := plugin.DefineCommand(channels) + subCommand, err := plugin.DefineCommand(channels.Items, channels.Errors) if err != nil { return fmt.Errorf("error while defining command for plugin %s: %s", plugin.GetName(), err.Error()) } @@ -187,22 +187,32 @@ func preRun(cmd *cobra.Command, args []string) error { return err } + channels.WaitGroup.Add(1) go func() { - for { - select { - case item := <-channels.Items: - report.TotalItemsScanned++ - channels.WaitGroup.Add(1) - go secrets.Detect(item, secretsChan, channels.WaitGroup, ignoreVar) - case secret := <-secretsChan: - report.TotalSecretsFound++ - report.Results[secret.ID] = append(report.Results[secret.ID], secret) - case err, ok := <-channels.Errors: - if !ok || ShowError("errors") { - return - } - errorChan <- err - } + defer channels.WaitGroup.Done() + + wgItems := &sync.WaitGroup{} + for item := range channels.Items { + report.TotalItemsScanned++ + wgItems.Add(1) + go secrets.Detect(item, secretsChan, wgItems, ignoreVar) + } + wgItems.Wait() + close(secretsChan) + }() + + channels.WaitGroup.Add(1) + go func() { + defer channels.WaitGroup.Done() + for secret := range secretsChan { + report.TotalSecretsFound++ + report.Results[secret.ID] = append(report.Results[secret.ID], secret) + } + }() + + go func() { + for err := range channels.Errors { + log.Fatal().Msg(err.Error()) } }() @@ -214,15 +224,6 @@ func postRun(cmd *cobra.Command, args []string) error { cfg := config.LoadConfig("2ms", Version) - // Wait for last secret to be added to report - time.Sleep(time.Millisecond * timeSleepInterval) - - if len(errorChan) != 0 { - errorInChan := <-errorChan - close(errorChan) - return errorInChan - } - // ------------------------------------- // Show Report if report.TotalItemsScanned > 0 { diff --git a/go.mod b/go.mod index 7846b375..96131a61 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.15.0 github.com/stretchr/testify v1.8.1 - github.com/zricethezav/gitleaks/v8 v8.16.1 + github.com/zricethezav/gitleaks/v8 v8.17.1-0.20230717122715-f0dcd4d9cfe9 golang.org/x/time v0.1.0 gopkg.in/yaml.v2 v2.4.0 ) diff --git a/go.sum b/go.sum index e4cfe166..11b55700 100644 --- a/go.sum +++ b/go.sum @@ -227,8 +227,8 @@ github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/zricethezav/gitleaks/v8 v8.16.1 h1:Y+x4GNIZx70fPQBJseuZydecadlgbKUJgirNeySSZw8= -github.com/zricethezav/gitleaks/v8 v8.16.1/go.mod h1:JzsbTtA88h1ioImguqG0BL2IV2JfblVO0qj/LSwgUKQ= +github.com/zricethezav/gitleaks/v8 v8.17.1-0.20230717122715-f0dcd4d9cfe9 h1:gw0iPgtVuWBW1XQoZed9Y0rWaZ9la1qOooa6aRHsEFo= +github.com/zricethezav/gitleaks/v8 v8.17.1-0.20230717122715-f0dcd4d9cfe9/go.mod h1:/0z7cslO7d0y29YRvHgYefeTu7UIqOmx95A4wMhcQtE= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= diff --git a/plugins/confluence.go b/plugins/confluence.go index 05f00fec..de579e51 100644 --- a/plugins/confluence.go +++ b/plugins/confluence.go @@ -46,7 +46,7 @@ func (p *ConfluencePlugin) GetAuthorizationHeader() string { return lib.CreateBasicAuthCredentials(p) } -func (p *ConfluencePlugin) DefineCommand(channels Channels) (*cobra.Command, error) { +func (p *ConfluencePlugin) DefineCommand(items chan Item, errors chan error) (*cobra.Command, error) { var confluenceCmd = &cobra.Command{ Use: fmt.Sprintf("%s --%s URL", p.GetName(), argUrl), Short: "Scan Confluence server", @@ -67,11 +67,14 @@ func (p *ConfluencePlugin) DefineCommand(channels Channels) (*cobra.Command, err confluenceCmd.Run = func(cmd *cobra.Command, args []string) { err := p.initialize(cmd) if err != nil { - channels.Errors <- fmt.Errorf("error while initializing confluence plugin: %w", err) + errors <- fmt.Errorf("error while initializing confluence plugin: %w", err) return } - p.getItems(channels.Items, channels.Errors, channels.WaitGroup) + wg := &sync.WaitGroup{} + p.getItems(items, errors, wg) + wg.Wait() + close(items) } return confluenceCmd, nil @@ -90,18 +93,14 @@ func (p *ConfluencePlugin) initialize(cmd *cobra.Command) error { } func (p *ConfluencePlugin) getItems(items chan Item, errs chan error, wg *sync.WaitGroup) { - p.getSpacesItems(items, errs, wg) -} - -func (p *ConfluencePlugin) getSpacesItems(items chan Item, errs chan error, wg *sync.WaitGroup) { spaces, err := p.getSpaces() if err != nil { errs <- err } for _, space := range spaces { - go p.getSpaceItems(items, errs, wg, space) wg.Add(1) + go p.getSpaceItems(items, errs, wg, space) } } diff --git a/plugins/discord.go b/plugins/discord.go index 6f6e5e12..ed49f591 100644 --- a/plugins/discord.go +++ b/plugins/discord.go @@ -38,7 +38,7 @@ func (p *DiscordPlugin) GetName() string { return "discord" } -func (p *DiscordPlugin) DefineCommand(channels Channels) (*cobra.Command, error) { +func (p *DiscordPlugin) DefineCommand(items chan Item, errors chan error) (*cobra.Command, error) { var discordCmd = &cobra.Command{ Use: fmt.Sprintf("%s --%s TOKEN --%s SERVER", p.GetName(), tokenFlag, serversFlag), Short: "Scan Discord server", @@ -63,11 +63,14 @@ func (p *DiscordPlugin) DefineCommand(channels Channels) (*cobra.Command, error) discordCmd.Run = func(cmd *cobra.Command, args []string) { err := p.initialize(cmd) if err != nil { - channels.Errors <- fmt.Errorf("discord plugin initialization failed: %w", err) + errors <- fmt.Errorf("discord plugin initialization failed: %w", err) return } - p.getItems(channels.Items, channels.Errors, channels.WaitGroup) + wg := &sync.WaitGroup{} + p.getItems(items, errors, wg) + wg.Wait() + close(items) } return discordCmd, nil diff --git a/plugins/filesystem.go b/plugins/filesystem.go index 3a2a7069..c6d58b73 100644 --- a/plugins/filesystem.go +++ b/plugins/filesystem.go @@ -29,17 +29,17 @@ func (p *FileSystemPlugin) GetName() string { return "filesystem" } -func (p *FileSystemPlugin) DefineCommand(channels Channels) (*cobra.Command, error) { +func (p *FileSystemPlugin) DefineCommand(items chan Item, errors chan error) (*cobra.Command, error) { var cmd = &cobra.Command{ Use: fmt.Sprintf("%s --%s PATH", p.GetName(), flagFolder), Short: "Scan local folder", Long: "Scan local folder for sensitive information", RunE: func(cmd *cobra.Command, args []string) error { log.Info().Msg("Folder plugin started") - if err := p.getFiles(channels.Items, channels.Errors, channels.WaitGroup); err != nil { - return err - } - return nil + wg := &sync.WaitGroup{} + p.getFiles(items, errors, wg) + wg.Wait() + close(items) }, } diff --git a/plugins/git.go b/plugins/git.go index 554f373a..a8f20d2a 100644 --- a/plugins/git.go +++ b/plugins/git.go @@ -4,6 +4,7 @@ import ( "fmt" "os" "strings" + "sync" "github.com/gitleaks/go-gitdiff/gitdiff" "github.com/rs/zerolog/log" @@ -29,8 +30,12 @@ func (p *GitPlugin) GetName() string { return "git" } -func (p *GitPlugin) DefineCommand(channels Channels) (*cobra.Command, error) { - p.Channels = channels +func (p *GitPlugin) DefineCommand(items chan Item, errors chan error) (*cobra.Command, error) { + p.Channels = Channels{ + Items: items, + Errors: errors, + WaitGroup: &sync.WaitGroup{}, + } command := &cobra.Command{ Use: fmt.Sprintf("%s ", p.GetName()), @@ -39,7 +44,9 @@ func (p *GitPlugin) DefineCommand(channels Channels) (*cobra.Command, error) { Args: cobra.MatchAll(cobra.ExactArgs(1), validGitRepoArgs), Run: func(cmd *cobra.Command, args []string) { log.Info().Msg("Git plugin started") - p.scanGit(args[0], p.buildScanOptions(), channels.Items, channels.Errors) + p.scanGit(args[0], p.buildScanOptions(), p.Channels.Items, p.Channels.Errors) + p.WaitGroup.Wait() + close(items) }, } flags := command.Flags() diff --git a/plugins/paligo.go b/plugins/paligo.go index 3f2a28d1..bbeaafa6 100644 --- a/plugins/paligo.go +++ b/plugins/paligo.go @@ -7,6 +7,7 @@ import ( "net/http" "strconv" "strings" + "sync" "time" "github.com/checkmarx/2ms/lib" @@ -54,8 +55,12 @@ func (p *PaligoPlugin) GetName() string { return "paligo" } -func (p *PaligoPlugin) DefineCommand(channels Channels) (*cobra.Command, error) { - p.Channels = channels +func (p *PaligoPlugin) DefineCommand(items chan Item, errors chan error) (*cobra.Command, error) { + p.Channels = Channels{ + Items: items, + Errors: errors, + WaitGroup: &sync.WaitGroup{}, + } command := &cobra.Command{ Use: fmt.Sprintf("%s --%s %s --%s %s --%s %s", @@ -73,6 +78,8 @@ func (p *PaligoPlugin) DefineCommand(channels Channels) (*cobra.Command, error) } log.Info().Msg("Paligo plugin started") p.getItems() + p.WaitGroup.Wait() + close(items) }, } diff --git a/plugins/plugins.go b/plugins/plugins.go index a22e2f67..f8236614 100644 --- a/plugins/plugins.go +++ b/plugins/plugins.go @@ -27,5 +27,5 @@ type Channels struct { type IPlugin interface { GetName() string - DefineCommand(channels Channels) (*cobra.Command, error) + DefineCommand(items chan Item, errors chan error) (*cobra.Command, error) } diff --git a/plugins/slack.go b/plugins/slack.go index d3862bd5..b88c9505 100644 --- a/plugins/slack.go +++ b/plugins/slack.go @@ -3,6 +3,7 @@ package plugins import ( "fmt" "strconv" + "sync" "time" "github.com/rs/zerolog/log" @@ -38,8 +39,12 @@ var ( messagesCountArg int ) -func (p *SlackPlugin) DefineCommand(channels Channels) (*cobra.Command, error) { - p.Channels = channels +func (p *SlackPlugin) DefineCommand(items chan Item, errors chan error) (*cobra.Command, error) { + p.Channels = Channels{ + Items: items, + Errors: errors, + WaitGroup: &sync.WaitGroup{}, + } command := &cobra.Command{ Use: fmt.Sprintf("%s --%s TOKEN --%s TEAM", p.GetName(), slackTokenFlag, slackTeamFlag), @@ -47,6 +52,8 @@ func (p *SlackPlugin) DefineCommand(channels Channels) (*cobra.Command, error) { Long: "Scan Slack team for sensitive information.", Run: func(cmd *cobra.Command, args []string) { p.getItems() + p.Channels.WaitGroup.Wait() + close(items) }, } diff --git a/secrets/rules/authenticated_url.go b/secrets/rules/authenticated_url.go new file mode 100644 index 00000000..999533f1 --- /dev/null +++ b/secrets/rules/authenticated_url.go @@ -0,0 +1,31 @@ +package rules + +import ( + "regexp" + + "github.com/zricethezav/gitleaks/v8/config" +) + +func AuthenticatedURL() *config.Rule { + regex, _ := regexp.Compile(`:\/\/(.+:.+)?@`) + rule := config.Rule{ + Description: "Identify username:password inside URLS", + RuleID: "authenticated-url", + Regex: regex, + Keywords: []string{}, + SecretGroup: 1, + } + + tPositives := []string{ + "mongodb+srv://radar:mytoken@io.dbb.mongodb.net/?retryWrites=true&w=majority", + "--output=https://elastic:bF21iC0bfTVXo3qhpJqTGs78@c22f5bc9787c4c268d3b069ad866bdc2.eu-central-1.aws.cloud.es.io:9243/tfs", + "https://abc:123@google.com", + } + + fPositives := []string{ + "https://google.com", + "https://google.com?user=abc&password=123", + } + + return validate(rule, tPositives, fPositives) +} diff --git a/secrets/rules/rule.go b/secrets/rules/rule.go new file mode 100644 index 00000000..59d4b36e --- /dev/null +++ b/secrets/rules/rule.go @@ -0,0 +1,37 @@ +package rules + +import ( + "strings" + + "github.com/rs/zerolog/log" + "github.com/zricethezav/gitleaks/v8/config" + "github.com/zricethezav/gitleaks/v8/detect" +) + +// Copied from https://github.com/gitleaks/gitleaks/blob/463d24618fa42fc7629dc30c9744ebe36c5df1ab/cmd/generate/config/rules/rule.go +func validate(r config.Rule, truePositives []string, falsePositives []string) *config.Rule { + // normalize keywords like in the config package + var keywords []string + for _, k := range r.Keywords { + keywords = append(keywords, strings.ToLower(k)) + } + r.Keywords = keywords + + rules := make(map[string]config.Rule) + rules[r.RuleID] = r + d := detect.NewDetector(config.Config{ + Rules: rules, + Keywords: keywords, + }) + for _, tp := range truePositives { + if len(d.DetectString(tp)) != 1 { + log.Fatal().Msgf("Failed to validate. For rule ID [%s], true positive [%s] was not detected by regexp [%s]", r.RuleID, tp, r.Regex) + } + } + for _, fp := range falsePositives { + if len(d.DetectString(fp)) != 0 { + log.Fatal().Msgf("Failed to validate. For rule ID [%s], false positive [%s] was detected by regexp [%s]", r.RuleID, fp, r.Regex) + } + } + return &r +} diff --git a/secrets/rules/rule_test.go b/secrets/rules/rule_test.go new file mode 100644 index 00000000..34e27da3 --- /dev/null +++ b/secrets/rules/rule_test.go @@ -0,0 +1,28 @@ +package rules_test + +import ( + "testing" + + "github.com/checkmarx/2ms/secrets/rules" + "github.com/zricethezav/gitleaks/v8/config" +) + +func Test2msRules(t *testing.T) { + t.Parallel() + + testsRules := []struct { + name string + validate func() *config.Rule + }{ + {name: "AuthenticatedURL", validate: rules.AuthenticatedURL}, + } + + for _, tRule := range testsRules { + testRule := tRule // fix for loop variable being captured by func literal + t.Run(testRule.name, func(t *testing.T) { + t.Parallel() + + testRule.validate() + }) + } +} diff --git a/secrets/secrets.go b/secrets/secrets.go index d98bd3b0..11c17a9d 100644 --- a/secrets/secrets.go +++ b/secrets/secrets.go @@ -11,6 +11,7 @@ import ( "github.com/checkmarx/2ms/plugins" "github.com/checkmarx/2ms/reporting" + internalRules "github.com/checkmarx/2ms/secrets/rules" "github.com/rs/zerolog/log" "github.com/spf13/cobra" "github.com/zricethezav/gitleaks/v8/cmd/generate/config/rules" @@ -292,6 +293,8 @@ func loadAllRules() ([]Rule, error) { allRules = append(allRules, Rule{Rule: *rules.Heroku(), Tags: []string{TagApiKey}}) allRules = append(allRules, Rule{Rule: *rules.HubSpot(), Tags: []string{TagApiToken, TagApiKey}}) allRules = append(allRules, Rule{Rule: *rules.Intercom(), Tags: []string{TagApiToken, TagApiKey}}) + allRules = append(allRules, Rule{Rule: *rules.JFrogAPIKey(), Tags: []string{TagApiKey}}) + allRules = append(allRules, Rule{Rule: *rules.JFrogIdentityToken(), Tags: []string{TagAccessToken}}) allRules = append(allRules, Rule{Rule: *rules.JWT(), Tags: []string{TagAccessToken}}) allRules = append(allRules, Rule{Rule: *rules.KrakenAccessToken(), Tags: []string{TagAccessToken}}) allRules = append(allRules, Rule{Rule: *rules.KucoinAccessToken(), Tags: []string{TagAccessToken}}) @@ -344,8 +347,15 @@ func loadAllRules() ([]Rule, error) { allRules = append(allRules, Rule{Rule: *rules.ShopifySharedSecret(), Tags: []string{TagPublicSecret}}) allRules = append(allRules, Rule{Rule: *rules.SidekiqSecret(), Tags: []string{TagSecretKey}}) allRules = append(allRules, Rule{Rule: *rules.SidekiqSensitiveUrl(), Tags: []string{TagSensitiveUrl}}) - allRules = append(allRules, Rule{Rule: *rules.SlackAccessToken(), Tags: []string{TagAccessToken}}) - allRules = append(allRules, Rule{Rule: *rules.SlackWebHook(), Tags: []string{TagWebhook}}) + allRules = append(allRules, Rule{Rule: *rules.SlackBotToken(), Tags: []string{TagAccessToken}}) + allRules = append(allRules, Rule{Rule: *rules.SlackAppLevelToken(), Tags: []string{TagAccessToken}}) + allRules = append(allRules, Rule{Rule: *rules.SlackLegacyToken(), Tags: []string{TagAccessToken}}) + allRules = append(allRules, Rule{Rule: *rules.SlackUserToken(), Tags: []string{TagAccessToken}}) + allRules = append(allRules, Rule{Rule: *rules.SlackConfigurationToken(), Tags: []string{TagAccessToken}}) + allRules = append(allRules, Rule{Rule: *rules.SlackConfigurationRefreshToken(), Tags: []string{TagRefreshToken}}) + allRules = append(allRules, Rule{Rule: *rules.SlackLegacyBotToken(), Tags: []string{TagAccessToken}}) + allRules = append(allRules, Rule{Rule: *rules.SlackLegacyWorkspaceToken(), Tags: []string{TagAccessToken}}) + allRules = append(allRules, Rule{Rule: *rules.SlackWebHookUrl(), Tags: []string{TagWebhook}}) allRules = append(allRules, Rule{Rule: *rules.StripeAccessToken(), Tags: []string{TagAccessToken}}) allRules = append(allRules, Rule{Rule: *rules.SquareAccessToken(), Tags: []string{TagAccessToken}}) allRules = append(allRules, Rule{Rule: *rules.SquareSpaceAccessToken(), Tags: []string{TagAccessToken}}) @@ -368,6 +378,7 @@ func loadAllRules() ([]Rule, error) { allRules = append(allRules, Rule{Rule: *rules.YandexAWSAccessToken(), Tags: []string{TagAccessToken}}) allRules = append(allRules, Rule{Rule: *rules.YandexAccessToken(), Tags: []string{TagAccessToken}}) allRules = append(allRules, Rule{Rule: *rules.ZendeskSecretKey(), Tags: []string{TagSecretKey}}) + allRules = append(allRules, Rule{Rule: *internalRules.AuthenticatedURL(), Tags: []string{TagSensitiveUrl}}) return allRules, nil } diff --git a/secrets/secrets_test.go b/secrets/secrets_test.go index 4fd2a451..7d662320 100644 --- a/secrets/secrets_test.go +++ b/secrets/secrets_test.go @@ -2,8 +2,11 @@ package secrets import ( "fmt" + "sync" "testing" + "github.com/checkmarx/2ms/plugins" + "github.com/checkmarx/2ms/reporting" "github.com/zricethezav/gitleaks/v8/config" ) @@ -354,3 +357,83 @@ func createRules(ruleIDs ...string) map[string]config.Rule { } return rules } + +func TestSecrets(t *testing.T) { + secrets := []struct { + Content string + Name string + ShouldFind bool + }{ + { + Content: "", + Name: "empty", + ShouldFind: false, + }, + { + Content: "mongodb+srv://radar:mytoken@io.dbb.mongodb.net/?retryWrites=true&w=majority", + Name: "Authenticated URL", + ShouldFind: true, + }, + { + Content: "--output=https://elastic:bF21iC0bfTVXo3qhpJqTGs78@c22f5bc9787c4c268d3b069ad866bdc2.eu-central-1.aws.cloud.es.io:9243/tfs", + Name: "Authenticated URL", + ShouldFind: true, + }, + { + Content: "https://abc:123@google.com", + Name: "Basic Authenticated URL", + ShouldFind: true, + }, + { + Content: "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", + Name: "GitHub Personal Access Token", + ShouldFind: true, + }, + { + Content: "AKCp8jRRiQSAbghbuZmHKZcaKGEqbAASGH2SAb3rxXJQsSq9dGga8gFXe6aHpcRmzuHxN6oaT", + Name: "JFROG Secret without keyword", + // gitleaks is using "keywords" to identify the next literal after the keyword is a secret, + // that is why we are not expecting to find this secret + ShouldFind: false, + }, + { + Content: "--set imagePullSecretJfrog.password=AKCp8kqqfQbYifrbyvqusjyk6N3QKprXTv9B8HTitLbJzXT1kW7dDticXTsJpCrbqtizAwK4D \\", + Name: "JFROG Secret with keyword (real example)", + ShouldFind: true, + }, + { + Content: "--docker-password=AKCp8kqX8yeKBTqgm2XExHsp8yVdJn6SAgQmS1nJMfMDmzxEqX74rUGhedaWu7Eovid3VsMwb", + Name: "JFROG Secret as kubectl argument", + ShouldFind: true, + }, + } + + detector, err := Init([]string{}, []string{}) + if err != nil { + t.Fatal(err) + } + + for _, secret := range secrets { + name := secret.Name + if name == "" { + name = secret.Content + } + t.Run(name, func(t *testing.T) { + fmt.Printf("Start test %s", name) + secretsChan := make(chan reporting.Secret, 1) + wg := &sync.WaitGroup{} + wg.Add(1) + detector.Detect(plugins.Item{Content: secret.Content}, secretsChan, wg, nil) + close(secretsChan) + + s := <-secretsChan + if s.Value == "" && secret.ShouldFind { + t.Errorf("secret \"%s\" not found", secret.Name) + } + if s.Value != "" && !secret.ShouldFind { + t.Errorf("should not find") + } + }) + } + +}