diff --git a/README.md b/README.md index e9c1f61b..61d3473c 100644 --- a/README.md +++ b/README.md @@ -103,7 +103,7 @@ We offer the following list of integrations in the form of plugins scans a [Confluence](https://www.atlassian.com/software/confluence) instance ``` -2ms confluence --url URL [flags] +2ms confluence [flags] ``` | Flag | Value | Default | Description | @@ -117,7 +117,7 @@ scans a [Confluence](https://www.atlassian.com/software/confluence) instance For example: ``` -2ms confluence --url https://checkmarx.atlassian.net/wiki --spaces secrets +2ms confluence https://checkmarx.atlassian.net/wiki --spaces secrets ``` - 💡 [The `secrets` Confluence site](https://checkmarx.atlassian.net/wiki/spaces/secrets) purposely created with plain example secrets as a test subject for this demo diff --git a/cmd/main.go b/cmd/main.go index 438f2f97..c357a1e9 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -11,7 +11,6 @@ import ( "github.com/checkmarx/2ms/lib" "sync" - "time" "github.com/checkmarx/2ms/plugins" "github.com/checkmarx/2ms/reporting" @@ -129,7 +128,7 @@ func Execute() { rootCmd.AddGroup(&cobra.Group{Title: group, ID: group}) for _, plugin := range allPlugins { - subCommand, err := plugin.DefineCommand(channels) + subCommand, err := plugin.DefineCommand(channels.Items, channels.Errors) if err != nil { log.Fatal().Msg(fmt.Sprintf("error while defining command for plugin %s: %s", plugin.GetName(), err.Error())) } @@ -170,22 +169,32 @@ func preRun(cmd *cobra.Command, args []string) { log.Fatal().Msg(err.Error()) } + channels.WaitGroup.Add(1) go func() { - for { - select { - case item := <-channels.Items: - report.TotalItemsScanned++ - channels.WaitGroup.Add(1) - go secrets.Detect(item, secretsChan, channels.WaitGroup, ignoreVar) - case secret := <-secretsChan: - report.TotalSecretsFound++ - report.Results[secret.ID] = append(report.Results[secret.ID], secret) - case err, ok := <-channels.Errors: - if !ok { - return - } - log.Fatal().Msg(err.Error()) - } + defer channels.WaitGroup.Done() + + wgItems := &sync.WaitGroup{} + for item := range channels.Items { + report.TotalItemsScanned++ + wgItems.Add(1) + go secrets.Detect(item, secretsChan, wgItems, ignoreVar) + } + wgItems.Wait() + close(secretsChan) + }() + + channels.WaitGroup.Add(1) + go func() { + defer channels.WaitGroup.Done() + for secret := range secretsChan { + report.TotalSecretsFound++ + report.Results[secret.ID] = append(report.Results[secret.ID], secret) + } + }() + + go func() { + for err := range channels.Errors { + log.Fatal().Msg(err.Error()) } }() } @@ -195,9 +204,6 @@ func postRun(cmd *cobra.Command, args []string) { cfg := config.LoadConfig("2ms", Version) - // Wait for last secret to be added to report - time.Sleep(time.Millisecond * timeSleepInterval) - // ------------------------------------- // Show Report if report.TotalItemsScanned > 0 { diff --git a/plugins/confluence.go b/plugins/confluence.go index 05f00fec..505bdf90 100644 --- a/plugins/confluence.go +++ b/plugins/confluence.go @@ -10,6 +10,8 @@ import ( "github.com/checkmarx/2ms/lib" "github.com/rs/zerolog/log" "github.com/spf13/cobra" + + "net/url" ) const ( @@ -46,40 +48,45 @@ func (p *ConfluencePlugin) GetAuthorizationHeader() string { return lib.CreateBasicAuthCredentials(p) } -func (p *ConfluencePlugin) DefineCommand(channels Channels) (*cobra.Command, error) { +func isValidURL(cmd *cobra.Command, args []string) error { + urlStr := args[0] + parsedURL, err := url.Parse(urlStr) + if err != nil && parsedURL.Scheme != "https" { + return fmt.Errorf("invalid URL format") + } + return nil +} + +func (p *ConfluencePlugin) DefineCommand(items chan Item, errors chan error) (*cobra.Command, error) { var confluenceCmd = &cobra.Command{ - Use: fmt.Sprintf("%s --%s URL", p.GetName(), argUrl), + Use: fmt.Sprintf("%s ", p.GetName()), Short: "Scan Confluence server", Long: "Scan Confluence server for sensitive information", + Args: cobra.MatchAll(cobra.ExactArgs(1), isValidURL), + Run: func(cmd *cobra.Command, args []string) { + err := p.initialize(cmd, args[0]) + if err != nil { + errors <- fmt.Errorf("error while initializing confluence plugin: %w", err) + } + wg := &sync.WaitGroup{} + p.getItems(items, errors, wg) + wg.Wait() + close(items) + }, } flags := confluenceCmd.Flags() - flags.StringVar(&p.URL, argUrl, "", "Confluence server URL (example: https://company.atlassian.net/wiki) [required]") flags.StringSliceVar(&p.Spaces, argSpaces, []string{}, "Confluence spaces: The names or IDs of the spaces to scan") flags.StringVar(&p.Username, argUsername, "", "Confluence user name or email for authentication") flags.StringVar(&p.Token, argToken, "", "The Confluence API token for authentication") flags.BoolVar(&p.History, argHistory, false, "Scan pages history") - err := confluenceCmd.MarkFlagRequired(argUrl) - if err != nil { - return nil, fmt.Errorf("error while marking '%s' flag as required: %w", argUrl, err) - } - - confluenceCmd.Run = func(cmd *cobra.Command, args []string) { - err := p.initialize(cmd) - if err != nil { - channels.Errors <- fmt.Errorf("error while initializing confluence plugin: %w", err) - return - } - - p.getItems(channels.Items, channels.Errors, channels.WaitGroup) - } return confluenceCmd, nil } -func (p *ConfluencePlugin) initialize(cmd *cobra.Command) error { +func (p *ConfluencePlugin) initialize(cmd *cobra.Command, urlArg string) error { - p.URL = strings.TrimRight(p.URL, "/") + p.URL = strings.TrimRight(urlArg, "/") if p.Username == "" || p.Token == "" { log.Warn().Msg("confluence credentials were not provided. The scan will be made anonymously only for the public pages") @@ -90,18 +97,14 @@ func (p *ConfluencePlugin) initialize(cmd *cobra.Command) error { } func (p *ConfluencePlugin) getItems(items chan Item, errs chan error, wg *sync.WaitGroup) { - p.getSpacesItems(items, errs, wg) -} - -func (p *ConfluencePlugin) getSpacesItems(items chan Item, errs chan error, wg *sync.WaitGroup) { spaces, err := p.getSpaces() if err != nil { errs <- err } for _, space := range spaces { - go p.getSpaceItems(items, errs, wg, space) wg.Add(1) + go p.getSpaceItems(items, errs, wg, space) } } diff --git a/plugins/discord.go b/plugins/discord.go index 6f6e5e12..ed49f591 100644 --- a/plugins/discord.go +++ b/plugins/discord.go @@ -38,7 +38,7 @@ func (p *DiscordPlugin) GetName() string { return "discord" } -func (p *DiscordPlugin) DefineCommand(channels Channels) (*cobra.Command, error) { +func (p *DiscordPlugin) DefineCommand(items chan Item, errors chan error) (*cobra.Command, error) { var discordCmd = &cobra.Command{ Use: fmt.Sprintf("%s --%s TOKEN --%s SERVER", p.GetName(), tokenFlag, serversFlag), Short: "Scan Discord server", @@ -63,11 +63,14 @@ func (p *DiscordPlugin) DefineCommand(channels Channels) (*cobra.Command, error) discordCmd.Run = func(cmd *cobra.Command, args []string) { err := p.initialize(cmd) if err != nil { - channels.Errors <- fmt.Errorf("discord plugin initialization failed: %w", err) + errors <- fmt.Errorf("discord plugin initialization failed: %w", err) return } - p.getItems(channels.Items, channels.Errors, channels.WaitGroup) + wg := &sync.WaitGroup{} + p.getItems(items, errors, wg) + wg.Wait() + close(items) } return discordCmd, nil diff --git a/plugins/filesystem.go b/plugins/filesystem.go index 520d69c1..c90e8744 100644 --- a/plugins/filesystem.go +++ b/plugins/filesystem.go @@ -29,14 +29,18 @@ func (p *FileSystemPlugin) GetName() string { return "filesystem" } -func (p *FileSystemPlugin) DefineCommand(channels Channels) (*cobra.Command, error) { +func (p *FileSystemPlugin) DefineCommand(items chan Item, errors chan error) (*cobra.Command, error) { var cmd = &cobra.Command{ Use: fmt.Sprintf("%s --%s PATH", p.GetName(), flagFolder), Short: "Scan local folder", Long: "Scan local folder for sensitive information", Run: func(cmd *cobra.Command, args []string) { log.Info().Msg("Folder plugin started") - p.getFiles(channels.Items, channels.Errors, channels.WaitGroup) + + wg := &sync.WaitGroup{} + p.getFiles(items, errors, wg) + wg.Wait() + close(items) }, } diff --git a/plugins/git.go b/plugins/git.go index 554f373a..a8f20d2a 100644 --- a/plugins/git.go +++ b/plugins/git.go @@ -4,6 +4,7 @@ import ( "fmt" "os" "strings" + "sync" "github.com/gitleaks/go-gitdiff/gitdiff" "github.com/rs/zerolog/log" @@ -29,8 +30,12 @@ func (p *GitPlugin) GetName() string { return "git" } -func (p *GitPlugin) DefineCommand(channels Channels) (*cobra.Command, error) { - p.Channels = channels +func (p *GitPlugin) DefineCommand(items chan Item, errors chan error) (*cobra.Command, error) { + p.Channels = Channels{ + Items: items, + Errors: errors, + WaitGroup: &sync.WaitGroup{}, + } command := &cobra.Command{ Use: fmt.Sprintf("%s ", p.GetName()), @@ -39,7 +44,9 @@ func (p *GitPlugin) DefineCommand(channels Channels) (*cobra.Command, error) { Args: cobra.MatchAll(cobra.ExactArgs(1), validGitRepoArgs), Run: func(cmd *cobra.Command, args []string) { log.Info().Msg("Git plugin started") - p.scanGit(args[0], p.buildScanOptions(), channels.Items, channels.Errors) + p.scanGit(args[0], p.buildScanOptions(), p.Channels.Items, p.Channels.Errors) + p.WaitGroup.Wait() + close(items) }, } flags := command.Flags() diff --git a/plugins/paligo.go b/plugins/paligo.go index 3f2a28d1..bbeaafa6 100644 --- a/plugins/paligo.go +++ b/plugins/paligo.go @@ -7,6 +7,7 @@ import ( "net/http" "strconv" "strings" + "sync" "time" "github.com/checkmarx/2ms/lib" @@ -54,8 +55,12 @@ func (p *PaligoPlugin) GetName() string { return "paligo" } -func (p *PaligoPlugin) DefineCommand(channels Channels) (*cobra.Command, error) { - p.Channels = channels +func (p *PaligoPlugin) DefineCommand(items chan Item, errors chan error) (*cobra.Command, error) { + p.Channels = Channels{ + Items: items, + Errors: errors, + WaitGroup: &sync.WaitGroup{}, + } command := &cobra.Command{ Use: fmt.Sprintf("%s --%s %s --%s %s --%s %s", @@ -73,6 +78,8 @@ func (p *PaligoPlugin) DefineCommand(channels Channels) (*cobra.Command, error) } log.Info().Msg("Paligo plugin started") p.getItems() + p.WaitGroup.Wait() + close(items) }, } diff --git a/plugins/plugins.go b/plugins/plugins.go index a22e2f67..f8236614 100644 --- a/plugins/plugins.go +++ b/plugins/plugins.go @@ -27,5 +27,5 @@ type Channels struct { type IPlugin interface { GetName() string - DefineCommand(channels Channels) (*cobra.Command, error) + DefineCommand(items chan Item, errors chan error) (*cobra.Command, error) } diff --git a/plugins/slack.go b/plugins/slack.go index d3862bd5..b88c9505 100644 --- a/plugins/slack.go +++ b/plugins/slack.go @@ -3,6 +3,7 @@ package plugins import ( "fmt" "strconv" + "sync" "time" "github.com/rs/zerolog/log" @@ -38,8 +39,12 @@ var ( messagesCountArg int ) -func (p *SlackPlugin) DefineCommand(channels Channels) (*cobra.Command, error) { - p.Channels = channels +func (p *SlackPlugin) DefineCommand(items chan Item, errors chan error) (*cobra.Command, error) { + p.Channels = Channels{ + Items: items, + Errors: errors, + WaitGroup: &sync.WaitGroup{}, + } command := &cobra.Command{ Use: fmt.Sprintf("%s --%s TOKEN --%s TEAM", p.GetName(), slackTokenFlag, slackTeamFlag), @@ -47,6 +52,8 @@ func (p *SlackPlugin) DefineCommand(channels Channels) (*cobra.Command, error) { Long: "Scan Slack team for sensitive information.", Run: func(cmd *cobra.Command, args []string) { p.getItems() + p.Channels.WaitGroup.Wait() + close(items) }, } diff --git a/secrets/rules/authenticated_url.go b/secrets/rules/authenticated_url.go new file mode 100644 index 00000000..999533f1 --- /dev/null +++ b/secrets/rules/authenticated_url.go @@ -0,0 +1,31 @@ +package rules + +import ( + "regexp" + + "github.com/zricethezav/gitleaks/v8/config" +) + +func AuthenticatedURL() *config.Rule { + regex, _ := regexp.Compile(`:\/\/(.+:.+)?@`) + rule := config.Rule{ + Description: "Identify username:password inside URLS", + RuleID: "authenticated-url", + Regex: regex, + Keywords: []string{}, + SecretGroup: 1, + } + + tPositives := []string{ + "mongodb+srv://radar:mytoken@io.dbb.mongodb.net/?retryWrites=true&w=majority", + "--output=https://elastic:bF21iC0bfTVXo3qhpJqTGs78@c22f5bc9787c4c268d3b069ad866bdc2.eu-central-1.aws.cloud.es.io:9243/tfs", + "https://abc:123@google.com", + } + + fPositives := []string{ + "https://google.com", + "https://google.com?user=abc&password=123", + } + + return validate(rule, tPositives, fPositives) +} diff --git a/secrets/rules/rule.go b/secrets/rules/rule.go new file mode 100644 index 00000000..59d4b36e --- /dev/null +++ b/secrets/rules/rule.go @@ -0,0 +1,37 @@ +package rules + +import ( + "strings" + + "github.com/rs/zerolog/log" + "github.com/zricethezav/gitleaks/v8/config" + "github.com/zricethezav/gitleaks/v8/detect" +) + +// Copied from https://github.com/gitleaks/gitleaks/blob/463d24618fa42fc7629dc30c9744ebe36c5df1ab/cmd/generate/config/rules/rule.go +func validate(r config.Rule, truePositives []string, falsePositives []string) *config.Rule { + // normalize keywords like in the config package + var keywords []string + for _, k := range r.Keywords { + keywords = append(keywords, strings.ToLower(k)) + } + r.Keywords = keywords + + rules := make(map[string]config.Rule) + rules[r.RuleID] = r + d := detect.NewDetector(config.Config{ + Rules: rules, + Keywords: keywords, + }) + for _, tp := range truePositives { + if len(d.DetectString(tp)) != 1 { + log.Fatal().Msgf("Failed to validate. For rule ID [%s], true positive [%s] was not detected by regexp [%s]", r.RuleID, tp, r.Regex) + } + } + for _, fp := range falsePositives { + if len(d.DetectString(fp)) != 0 { + log.Fatal().Msgf("Failed to validate. For rule ID [%s], false positive [%s] was detected by regexp [%s]", r.RuleID, fp, r.Regex) + } + } + return &r +} diff --git a/secrets/rules/rule_test.go b/secrets/rules/rule_test.go new file mode 100644 index 00000000..34e27da3 --- /dev/null +++ b/secrets/rules/rule_test.go @@ -0,0 +1,28 @@ +package rules_test + +import ( + "testing" + + "github.com/checkmarx/2ms/secrets/rules" + "github.com/zricethezav/gitleaks/v8/config" +) + +func Test2msRules(t *testing.T) { + t.Parallel() + + testsRules := []struct { + name string + validate func() *config.Rule + }{ + {name: "AuthenticatedURL", validate: rules.AuthenticatedURL}, + } + + for _, tRule := range testsRules { + testRule := tRule // fix for loop variable being captured by func literal + t.Run(testRule.name, func(t *testing.T) { + t.Parallel() + + testRule.validate() + }) + } +} diff --git a/secrets/secrets.go b/secrets/secrets.go index 3a6d720d..11c17a9d 100644 --- a/secrets/secrets.go +++ b/secrets/secrets.go @@ -11,6 +11,7 @@ import ( "github.com/checkmarx/2ms/plugins" "github.com/checkmarx/2ms/reporting" + internalRules "github.com/checkmarx/2ms/secrets/rules" "github.com/rs/zerolog/log" "github.com/spf13/cobra" "github.com/zricethezav/gitleaks/v8/cmd/generate/config/rules" @@ -377,6 +378,7 @@ func loadAllRules() ([]Rule, error) { allRules = append(allRules, Rule{Rule: *rules.YandexAWSAccessToken(), Tags: []string{TagAccessToken}}) allRules = append(allRules, Rule{Rule: *rules.YandexAccessToken(), Tags: []string{TagAccessToken}}) allRules = append(allRules, Rule{Rule: *rules.ZendeskSecretKey(), Tags: []string{TagSecretKey}}) + allRules = append(allRules, Rule{Rule: *internalRules.AuthenticatedURL(), Tags: []string{TagSensitiveUrl}}) return allRules, nil } diff --git a/secrets/secrets_test.go b/secrets/secrets_test.go index a9ec501c..7d662320 100644 --- a/secrets/secrets_test.go +++ b/secrets/secrets_test.go @@ -364,6 +364,31 @@ func TestSecrets(t *testing.T) { Name string ShouldFind bool }{ + { + Content: "", + Name: "empty", + ShouldFind: false, + }, + { + Content: "mongodb+srv://radar:mytoken@io.dbb.mongodb.net/?retryWrites=true&w=majority", + Name: "Authenticated URL", + ShouldFind: true, + }, + { + Content: "--output=https://elastic:bF21iC0bfTVXo3qhpJqTGs78@c22f5bc9787c4c268d3b069ad866bdc2.eu-central-1.aws.cloud.es.io:9243/tfs", + Name: "Authenticated URL", + ShouldFind: true, + }, + { + Content: "https://abc:123@google.com", + Name: "Basic Authenticated URL", + ShouldFind: true, + }, + { + Content: "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", + Name: "GitHub Personal Access Token", + ShouldFind: true, + }, { Content: "AKCp8jRRiQSAbghbuZmHKZcaKGEqbAASGH2SAb3rxXJQsSq9dGga8gFXe6aHpcRmzuHxN6oaT", Name: "JFROG Secret without keyword",