merging changes from master

cmd/main.go

@@ -2,10 +2,10 @@ package cmd
 
 import (
 	"fmt"
+	"log"
 	"path/filepath"
 	"regexp"
 	"strings"
-	"time"
 
 	"github.com/checkmarx/2ms/config"
 	"github.com/checkmarx/2ms/lib"
@@ -131,7 +131,7 @@ func Execute() error {
 	rootCmd.AddGroup(&cobra.Group{Title: group, ID: group})
 
 	for _, plugin := range allPlugins {
-		subCommand, err := plugin.DefineCommand(channels)
+		subCommand, err := plugin.DefineCommand(channels.Items, channels.Errors)
 		if err != nil {
 			return fmt.Errorf("error while defining command for plugin %s: %s", plugin.GetName(), err.Error())
 		}
@@ -187,22 +187,32 @@ func preRun(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
+	channels.WaitGroup.Add(1)
 	go func() {
-		for {
-			select {
-			case item := <-channels.Items:
-				report.TotalItemsScanned++
-				channels.WaitGroup.Add(1)
-				go secrets.Detect(item, secretsChan, channels.WaitGroup, ignoreVar)
-			case secret := <-secretsChan:
-				report.TotalSecretsFound++
-				report.Results[secret.ID] = append(report.Results[secret.ID], secret)
-			case err, ok := <-channels.Errors:
-				if !ok || ShowError("errors") {
-					return
-				}
-				errorChan <- err
-			}
+		defer channels.WaitGroup.Done()
+
+		wgItems := &sync.WaitGroup{}
+		for item := range channels.Items {
+			report.TotalItemsScanned++
+			wgItems.Add(1)
+			go secrets.Detect(item, secretsChan, wgItems, ignoreVar)
+		}
+		wgItems.Wait()
+		close(secretsChan)
+	}()
+
+	channels.WaitGroup.Add(1)
+	go func() {
+		defer channels.WaitGroup.Done()
+		for secret := range secretsChan {
+			report.TotalSecretsFound++
+			report.Results[secret.ID] = append(report.Results[secret.ID], secret)
+		}
+	}()
+
+	go func() {
+		for err := range channels.Errors {
+			log.Fatal().Msg(err.Error())
 		}
 	}()
 
@@ -214,15 +224,6 @@ func postRun(cmd *cobra.Command, args []string) error {
 
 	cfg := config.LoadConfig("2ms", Version)
 
-	// Wait for last secret to be added to report
-	time.Sleep(time.Millisecond * timeSleepInterval)
-
-	if len(errorChan) != 0 {
-		errorInChan := <-errorChan
-		close(errorChan)
-		return errorInChan
-	}
-
 	// -------------------------------------
 	// Show Report
 	if report.TotalItemsScanned > 0 {

go.mod

@@ -11,7 +11,7 @@ require (
 	github.com/spf13/pflag v1.0.5
 	github.com/spf13/viper v1.15.0
 	github.com/stretchr/testify v1.8.1
-	github.com/zricethezav/gitleaks/v8 v8.16.1
+	github.com/zricethezav/gitleaks/v8 v8.17.1-0.20230717122715-f0dcd4d9cfe9
 	golang.org/x/time v0.1.0
 	gopkg.in/yaml.v2 v2.4.0
 )

go.sum

@@ -227,8 +227,8 @@ github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-github.com/zricethezav/gitleaks/v8 v8.16.1 h1:Y+x4GNIZx70fPQBJseuZydecadlgbKUJgirNeySSZw8=
-github.com/zricethezav/gitleaks/v8 v8.16.1/go.mod h1:JzsbTtA88h1ioImguqG0BL2IV2JfblVO0qj/LSwgUKQ=
+github.com/zricethezav/gitleaks/v8 v8.17.1-0.20230717122715-f0dcd4d9cfe9 h1:gw0iPgtVuWBW1XQoZed9Y0rWaZ9la1qOooa6aRHsEFo=
+github.com/zricethezav/gitleaks/v8 v8.17.1-0.20230717122715-f0dcd4d9cfe9/go.mod h1:/0z7cslO7d0y29YRvHgYefeTu7UIqOmx95A4wMhcQtE=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=

plugins/confluence.go

@@ -46,7 +46,7 @@ func (p *ConfluencePlugin) GetAuthorizationHeader() string {
 	return lib.CreateBasicAuthCredentials(p)
 }
 
-func (p *ConfluencePlugin) DefineCommand(channels Channels) (*cobra.Command, error) {
+func (p *ConfluencePlugin) DefineCommand(items chan Item, errors chan error) (*cobra.Command, error) {
 	var confluenceCmd = &cobra.Command{
 		Use:   fmt.Sprintf("%s --%s URL", p.GetName(), argUrl),
 		Short: "Scan Confluence server",
@@ -67,11 +67,14 @@ func (p *ConfluencePlugin) DefineCommand(channels Channels) (*cobra.Command, err
 	confluenceCmd.Run = func(cmd *cobra.Command, args []string) {
 		err := p.initialize(cmd)
 		if err != nil {
-			channels.Errors <- fmt.Errorf("error while initializing confluence plugin: %w", err)
+			errors <- fmt.Errorf("error while initializing confluence plugin: %w", err)
 			return
 		}
 
-		p.getItems(channels.Items, channels.Errors, channels.WaitGroup)
+		wg := &sync.WaitGroup{}
+		p.getItems(items, errors, wg)
+		wg.Wait()
+		close(items)
 	}
 
 	return confluenceCmd, nil
@@ -90,18 +93,14 @@ func (p *ConfluencePlugin) initialize(cmd *cobra.Command) error {
 }
 
 func (p *ConfluencePlugin) getItems(items chan Item, errs chan error, wg *sync.WaitGroup) {
-	p.getSpacesItems(items, errs, wg)
-}
-
-func (p *ConfluencePlugin) getSpacesItems(items chan Item, errs chan error, wg *sync.WaitGroup) {
 	spaces, err := p.getSpaces()
 	if err != nil {
 		errs <- err
 	}
 
 	for _, space := range spaces {
-		go p.getSpaceItems(items, errs, wg, space)
 		wg.Add(1)
+		go p.getSpaceItems(items, errs, wg, space)
 	}
 }
 

plugins/discord.go

@@ -38,7 +38,7 @@ func (p *DiscordPlugin) GetName() string {
 	return "discord"
 }
 
-func (p *DiscordPlugin) DefineCommand(channels Channels) (*cobra.Command, error) {
+func (p *DiscordPlugin) DefineCommand(items chan Item, errors chan error) (*cobra.Command, error) {
 	var discordCmd = &cobra.Command{
 		Use:   fmt.Sprintf("%s --%s TOKEN --%s SERVER", p.GetName(), tokenFlag, serversFlag),
 		Short: "Scan Discord server",
@@ -63,11 +63,14 @@ func (p *DiscordPlugin) DefineCommand(channels Channels) (*cobra.Command, error)
 	discordCmd.Run = func(cmd *cobra.Command, args []string) {
 		err := p.initialize(cmd)
 		if err != nil {
-			channels.Errors <- fmt.Errorf("discord plugin initialization failed: %w", err)
+			errors <- fmt.Errorf("discord plugin initialization failed: %w", err)
 			return
 		}
 
-		p.getItems(channels.Items, channels.Errors, channels.WaitGroup)
+		wg := &sync.WaitGroup{}
+		p.getItems(items, errors, wg)
+		wg.Wait()
+		close(items)
 	}
 
 	return discordCmd, nil

plugins/filesystem.go

@@ -29,17 +29,17 @@ func (p *FileSystemPlugin) GetName() string {
 	return "filesystem"
 }
 
-func (p *FileSystemPlugin) DefineCommand(channels Channels) (*cobra.Command, error) {
+func (p *FileSystemPlugin) DefineCommand(items chan Item, errors chan error) (*cobra.Command, error) {
 	var cmd = &cobra.Command{
 		Use:   fmt.Sprintf("%s --%s PATH", p.GetName(), flagFolder),
 		Short: "Scan local folder",
 		Long:  "Scan local folder for sensitive information",
 		RunE: func(cmd *cobra.Command, args []string) error {
 			log.Info().Msg("Folder plugin started")
-			if err := p.getFiles(channels.Items, channels.Errors, channels.WaitGroup); err != nil {
-				return err
-			}
-			return nil
+			wg := &sync.WaitGroup{}
+			p.getFiles(items, errors, wg)
+			wg.Wait()
+			close(items)
 		},
 	}
 

plugins/git.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"os"
 	"strings"
+	"sync"
 
 	"github.com/gitleaks/go-gitdiff/gitdiff"
 	"github.com/rs/zerolog/log"
@@ -29,8 +30,12 @@ func (p *GitPlugin) GetName() string {
 	return "git"
 }
 
-func (p *GitPlugin) DefineCommand(channels Channels) (*cobra.Command, error) {
-	p.Channels = channels
+func (p *GitPlugin) DefineCommand(items chan Item, errors chan error) (*cobra.Command, error) {
+	p.Channels = Channels{
+		Items:     items,
+		Errors:    errors,
+		WaitGroup: &sync.WaitGroup{},
+	}
 
 	command := &cobra.Command{
 		Use:   fmt.Sprintf("%s <CLONED_REPO>", p.GetName()),
@@ -39,7 +44,9 @@ func (p *GitPlugin) DefineCommand(channels Channels) (*cobra.Command, error) {
 		Args:  cobra.MatchAll(cobra.ExactArgs(1), validGitRepoArgs),
 		Run: func(cmd *cobra.Command, args []string) {
 			log.Info().Msg("Git plugin started")
-			p.scanGit(args[0], p.buildScanOptions(), channels.Items, channels.Errors)
+			p.scanGit(args[0], p.buildScanOptions(), p.Channels.Items, p.Channels.Errors)
+			p.WaitGroup.Wait()
+			close(items)
 		},
 	}
 	flags := command.Flags()

plugins/paligo.go

@@ -7,6 +7,7 @@ import (
 	"net/http"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/checkmarx/2ms/lib"
@@ -54,8 +55,12 @@ func (p *PaligoPlugin) GetName() string {
 	return "paligo"
 }
 
-func (p *PaligoPlugin) DefineCommand(channels Channels) (*cobra.Command, error) {
-	p.Channels = channels
+func (p *PaligoPlugin) DefineCommand(items chan Item, errors chan error) (*cobra.Command, error) {
+	p.Channels = Channels{
+		Items:     items,
+		Errors:    errors,
+		WaitGroup: &sync.WaitGroup{},
+	}
 
 	command := &cobra.Command{
 		Use: fmt.Sprintf("%s --%s %s --%s %s --%s %s",
@@ -73,6 +78,8 @@ func (p *PaligoPlugin) DefineCommand(channels Channels) (*cobra.Command, error)
 			}
 			log.Info().Msg("Paligo plugin started")
 			p.getItems()
+			p.WaitGroup.Wait()
+			close(items)
 		},
 	}
 

plugins/plugins.go

@@ -27,5 +27,5 @@ type Channels struct {
 
 type IPlugin interface {
 	GetName() string
-	DefineCommand(channels Channels) (*cobra.Command, error)
+	DefineCommand(items chan Item, errors chan error) (*cobra.Command, error)
 }

plugins/slack.go

@@ -3,6 +3,7 @@ package plugins
 import (
 	"fmt"
 	"strconv"
+	"sync"
 	"time"
 
 	"github.com/rs/zerolog/log"
@@ -38,15 +39,21 @@ var (
 	messagesCountArg    int
 )
 
-func (p *SlackPlugin) DefineCommand(channels Channels) (*cobra.Command, error) {
-	p.Channels = channels
+func (p *SlackPlugin) DefineCommand(items chan Item, errors chan error) (*cobra.Command, error) {
+	p.Channels = Channels{
+		Items:     items,
+		Errors:    errors,
+		WaitGroup: &sync.WaitGroup{},
+	}
 
 	command := &cobra.Command{
 		Use:   fmt.Sprintf("%s --%s TOKEN --%s TEAM", p.GetName(), slackTokenFlag, slackTeamFlag),
 		Short: "Scan Slack team",
 		Long:  "Scan Slack team for sensitive information.",
 		Run: func(cmd *cobra.Command, args []string) {
 			p.getItems()
+			p.Channels.WaitGroup.Wait()
+			close(items)
 		},
 	}
 

secrets/rules/authenticated_url.go

@@ -0,0 +1,31 @@
+package rules
+
+import (
+	"regexp"
+
+	"github.com/zricethezav/gitleaks/v8/config"
+)
+
+func AuthenticatedURL() *config.Rule {
+	regex, _ := regexp.Compile(`:\/\/(.+:.+)?@`)
+	rule := config.Rule{
+		Description: "Identify username:password inside URLS",
+		RuleID:      "authenticated-url",
+		Regex:       regex,
+		Keywords:    []string{},
+		SecretGroup: 1,
+	}
+
+	tPositives := []string{
+		"mongodb+srv://radar:[email protected]/?retryWrites=true&w=majority",
+		"--output=https://elastic:bF21iC0bfTVXo3qhpJqTGs78@c22f5bc9787c4c268d3b069ad866bdc2.eu-central-1.aws.cloud.es.io:9243/tfs",
+		"https://abc:[email protected]",
+	}
+
+	fPositives := []string{
+		"https://google.com",
+		"https://google.com?user=abc&password=123",
+	}
+
+	return validate(rule, tPositives, fPositives)
+}

secrets/rules/rule.go

@@ -0,0 +1,37 @@
+package rules
+
+import (
+	"strings"
+
+	"github.com/rs/zerolog/log"
+	"github.com/zricethezav/gitleaks/v8/config"
+	"github.com/zricethezav/gitleaks/v8/detect"
+)
+
+// Copied from https://github.com/gitleaks/gitleaks/blob/463d24618fa42fc7629dc30c9744ebe36c5df1ab/cmd/generate/config/rules/rule.go
+func validate(r config.Rule, truePositives []string, falsePositives []string) *config.Rule {
+	// normalize keywords like in the config package
+	var keywords []string
+	for _, k := range r.Keywords {
+		keywords = append(keywords, strings.ToLower(k))
+	}
+	r.Keywords = keywords
+
+	rules := make(map[string]config.Rule)
+	rules[r.RuleID] = r
+	d := detect.NewDetector(config.Config{
+		Rules:    rules,
+		Keywords: keywords,
+	})
+	for _, tp := range truePositives {
+		if len(d.DetectString(tp)) != 1 {
+			log.Fatal().Msgf("Failed to validate. For rule ID [%s], true positive [%s] was not detected by regexp [%s]", r.RuleID, tp, r.Regex)
+		}
+	}
+	for _, fp := range falsePositives {
+		if len(d.DetectString(fp)) != 0 {
+			log.Fatal().Msgf("Failed to validate. For rule ID [%s], false positive [%s] was detected by regexp [%s]", r.RuleID, fp, r.Regex)
+		}
+	}
+	return &r
+}