Merge branch 'master' into add_test

README.md

@@ -103,7 +103,7 @@ We offer the following list of integrations in the form of plugins
 scans a [Confluence](https://www.atlassian.com/software/confluence) instance
 
 ```
-2ms confluence --url URL [flags]
+2ms confluence <URL> [flags]
 ```
 
 | Flag         | Value  | Default                        | Description                                                                      |
@@ -117,7 +117,7 @@ scans a [Confluence](https://www.atlassian.com/software/confluence) instance
 For example:
 
 ```
-2ms confluence --url https://checkmarx.atlassian.net/wiki --spaces secrets
+2ms confluence https://checkmarx.atlassian.net/wiki --spaces secrets
 ```
 
 - 💡 [The `secrets` Confluence site](https://checkmarx.atlassian.net/wiki/spaces/secrets) purposely created with plain example secrets as a test subject for this demo

cmd/main.go

@@ -11,7 +11,6 @@ import (
 	"github.com/checkmarx/2ms/lib"
 
 	"sync"
-	"time"
 
 	"github.com/checkmarx/2ms/plugins"
 	"github.com/checkmarx/2ms/reporting"
@@ -129,7 +128,7 @@ func Execute() {
 	rootCmd.AddGroup(&cobra.Group{Title: group, ID: group})
 
 	for _, plugin := range allPlugins {
-		subCommand, err := plugin.DefineCommand(channels)
+		subCommand, err := plugin.DefineCommand(channels.Items, channels.Errors)
 		if err != nil {
 			log.Fatal().Msg(fmt.Sprintf("error while defining command for plugin %s: %s", plugin.GetName(), err.Error()))
 		}
@@ -170,22 +169,32 @@ func preRun(cmd *cobra.Command, args []string) {
 		log.Fatal().Msg(err.Error())
 	}
 
+	channels.WaitGroup.Add(1)
 	go func() {
-		for {
-			select {
-			case item := <-channels.Items:
-				report.TotalItemsScanned++
-				channels.WaitGroup.Add(1)
-				go secrets.Detect(item, secretsChan, channels.WaitGroup, ignoreVar)
-			case secret := <-secretsChan:
-				report.TotalSecretsFound++
-				report.Results[secret.ID] = append(report.Results[secret.ID], secret)
-			case err, ok := <-channels.Errors:
-				if !ok {
-					return
-				}
-				log.Fatal().Msg(err.Error())
-			}
+		defer channels.WaitGroup.Done()
+
+		wgItems := &sync.WaitGroup{}
+		for item := range channels.Items {
+			report.TotalItemsScanned++
+			wgItems.Add(1)
+			go secrets.Detect(item, secretsChan, wgItems, ignoreVar)
+		}
+		wgItems.Wait()
+		close(secretsChan)
+	}()
+
+	channels.WaitGroup.Add(1)
+	go func() {
+		defer channels.WaitGroup.Done()
+		for secret := range secretsChan {
+			report.TotalSecretsFound++
+			report.Results[secret.ID] = append(report.Results[secret.ID], secret)
+		}
+	}()
+
+	go func() {
+		for err := range channels.Errors {
+			log.Fatal().Msg(err.Error())
 		}
 	}()
 }
@@ -195,9 +204,6 @@ func postRun(cmd *cobra.Command, args []string) {
 
 	cfg := config.LoadConfig("2ms", Version)
 
-	// Wait for last secret to be added to report
-	time.Sleep(time.Millisecond * timeSleepInterval)
-
 	// -------------------------------------
 	// Show Report
 	if report.TotalItemsScanned > 0 {

plugins/confluence.go

@@ -10,6 +10,8 @@ import (
 	"github.com/checkmarx/2ms/lib"
 	"github.com/rs/zerolog/log"
 	"github.com/spf13/cobra"
+
+	"net/url"
 )
 
 const (
@@ -46,40 +48,45 @@ func (p *ConfluencePlugin) GetAuthorizationHeader() string {
 	return lib.CreateBasicAuthCredentials(p)
 }
 
-func (p *ConfluencePlugin) DefineCommand(channels Channels) (*cobra.Command, error) {
+func isValidURL(cmd *cobra.Command, args []string) error {
+	urlStr := args[0]
+	parsedURL, err := url.Parse(urlStr)
+	if err != nil && parsedURL.Scheme != "https" {
+		return fmt.Errorf("invalid URL format")
+	}
+	return nil
+}
+
+func (p *ConfluencePlugin) DefineCommand(items chan Item, errors chan error) (*cobra.Command, error) {
 	var confluenceCmd = &cobra.Command{
-		Use:   fmt.Sprintf("%s --%s URL", p.GetName(), argUrl),
+		Use:   fmt.Sprintf("%s <URL>", p.GetName()),
 		Short: "Scan Confluence server",
 		Long:  "Scan Confluence server for sensitive information",
+		Args:  cobra.MatchAll(cobra.ExactArgs(1), isValidURL),
+		Run: func(cmd *cobra.Command, args []string) {
+			err := p.initialize(cmd, args[0])
+			if err != nil {
+				errors <- fmt.Errorf("error while initializing confluence plugin: %w", err)
+			}
+			wg := &sync.WaitGroup{}
+			p.getItems(items, errors, wg)
+			wg.Wait()
+			close(items)
+		},
 	}
 
 	flags := confluenceCmd.Flags()
-	flags.StringVar(&p.URL, argUrl, "", "Confluence server URL (example: https://company.atlassian.net/wiki) [required]")
 	flags.StringSliceVar(&p.Spaces, argSpaces, []string{}, "Confluence spaces: The names or IDs of the spaces to scan")
 	flags.StringVar(&p.Username, argUsername, "", "Confluence user name or email for authentication")
 	flags.StringVar(&p.Token, argToken, "", "The Confluence API token for authentication")
 	flags.BoolVar(&p.History, argHistory, false, "Scan pages history")
-	err := confluenceCmd.MarkFlagRequired(argUrl)
-	if err != nil {
-		return nil, fmt.Errorf("error while marking '%s' flag as required: %w", argUrl, err)
-	}
-
-	confluenceCmd.Run = func(cmd *cobra.Command, args []string) {
-		err := p.initialize(cmd)
-		if err != nil {
-			channels.Errors <- fmt.Errorf("error while initializing confluence plugin: %w", err)
-			return
-		}
-
-		p.getItems(channels.Items, channels.Errors, channels.WaitGroup)
-	}
 
 	return confluenceCmd, nil
 }
 
-func (p *ConfluencePlugin) initialize(cmd *cobra.Command) error {
+func (p *ConfluencePlugin) initialize(cmd *cobra.Command, urlArg string) error {
 
-	p.URL = strings.TrimRight(p.URL, "/")
+	p.URL = strings.TrimRight(urlArg, "/")
 
 	if p.Username == "" || p.Token == "" {
 		log.Warn().Msg("confluence credentials were not provided. The scan will be made anonymously only for the public pages")
@@ -90,18 +97,14 @@ func (p *ConfluencePlugin) initialize(cmd *cobra.Command) error {
 }
 
 func (p *ConfluencePlugin) getItems(items chan Item, errs chan error, wg *sync.WaitGroup) {
-	p.getSpacesItems(items, errs, wg)
-}
-
-func (p *ConfluencePlugin) getSpacesItems(items chan Item, errs chan error, wg *sync.WaitGroup) {
 	spaces, err := p.getSpaces()
 	if err != nil {
 		errs <- err
 	}
 
 	for _, space := range spaces {
-		go p.getSpaceItems(items, errs, wg, space)
 		wg.Add(1)
+		go p.getSpaceItems(items, errs, wg, space)
 	}
 }
 

plugins/discord.go

@@ -38,7 +38,7 @@ func (p *DiscordPlugin) GetName() string {
 	return "discord"
 }
 
-func (p *DiscordPlugin) DefineCommand(channels Channels) (*cobra.Command, error) {
+func (p *DiscordPlugin) DefineCommand(items chan Item, errors chan error) (*cobra.Command, error) {
 	var discordCmd = &cobra.Command{
 		Use:   fmt.Sprintf("%s --%s TOKEN --%s SERVER", p.GetName(), tokenFlag, serversFlag),
 		Short: "Scan Discord server",
@@ -63,11 +63,14 @@ func (p *DiscordPlugin) DefineCommand(channels Channels) (*cobra.Command, error)
 	discordCmd.Run = func(cmd *cobra.Command, args []string) {
 		err := p.initialize(cmd)
 		if err != nil {
-			channels.Errors <- fmt.Errorf("discord plugin initialization failed: %w", err)
+			errors <- fmt.Errorf("discord plugin initialization failed: %w", err)
 			return
 		}
 
-		p.getItems(channels.Items, channels.Errors, channels.WaitGroup)
+		wg := &sync.WaitGroup{}
+		p.getItems(items, errors, wg)
+		wg.Wait()
+		close(items)
 	}
 
 	return discordCmd, nil

plugins/filesystem.go

@@ -29,14 +29,18 @@ func (p *FileSystemPlugin) GetName() string {
 	return "filesystem"
 }
 
-func (p *FileSystemPlugin) DefineCommand(channels Channels) (*cobra.Command, error) {
+func (p *FileSystemPlugin) DefineCommand(items chan Item, errors chan error) (*cobra.Command, error) {
 	var cmd = &cobra.Command{
 		Use:   fmt.Sprintf("%s --%s PATH", p.GetName(), flagFolder),
 		Short: "Scan local folder",
 		Long:  "Scan local folder for sensitive information",
 		Run: func(cmd *cobra.Command, args []string) {
 			log.Info().Msg("Folder plugin started")
-			p.getFiles(channels.Items, channels.Errors, channels.WaitGroup)
+
+			wg := &sync.WaitGroup{}
+			p.getFiles(items, errors, wg)
+			wg.Wait()
+			close(items)
 		},
 	}
 

plugins/git.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"os"
 	"strings"
+	"sync"
 
 	"github.com/gitleaks/go-gitdiff/gitdiff"
 	"github.com/rs/zerolog/log"
@@ -29,8 +30,12 @@ func (p *GitPlugin) GetName() string {
 	return "git"
 }
 
-func (p *GitPlugin) DefineCommand(channels Channels) (*cobra.Command, error) {
-	p.Channels = channels
+func (p *GitPlugin) DefineCommand(items chan Item, errors chan error) (*cobra.Command, error) {
+	p.Channels = Channels{
+		Items:     items,
+		Errors:    errors,
+		WaitGroup: &sync.WaitGroup{},
+	}
 
 	command := &cobra.Command{
 		Use:   fmt.Sprintf("%s <CLONED_REPO>", p.GetName()),
@@ -39,7 +44,9 @@ func (p *GitPlugin) DefineCommand(channels Channels) (*cobra.Command, error) {
 		Args:  cobra.MatchAll(cobra.ExactArgs(1), validGitRepoArgs),
 		Run: func(cmd *cobra.Command, args []string) {
 			log.Info().Msg("Git plugin started")
-			p.scanGit(args[0], p.buildScanOptions(), channels.Items, channels.Errors)
+			p.scanGit(args[0], p.buildScanOptions(), p.Channels.Items, p.Channels.Errors)
+			p.WaitGroup.Wait()
+			close(items)
 		},
 	}
 	flags := command.Flags()

plugins/paligo.go

@@ -7,6 +7,7 @@ import (
 	"net/http"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/checkmarx/2ms/lib"
@@ -54,8 +55,12 @@ func (p *PaligoPlugin) GetName() string {
 	return "paligo"
 }
 
-func (p *PaligoPlugin) DefineCommand(channels Channels) (*cobra.Command, error) {
-	p.Channels = channels
+func (p *PaligoPlugin) DefineCommand(items chan Item, errors chan error) (*cobra.Command, error) {
+	p.Channels = Channels{
+		Items:     items,
+		Errors:    errors,
+		WaitGroup: &sync.WaitGroup{},
+	}
 
 	command := &cobra.Command{
 		Use: fmt.Sprintf("%s --%s %s --%s %s --%s %s",
@@ -73,6 +78,8 @@ func (p *PaligoPlugin) DefineCommand(channels Channels) (*cobra.Command, error)
 			}
 			log.Info().Msg("Paligo plugin started")
 			p.getItems()
+			p.WaitGroup.Wait()
+			close(items)
 		},
 	}
 

plugins/plugins.go

@@ -27,5 +27,5 @@ type Channels struct {
 
 type IPlugin interface {
 	GetName() string
-	DefineCommand(channels Channels) (*cobra.Command, error)
+	DefineCommand(items chan Item, errors chan error) (*cobra.Command, error)
 }

plugins/slack.go

@@ -3,6 +3,7 @@ package plugins
 import (
 	"fmt"
 	"strconv"
+	"sync"
 	"time"
 
 	"github.com/rs/zerolog/log"
@@ -38,15 +39,21 @@ var (
 	messagesCountArg    int
 )
 
-func (p *SlackPlugin) DefineCommand(channels Channels) (*cobra.Command, error) {
-	p.Channels = channels
+func (p *SlackPlugin) DefineCommand(items chan Item, errors chan error) (*cobra.Command, error) {
+	p.Channels = Channels{
+		Items:     items,
+		Errors:    errors,
+		WaitGroup: &sync.WaitGroup{},
+	}
 
 	command := &cobra.Command{
 		Use:   fmt.Sprintf("%s --%s TOKEN --%s TEAM", p.GetName(), slackTokenFlag, slackTeamFlag),
 		Short: "Scan Slack team",
 		Long:  "Scan Slack team for sensitive information.",
 		Run: func(cmd *cobra.Command, args []string) {
 			p.getItems()
+			p.Channels.WaitGroup.Wait()
+			close(items)
 		},
 	}
 

secrets/rules/authenticated_url.go

@@ -0,0 +1,31 @@
+package rules
+
+import (
+	"regexp"
+
+	"github.com/zricethezav/gitleaks/v8/config"
+)
+
+func AuthenticatedURL() *config.Rule {
+	regex, _ := regexp.Compile(`:\/\/(.+:.+)?@`)
+	rule := config.Rule{
+		Description: "Identify username:password inside URLS",
+		RuleID:      "authenticated-url",
+		Regex:       regex,
+		Keywords:    []string{},
+		SecretGroup: 1,
+	}
+
+	tPositives := []string{
+		"mongodb+srv://radar:[email protected]/?retryWrites=true&w=majority",
+		"--output=https://elastic:bF21iC0bfTVXo3qhpJqTGs78@c22f5bc9787c4c268d3b069ad866bdc2.eu-central-1.aws.cloud.es.io:9243/tfs",
+		"https://abc:[email protected]",
+	}
+
+	fPositives := []string{
+		"https://google.com",
+		"https://google.com?user=abc&password=123",
+	}
+
+	return validate(rule, tPositives, fPositives)
+}