From 3077215eef6e8c60bd0a857c7b50bc1c23bcf998 Mon Sep 17 00:00:00 2001 From: Richard Gomez <32133502+rgmz@users.noreply.github.com> Date: Thu, 7 Nov 2024 11:18:21 -0500 Subject: [PATCH] fix(mongodb): ignore invalid URLs (#3440) --- pkg/detectors/mongodb/mongodb.go | 93 +++++---- .../mongodb/mongodb_integration_test.go | 153 +++++++++++++++ pkg/detectors/mongodb/mongodb_test.go | 185 ++---------------- 3 files changed, 221 insertions(+), 210 deletions(-) diff --git a/pkg/detectors/mongodb/mongodb.go b/pkg/detectors/mongodb/mongodb.go index 5a4b0cde5e0f..f6e77bac1cce 100644 --- a/pkg/detectors/mongodb/mongodb.go +++ b/pkg/detectors/mongodb/mongodb.go @@ -7,15 +7,15 @@ import ( "strings" "time" - regexp "github.com/wasilibs/go-re2" + logContext "github.com/trufflesecurity/trufflehog/v3/pkg/context" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" + regexp "github.com/wasilibs/go-re2" "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" "go.mongodb.org/mongo-driver/mongo/readpref" "go.mongodb.org/mongo-driver/x/mongo/driver/auth" - - "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) type Scanner struct { @@ -42,11 +42,11 @@ func (s Scanner) Keywords() []string { // FromData will find and optionally verify MongoDB secrets in a given set of bytes. func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { + logger := logContext.AddLogger(ctx).Logger().WithName("mongodb") dataStr := string(data) - matches := connStrPat.FindAllStringSubmatch(dataStr, -1) - - for _, match := range matches { + uniqueMatches := make(map[string]string) + for _, match := range connStrPat.FindAllStringSubmatch(dataStr, -1) { // Filter out common placeholder passwords. password := match[3] if password == "" || placeholderPasswordPat.MatchString(password) { @@ -54,13 +54,40 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } // If the query string contains `&` the options will not be parsed. - resMatch := strings.Replace(strings.TrimSpace(match[1]), "&", "&", -1) - s1 := detectors.Result{ - DetectorType: detectorspb.DetectorType_MongoDB, - Raw: []byte(resMatch), + connStr := strings.Replace(strings.TrimSpace(match[1]), "&", "&", -1) + connUrl, err := url.Parse(connStr) + if err != nil { + logger.V(3).Info("Skipping invalid URL", "err", err) + continue } - s1.ExtraData = map[string]string{ - "rotation_guide": "https://howtorotate.com/docs/tutorials/mongo/", + + params := connUrl.Query() + for k, v := range connUrl.Query() { + if len(v) > 0 { + switch k { + case "tls": + if v[0] == "false" { + params.Set("tls", "false") + } else { + params.Set("tls", "true") + } + } + } + } + + connUrl.RawQuery = params.Encode() + connStr = connUrl.String() + + uniqueMatches[connStr] = password + } + + for connStr, password := range uniqueMatches { + r := detectors.Result{ + DetectorType: detectorspb.DetectorType_MongoDB, + Raw: []byte(connStr), + ExtraData: map[string]string{ + "rotation_guide": "https://howtorotate.com/docs/tutorials/mongo/", + }, } if verify { @@ -68,13 +95,15 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result if timeout == 0 { timeout = defaultTimeout } - isVerified, verificationErr := verifyUri(ctx, resMatch, timeout) - s1.Verified = isVerified - if !isErrDeterminate(verificationErr) { - s1.SetVerificationError(verificationErr, resMatch) + + isVerified, vErr := verifyUri(ctx, connStr, timeout) + r.Verified = isVerified + if isErrDeterminate(vErr) { + continue } + r.SetVerificationError(vErr, password) } - results = append(results, s1) + results = append(results, r) } return results, nil @@ -93,34 +122,12 @@ func isErrDeterminate(err error) bool { return errors.As(err, &authErr) } -func verifyUri(ctx context.Context, uri string, timeout time.Duration) (bool, error) { - parsed, err := url.Parse(uri) - if err != nil { - return false, err - } - - params := url.Values{} - for k, v := range parsed.Query() { - if len(v) > 0 { - switch k { - case "tls": - if v[0] == "false" { - params.Set("tls", "false") - } else { - params.Set("tls", "true") - } - } - } - } - parsed.RawQuery = params.Encode() - parsed.Path = "/" - uri = parsed.String() - +func verifyUri(ctx context.Context, connStr string, timeout time.Duration) (bool, error) { ctx, cancel := context.WithTimeout(ctx, timeout) defer cancel() - clientOptions := options.Client().SetTimeout(timeout).ApplyURI(uri) - if err = clientOptions.Validate(); err != nil { + clientOptions := options.Client().SetTimeout(timeout).ApplyURI(connStr) + if err := clientOptions.Validate(); err != nil { return false, err } diff --git a/pkg/detectors/mongodb/mongodb_integration_test.go b/pkg/detectors/mongodb/mongodb_integration_test.go index 9d355ffd6fef..17d219a2df88 100644 --- a/pkg/detectors/mongodb/mongodb_integration_test.go +++ b/pkg/detectors/mongodb/mongodb_integration_test.go @@ -180,3 +180,156 @@ func TestIntegrationMongoDB_FromChunk(t *testing.T) { }) } } + +func TestMongoDB_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + secret := testSecrets.MustGetField("MONGODB_URI") + inactiveSecret := testSecrets.MustGetField("MONGODB_INACTIVE_URI") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + wantVerificationErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a mongodb secret %s within", secret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_MongoDB, + Verified: true, + ExtraData: map[string]string{ + "rotation_guide": "https://howtorotate.com/docs/tutorials/mongo/", + }, + }, + }, + wantErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a mongodb secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_MongoDB, + Verified: false, + ExtraData: map[string]string{ + "rotation_guide": "https://howtorotate.com/docs/tutorials/mongo/", + }, + }, + }, + wantErr: false, + }, + { + name: "found, would be verified but for connection timeout", + s: Scanner{timeout: 1 * time.Microsecond}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a mongodb secret %s within", secret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_MongoDB, + Verified: false, + ExtraData: map[string]string{ + "rotation_guide": "https://howtorotate.com/docs/tutorials/mongo/", + }, + }, + }, + wantErr: false, + wantVerificationErr: true, + }, + { + name: "found, bad host", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a mongodb secret %s within", strings.ReplaceAll(secret, ".mongodb.net", ".mongodb.net.bad"))), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_MongoDB, + Verified: false, + ExtraData: map[string]string{ + "rotation_guide": "https://howtorotate.com/docs/tutorials/mongo/", + }, + }, + }, + wantErr: false, + wantVerificationErr: true, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("MongoDB.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + got[i].Raw = nil + if (got[i].VerificationError() != nil) != tt.wantVerificationErr { + t.Fatalf("wantVerificationErr = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError()) + } + } + ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "RawV2", "verificationError") + if diff := cmp.Diff(tt.want, got, ignoreOpts); diff != "" { + t.Errorf("MongoDB.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/mongodb/mongodb_test.go b/pkg/detectors/mongodb/mongodb_test.go index 3b0d72dfc508..9482fcca7cf3 100644 --- a/pkg/detectors/mongodb/mongodb_test.go +++ b/pkg/detectors/mongodb/mongodb_test.go @@ -1,22 +1,8 @@ -//go:build detectors -// +build detectors - package mongodb import ( "context" - "fmt" - "strings" "testing" - "time" - - "github.com/google/go-cmp/cmp" - "github.com/google/go-cmp/cmp/cmpopts" - - "github.com/trufflesecurity/trufflehog/v3/pkg/common" - "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" - - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) func TestMongoDB_Pattern(t *testing.T) { @@ -25,22 +11,26 @@ func TestMongoDB_Pattern(t *testing.T) { data string shouldMatch bool match string + skip bool }{ // True positives { name: "long_password", data: `mongodb://agenda-live:m21w7PFfRXQwfHZU1Fgx0rTX29ZBQaWMODLeAjsmyslVcMmcmy6CnLyu3byVDtdLYcCokze8lIE4KyAgSCGZxQ==@agenda-live.mongo.cosmos.azure.com:10255/?retryWrites=false&ssl=true&replicaSet=globaldb&maxIdleTimeMS=120000&appName=@agenda-live@`, shouldMatch: true, + match: `mongodb://agenda-live:m21w7PFfRXQwfHZU1Fgx0rTX29ZBQaWMODLeAjsmyslVcMmcmy6CnLyu3byVDtdLYcCokze8lIE4KyAgSCGZxQ==@agenda-live.mongo.cosmos.azure.com:10255/?appName=%40agenda-live%40&maxIdleTimeMS=120000&replicaSet=globaldb&retryWrites=false&ssl=true`, }, { name: "long_password2", data: `mongodb://csb0230eada-2354-4c73-b3e4-8a1aaa996894:AiNtEyASbdXR5neJmTStMzKGItX2xvKuyEkcy65rviKD0ggZR19E1iVFIJ5ZAIY1xvvAiS5tOXsmACDbKDJIhQ==@csb0230eada-2354-4c73-b3e4-8a1aaa996894.mongo.cosmos.cloud-hostname.com:10255/csb-db0230eada-2354-4c73-b3e4-8a1aaa996894?ssl=true&replicaSet=globaldb&retrywrites=false&maxIdleTimeMS=120000&appName=@csb0230eada-2354-4c73-b3e4-8a1aaa996894@`, shouldMatch: true, + match: `mongodb://csb0230eada-2354-4c73-b3e4-8a1aaa996894:AiNtEyASbdXR5neJmTStMzKGItX2xvKuyEkcy65rviKD0ggZR19E1iVFIJ5ZAIY1xvvAiS5tOXsmACDbKDJIhQ==@csb0230eada-2354-4c73-b3e4-8a1aaa996894.mongo.cosmos.cloud-hostname.com:10255/csb-db0230eada-2354-4c73-b3e4-8a1aaa996894?appName=%40csb0230eada-2354-4c73-b3e4-8a1aaa996894%40&maxIdleTimeMS=120000&replicaSet=globaldb&retrywrites=false&ssl=true`, }, { name: "long_password3", data: `mongodb://amsdfasfsadfdfdfpshot:6xNRRsdfsdfafd9NodO8vAFFBEHidfdfdfa87QDKXdCMubACDbhfQH1g==@amssdfafdafdadbsnapshot.mongo.cosmos.azure.com:10255/?ssl=true&replicaSet=globaldb&retrywrites=false&maxIdleTimeMS=120000&appName=@amssadfasdfdbsnsdfadfapshot@`, shouldMatch: true, + match: `mongodb://amsdfasfsadfdfdfpshot:6xNRRsdfsdfafd9NodO8vAFFBEHidfdfdfa87QDKXdCMubACDbhfQH1g==@amssdfafdafdadbsnapshot.mongo.cosmos.azure.com:10255/?appName=%40amssadfasdfdbsnsdfadfapshot%40&maxIdleTimeMS=120000&replicaSet=globaldb&retrywrites=false&ssl=true`, }, { name: "single_host", @@ -86,6 +76,7 @@ func TestMongoDB_Pattern(t *testing.T) { name: "multiple_hosts+options", data: `mongodb://username:password@mongodb1.example.com:27317,mongodb2.example.com,mongodb2.example.com:270/?connectTimeoutMS=300000&replicaSet=mySet&authSource=aDifferentAuthDB`, shouldMatch: true, + match: `mongodb://username:password@mongodb1.example.com:27317,mongodb2.example.com,mongodb2.example.com:270/?authSource=aDifferentAuthDB&connectTimeoutMS=300000&replicaSet=mySet`, }, { name: "multiple_hosts2", @@ -109,10 +100,13 @@ func TestMongoDB_Pattern(t *testing.T) { shouldMatch: true, match: "mongodb://cefapp:MdTc8Kc8DzlTE1RUl1JVDGS4zw1U1t6145sPWqeStWA50xEUKPfUCGlnk3ACkfqH6qLAwpnm9awpY1m8dg0YlQ==@cefapp.documents.azure.com:10250/?ssl=true&sslverifycertificate=false", }, + // TODO: `%2Ftmp%2Fmongodb-27017.sock` fails with url.Parse. + // Then again, TruffleHog will never be able to verify a local socket on a remote machine. { name: "unix_socket", data: `mongodb://u%24ername:pa%24%24w%7B%7Drd@%2Ftmp%2Fmongodb-27017.sock/test`, shouldMatch: true, + skip: true, }, { name: "dashes", @@ -145,11 +139,13 @@ func TestMongoDB_Pattern(t *testing.T) { name: "docker_internal_host", data: `mongodb://username:password@host.docker.internal:27018/?authMechanism=PLAIN&tls=true&tlsCertificateKeyFile=/etc/certs/client.pem&tlsCaFile=/etc/certs/rootCA-cert.pem`, shouldMatch: true, + match: `mongodb://username:password@host.docker.internal:27018/?authMechanism=PLAIN&tls=true&tlsCaFile=%2Fetc%2Fcerts%2FrootCA-cert.pem&tlsCertificateKeyFile=%2Fetc%2Fcerts%2Fclient.pem`, }, { name: "options_authsource_external", data: `mongodb://AKIAAAAAAAAAAAA:t9t2mawssecretkey@localhost:27017/?authMechanism=MONGODB-AWS&authsource=$external`, shouldMatch: true, + match: `mongodb://AKIAAAAAAAAAAAA:t9t2mawssecretkey@localhost:27017/?authMechanism=MONGODB-AWS&authsource=%24external`, }, { name: "generic1", @@ -168,6 +164,11 @@ func TestMongoDB_Pattern(t *testing.T) { data: `mongodb://username:@mongodb0.example.com:27017/?replicaSet=myRepl`, shouldMatch: false, }, + { + name: "invalid_userinfo", + data: `mongodb+srv://:@myMongoCluster.mongocluster.cosmos.azure.com`, + shouldMatch: false, + }, { name: "placeholders_x+single_host", data: `mongodb://xxxx:xxxxx@xxxxxxx:3717/zkquant?replicaSet=mgset-3017917`, @@ -182,6 +183,9 @@ func TestMongoDB_Pattern(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { + if test.skip { + t.SkipNow() + } s := Scanner{} results, err := s.FromData(context.Background(), false, []byte(test.data)) @@ -213,156 +217,3 @@ func TestMongoDB_Pattern(t *testing.T) { }) } } - -func TestMongoDB_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } - secret := testSecrets.MustGetField("MONGODB_URI") - inactiveSecret := testSecrets.MustGetField("MONGODB_INACTIVE_URI") - - type args struct { - ctx context.Context - data []byte - verify bool - } - tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool - wantVerificationErr bool - }{ - { - name: "found, verified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a mongodb secret %s within", secret)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_MongoDB, - Verified: true, - ExtraData: map[string]string{ - "rotation_guide": "https://howtorotate.com/docs/tutorials/mongo/", - }, - }, - }, - wantErr: false, - }, - { - name: "found, unverified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a mongodb secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_MongoDB, - Verified: false, - ExtraData: map[string]string{ - "rotation_guide": "https://howtorotate.com/docs/tutorials/mongo/", - }, - }, - }, - wantErr: false, - }, - { - name: "found, would be verified but for connection timeout", - s: Scanner{timeout: 1 * time.Microsecond}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a mongodb secret %s within", secret)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_MongoDB, - Verified: false, - ExtraData: map[string]string{ - "rotation_guide": "https://howtorotate.com/docs/tutorials/mongo/", - }, - }, - }, - wantErr: false, - wantVerificationErr: true, - }, - { - name: "found, bad host", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a mongodb secret %s within", strings.ReplaceAll(secret, ".mongodb.net", ".mongodb.net.bad"))), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_MongoDB, - Verified: false, - ExtraData: map[string]string{ - "rotation_guide": "https://howtorotate.com/docs/tutorials/mongo/", - }, - }, - }, - wantErr: false, - wantVerificationErr: true, - }, - { - name: "not found", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("MongoDB.FromData() error = %v, wantErr %v", err, tt.wantErr) - return - } - for i := range got { - if len(got[i].Raw) == 0 { - t.Fatalf("no raw secret present: \n %+v", got[i]) - } - got[i].Raw = nil - if (got[i].VerificationError() != nil) != tt.wantVerificationErr { - t.Fatalf("wantVerificationErr = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError()) - } - } - ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "RawV2", "verificationError") - if diff := cmp.Diff(tt.want, got, ignoreOpts); diff != "" { - t.Errorf("MongoDB.FromData() %s diff: (-got +want)\n%s", tt.name, diff) - } - }) - } -} - -func BenchmarkFromData(benchmark *testing.B) { - ctx := context.Background() - s := Scanner{} - for name, data := range detectors.MustGetBenchmarkData() { - benchmark.Run(name, func(b *testing.B) { - b.ResetTimer() - for n := 0; n < b.N; n++ { - _, err := s.FromData(ctx, false, data) - if err != nil { - b.Fatal(err) - } - } - }) - } -}