feat: validate pairs of secrets (#210)

After adding the simple validation process on #206, I'm now adding a
validation process for cases where both _access key_ and _secret key_
are needed together.

For these cases, the engine will collect those secrets and after the
scan is finished, it will validate all the pairs.

cmd/main.go

@@ -141,7 +141,7 @@ func preRun(cmd *cobra.Command, args []string) error {
 
 	if validateVar {
 		channels.WaitGroup.Add(1)
-		go processValidation()
+		go processValidation(engine)
 	}
 
 	return nil

cmd/workers.go

@@ -32,13 +32,15 @@ func processSecrets() {
 	close(validationChan)
 }
 
-func processValidation() {
+func processValidation(engine *secrets.Engine) {
 	defer channels.WaitGroup.Done()
 
 	wgValidation := &sync.WaitGroup{}
 	for secret := range validationChan {
 		wgValidation.Add(1)
-		go secret.Validate(wgValidation)
+		go engine.RegisterForValidation(secret, wgValidation)
 	}
 	wgValidation.Wait()
+
+	engine.Validate()
 }

docs/list-of-rules.md

@@ -11,8 +11,8 @@ Here is a complete list of all the rules that are currently implemented.
 | age secret key | Age secret key | secret-key |  |
 | airtable-api-key | Airtable API Key | api-key |  |
 | algolia-api-key | Algolia API Key | api-key |  |
-| alibaba-access-key-id | Alibaba AccessKey ID | access-key,access-id |  |
-| alibaba-secret-key | Alibaba Secret Key | secret-key |  |
+| alibaba-access-key-id | Alibaba AccessKey ID | access-key,access-id | V |
+| alibaba-secret-key | Alibaba Secret Key | secret-key | V |
 | asana-client-id | Asana Client ID | client-id |  |
 | asana-client-secret | Asana Client Secret | client-secret |  |
 | atlassian-api-token | Atlassian API token | api-token |  |

secrets/alibaba.go

@@ -0,0 +1,89 @@
+package secrets
+
+import (
+	"crypto/hmac"
+	"crypto/sha1"
+	"encoding/base64"
+	"fmt"
+	"net/http"
+	"net/url"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/rs/zerolog/log"
+)
+
+// https://www.alibabacloud.com/help/en/sdk/alibaba-cloud-api-overview
+// https://www.alibabacloud.com/help/en/sdk/product-overview/rpc-mechanism#sectiondiv-y9b-x9s-wvp
+
+func validateAlibaba(secrets pairsByRuleId) {
+
+	accessKeys := secrets["alibaba-access-key-id"]
+	secretKeys := secrets["alibaba-secret-key"]
+
+	for _, accessKey := range accessKeys {
+		accessKey.ValidationStatus = Unknown
+
+		for _, secretKey := range secretKeys {
+			status, err := alibabaRequest(accessKey.Value, secretKey.Value)
+			if err != nil {
+				log.Warn().Err(err).Str("service", "alibaba").Msg("Failed to validate secret")
+			}
+
+			secretKey.ValidationStatus = status
+			if accessKey.ValidationStatus.CompareTo(status) == second {
+				accessKey.ValidationStatus = status
+			}
+		}
+	}
+}
+
+func alibabaRequest(accessKey, secretKey string) (validationResult, error) {
+	req, err := http.NewRequest("GET", "https://ecs.aliyuncs.com/", nil)
+	if err != nil {
+		return Unknown, err
+	}
+
+	// Workaround for gitleaks returns the key ends with "
+	// https://github.com/gitleaks/gitleaks/pull/1350
+	accessKey = strings.TrimSuffix(accessKey, "\"")
+	secretKey = strings.TrimSuffix(secretKey, "\"")
+
+	params := req.URL.Query()
+	params.Add("AccessKeyId", accessKey)
+	params.Add("Action", "DescribeRegions")
+	params.Add("SignatureMethod", "HMAC-SHA1")
+	params.Add("SignatureNonce", strconv.FormatInt(time.Now().UnixNano(), 10))
+	params.Add("SignatureVersion", "1.0")
+	params.Add("Timestamp", time.Now().UTC().Format(time.RFC3339))
+	params.Add("Version", "2014-05-26")
+
+	stringToSign := "GET&%2F&" + url.QueryEscape(params.Encode())
+	hmac := hmac.New(sha1.New, []byte(secretKey+"&"))
+	hmac.Write([]byte(stringToSign))
+	signature := base64.StdEncoding.EncodeToString(hmac.Sum(nil))
+
+	params.Add("Signature", signature)
+	req.URL.RawQuery = params.Encode()
+
+	client := &http.Client{}
+	resp, err := client.Do(req)
+	if err != nil {
+		return Unknown, err
+	}
+	log.Debug().Str("service", "alibaba").Int("status_code", resp.StatusCode)
+
+	// If the access key is invalid, the response will be 404
+	// If the secret key is invalid, the response will be 400 along with other signautre Errors
+	if resp.StatusCode == http.StatusNotFound || resp.StatusCode == http.StatusBadRequest {
+		return Revoked, nil
+	}
+
+	if resp.StatusCode == http.StatusOK {
+		return Valid, nil
+	}
+
+	err = fmt.Errorf("unexpected status code: %d", resp.StatusCode)
+	return Unknown, err
+}

secrets/engine.go

@@ -19,8 +19,9 @@ import (
 )
 
 type Engine struct {
-	rules    map[string]config.Rule
-	detector detect.Detector
+	rules     map[string]config.Rule
+	detector  detect.Detector
+	validator Validator
 }
 
 const customRegexRuleIdFormat = "custom-regex-%d"
@@ -52,8 +53,9 @@ func Init(engineConfig EngineConfig) (*Engine, error) {
 	detector.MaxTargetMegaBytes = engineConfig.MaxTargetMegabytes
 
 	return &Engine{
-		rules:    rulesToBeApplied,
-		detector: *detector,
+		rules:     rulesToBeApplied,
+		detector:  *detector,
+		validator: *NewValidator(),
 	}, nil
 }
 
@@ -100,6 +102,15 @@ func (s *Engine) AddRegexRules(patterns []string) error {
 	return nil
 }
 
+func (s *Engine) RegisterForValidation(secret *Secret, wg *sync.WaitGroup) {
+	defer wg.Done()
+	s.validator.RegisterForValidation(secret)
+}
+
+func (s *Engine) Validate() {
+	s.validator.Validate()
+}
+
 func getFindingId(item plugins.Item, finding report.Finding) string {
 	idParts := []string{item.ID, finding.RuleID, finding.Secret}
 	sha := sha1.Sum([]byte(strings.Join(idParts, "-")))

secrets/pairs.go

@@ -0,0 +1,64 @@
+package secrets
+
+import (
+	"sync"
+)
+
+type pairsByRuleId map[string][]*Secret
+type pairsBySource map[string]pairsByRuleId
+type pairsByGeneralKey map[string]pairsBySource
+
+type pairsCollector struct {
+	pairs pairsByGeneralKey
+}
+
+func newPairsCollector() *pairsCollector {
+	return &pairsCollector{pairs: make(pairsByGeneralKey)}
+}
+
+func (p *pairsCollector) addIfNeeded(secret *Secret) bool {
+	generalKey, ok := ruleToGeneralKey[secret.RuleID]
+	if !ok {
+		return false
+	}
+
+	if _, ok := p.pairs[generalKey]; !ok {
+		p.pairs[generalKey] = make(pairsBySource)
+	}
+	if _, ok := p.pairs[generalKey][secret.Source]; !ok {
+		p.pairs[generalKey][secret.Source] = make(pairsByRuleId)
+	}
+	if _, ok := p.pairs[generalKey][secret.Source][secret.RuleID]; !ok {
+		p.pairs[generalKey][secret.Source][secret.RuleID] = make([]*Secret, 0)
+	}
+
+	p.pairs[generalKey][secret.Source][secret.RuleID] = append(p.pairs[generalKey][secret.Source][secret.RuleID], secret)
+	return true
+}
+
+func (p *pairsCollector) validate(generalKey string, rulesById pairsByRuleId, wg *sync.WaitGroup) {
+	defer wg.Done()
+	generalKeyToValidation[generalKey](rulesById)
+}
+
+type pairsValidationFunc func(pairsByRuleId)
+
+var generalKeyToValidation = map[string]pairsValidationFunc{
+	"alibaba": validateAlibaba,
+}
+
+var generalKeyToRules = map[string][]string{
+	"alibaba": {"alibaba-access-key-id", "alibaba-secret-key"},
+}
+
+func generateRuleToGeneralKey() map[string]string {
+	ruleToGeneralKey := make(map[string]string)
+	for key, rules := range generalKeyToRules {
+		for _, rule := range rules {
+			ruleToGeneralKey[rule] = key
+		}
+	}
+	return ruleToGeneralKey
+}
+
+var ruleToGeneralKey = generateRuleToGeneralKey()

secrets/secret.go

@@ -1,20 +1,7 @@
 package secrets
 
-import (
-	"fmt"
-	"net/http"
-	"sync"
-
-	"github.com/rs/zerolog/log"
-)
-
-type ValidationResult string
-
-const (
-	Valid   ValidationResult = "Valid"
-	Revoked ValidationResult = "Revoked"
-	Unknown ValidationResult = "Unknown"
-)
+// TODO: rename package to engine and move secrets into subpackage
+// Then move the validators into a subpackage too
 
 type Secret struct {
 	ID               string           `json:"id"`
@@ -25,49 +12,16 @@ type Secret struct {
 	StartColumn      int              `json:"startColumn"`
 	EndColumn        int              `json:"endColumn"`
 	Value            string           `json:"value"`
-	ValidationStatus ValidationResult `json:"validationStatus,omitempty"`
-}
-
-type validationFunc = func(*Secret) ValidationResult
-
-var ruleIDToFunction = map[string]validationFunc{
-	"github-fine-grained-pat": validateGithub,
-	"github-pat":              validateGithub,
+	ValidationStatus validationResult `json:"validationStatus,omitempty"`
 }
 
 func isCanValidateRule(ruleID string) bool {
-	_, ok := ruleIDToFunction[ruleID]
-	return ok
-}
-
-func (s *Secret) Validate(wg *sync.WaitGroup) {
-	defer wg.Done()
-	if f, ok := ruleIDToFunction[s.RuleID]; ok {
-		s.ValidationStatus = f(s)
-	} else {
-		s.ValidationStatus = Unknown
+	if _, ok := ruleIDToFunction[ruleID]; ok {
+		return true
 	}
-}
-
-func validateGithub(s *Secret) ValidationResult {
-	const githubURL = "https://api.github.com/"
-
-	req, err := http.NewRequest("GET", githubURL, nil)
-	if err != nil {
-		log.Warn().Err(err).Msg("Failed to validate secret")
-		return Unknown
+	if _, ok := ruleToGeneralKey[ruleID]; ok {
+		return true
 	}
-	req.Header.Set("Authorization", fmt.Sprintf("token %s", s.Value))
 
-	client := &http.Client{}
-	resp, err := client.Do(req)
-	if err != nil {
-		log.Warn().Err(err).Msg("Failed to validate secret")
-		return Unknown
-	}
-
-	if resp.StatusCode == http.StatusOK {
-		return Valid
-	}
-	return Revoked
+	return false
 }