From 70662ce055b5a0cb6b752c0ae9ad27a838061b62 Mon Sep 17 00:00:00 2001 From: Harshvardhan Karn Date: Thu, 11 Jul 2024 08:14:28 +0000 Subject: [PATCH] update config and remove dead code --- config.yaml | 468 +--------------------------------------------------- main.go | 93 ----------- 2 files changed, 3 insertions(+), 558 deletions(-) diff --git a/config.yaml b/config.yaml index 3a87209..98a58f3 100644 --- a/config.yaml +++ b/config.yaml @@ -1,466 +1,4 @@ # Secret Scanner Configuration File -exclude_extensions: [ ".exe", ".jpg", ".jpeg", ".png", ".gif", ".bmp", ".tiff", ".tif", ".psd", ".xcf", ".zip", ".tar", ".tar.gz", ".ttf", ".lock", ".pem", ".so", ".jar", ".gz" ] -exclude_paths: ["/var/lib/docker", "/var/lib/containerd", "/dev", "/proc", "/usr/lib", "/sys", "/boot", "/run", ".home/kubernetes"] -max_file_size: 1073741824 - -signatures: -- part: 'extension' - match: '.pem' - name: 'Potential cryptographic private key' -- part: 'extension' - match: '.pkcs12' - name: 'Potential cryptographic key bundle' -- part: 'extension' - match: '.p12' - name: 'Potential cryptographic key bundle' -- part: 'extension' - match: '.pfx' - name: 'Potential cryptographic key bundle' -- part: 'extension' - match: '.asc' - name: 'Potential cryptographic key bundle' -- part: 'filename' - match: 'token' - name: 'Kubernetes service account token' -- part: 'filename' - match: 'otr.private_key' - name: 'Pidgin OTR private key' -- part: 'extension' - match: '.ovpn' - name: 'OpenVPN client configuration file' -- part: 'extension' - match: '.cscfg' - name: 'Azure service configuration schema file' -- part: 'extension' - match: '.rdp' - name: 'Remote Desktop connection file' -- part: 'extension' - match: '.mdf' - name: 'Microsoft SQL database file' -- part: 'extension' - match: '.sdf' - name: 'Microsoft SQL server compact database file' -- part: 'extension' - match: '.sqlite' - name: 'SQLite database file' -- part: 'extension' - match: '.sqlite3' - name: 'SQLite3 database file' -- part: 'extension' - match: '.bek' - name: 'Microsoft BitLocker recovery key file' -- part: 'extension' - match: '.tpm' - name: 'Microsoft BitLocker Trusted Platform Module password file' -- part: 'extension' - match: '.fve' - name: 'Windows BitLocker full volume encrypted data file' -- part: 'extension' - match: '.jks' - name: 'Java keystore file' -- part: 'extension' - match: '.psafe3' - name: 'Password Safe database file' -- part: 'filename' - match: 'secret_token.rb' - name: 'Ruby On Rails secret token configuration file' -- part: 'filename' - match: 'carrierwave.rb' - name: 'Carrierwave configuration file' -- part: 'filename' - match: 'database.yml' - name: 'Potential Ruby On Rails database configuration file' -- part: 'filename' - match: 'omniauth.rb' - name: 'OmniAuth configuration file' -- part: 'extension' - match: '.agilekeychain' - name: '1Password password manager database file' -- part: 'extension' - match: '.keychain' - name: 'Apple Keychain database file' -- part: 'extension' - match: '.pcap' - name: 'Network traffic capture file' -- part: 'extension' - match: '.gnucash' - name: 'GnuCash database file' -- part: 'filename' - match: 'jenkins.plugins.publish_over_ssh.BapSshPublisherPlugin.xml' - name: 'Jenkins publish over SSH plugin file' -- part: 'filename' - match: 'credentials.xml' - name: 'Potential Jenkins credentials file' -- part: 'extension' - match: '.kwallet' - name: 'KDE Wallet Manager database file' -- part: 'filename' - match: 'LocalSettings.php' - name: 'Potential MediaWiki configuration file' -- part: 'extension' - match: '.tblk' - name: 'Tunnelblick VPN configuration file' -- part: 'filename' - match: 'Favorites.plist' - name: 'Sequel Pro MySQL database manager bookmark file' -- part: 'filename' - match: 'configuration.user.xpl' - name: 'Little Snitch firewall configuration file' -- part: 'extension' - match: '.dayone' - name: 'Day One journal file' -- part: 'filename' - match: 'journal.txt' - name: 'Potential jrnl journal file' -- part: 'filename' - match: 'knife.rb' - name: 'Chef Knife configuration file' -- part: 'filename' - match: 'proftpdpasswd' - name: 'cPanel backup ProFTPd credentials file' -- part: 'filename' - match: 'robomongo.json' - name: 'Robomongo MongoDB manager configuration file' -- part: 'filename' - match: 'filezilla.xml' - name: 'FileZilla FTP configuration file' -- part: 'filename' - match: 'recentservers.xml' - name: 'FileZilla FTP recent servers file' -- part: 'filename' - match: 'ventrilo_srv.ini' - name: 'Ventrilo server configuration file' -- part: 'filename' - match: 'terraform.tfvars' - name: 'Terraform variable config file' -- part: 'filename' - match: '.exports' - name: 'Shell configuration file' -- part: 'filename' - match: '.functions' - name: 'Shell configuration file' -- part: 'filename' - match: '.extra' - name: 'Shell configuration file' - -- part: 'filename' - regex: '^.*_rsa$' - name: 'Private SSH key' -- part: 'filename' - regex: '^.*_dsa$' - name: 'Private SSH key' -- part: 'filename' - regex: '^.*_ed25519$' - name: 'Private SSH key' -- part: 'filename' - regex: '^.*_ecdsa$' - name: 'Private SSH key' -- part: 'path' - regex: '\.?ssh/config$' - name: 'SSH configuration file' -- part: 'extension' - regex: '^key(pair)?$' - name: 'Potential cryptographic private key' -- part: 'filename' - regex: '^\.?mysql_history$' - name: 'MySQL client command history file' -- part: 'filename' - regex: '^\.?psql_history$' - name: 'PostgreSQL client command history file' -- part: 'filename' - regex: '^\.?pgpass$' - name: 'PostgreSQL password file' -- part: 'filename' - regex: '^\.?irb_history$' - name: 'Ruby IRB console history file' -- part: 'path' - regex: '\.?purple/accounts\.xml$' - name: 'Pidgin chat client account configuration file' -- part: 'path' - regex: '\.?xchat2?/servlist_?\.conf$' - name: 'Hexchat/XChat IRC client server list configuration file' -- part: 'path' - regex: '\.?irssi/config$' - name: 'Irssi IRC client configuration file' -- part: 'path' - regex: '\.?recon-ng/keys\.db$' - name: 'Recon-ng web reconnaissance framework API key database' -- part: 'filename' - regex: '^\.?dbeaver-data-sources.xml$' - name: 'DBeaver SQL database manager configuration file' -- part: 'filename' - regex: '^\.?muttrc$' - name: 'Mutt e-mail client configuration file' -- part: 'filename' - regex: '^\.?s3cfg$' - name: 'S3cmd configuration file' -- part: 'path' - regex: '\.?aws/credentials$' - name: 'AWS CLI credentials file' -- part: 'filename' - regex: '^sftp-config(\.json)?$' - name: 'SFTP connection configuration file' -- part: 'filename' - regex: '^\.?trc$' - name: 'T command-line Twitter client configuration file' -- part: 'filename' - regex: 'config(\.inc)?\.php$' - name: 'PHP configuration file' -- part: 'extension' - regex: '^key(store|ring)$' - name: 'GNOME Keyring database file' -- part: 'extension' - regex: '^kdbx?$' - name: 'KeePass password manager database file' -- part: 'extension' - regex: '^sql(dump)?$' - name: 'SQL dump file' -- part: 'filename' - regex: '^\.?htpasswd$' - name: 'Apache htpasswd file' -- part: 'filename' - regex: '^(\.|_)?netrc$' - name: 'Configuration file for auto-login process' -- part: 'path' - regex: '\.?gem/credentials$' - name: 'Rubygems credentials file' -- part: 'filename' - regex: '^\.?tugboat$' - name: 'Tugboat DigitalOcean management tool configuration' -- part: 'path' - regex: 'doctl/config.yaml$' - name: 'DigitalOcean doctl command-line client configuration file' -- part: 'filename' - regex: '^\.?git-credentials$' - name: 'git-credential-store helper credentials file' -- part: 'path' - regex: 'config/hub$' - name: 'GitHub Hub command-line client configuration file' -- part: 'filename' - regex: '^\.?gitconfig$' - name: 'Git configuration file' -- part: 'path' - regex: '\.?chef/(.*)\.pem$' - name: 'Chef private key' -- part: 'filename' - regex: '^\.?dockercfg$' - name: 'Docker configuration file' -- part: 'filename' - regex: '^\.?npmrc$' - name: 'NPM configuration file' - -- part: 'contents' - regex: '(A3T[A-Z0-9]|AKIA|AGPA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16}' - name: 'AWS Access Key ID Value' -- part: 'contents' - regex: "((\\\"|'|`)?((?i)aws)?_?((?i)access)_?((?i)key)?_?((?i)id)?(\\\"|'|`)?(\\\\s{0,50})?(:|=>|=)(\\\\s{0,50})?(\\\"|'|`)?(A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16}(\\\"|'|`)?)" - regextype: 'large' - name: 'AWS Access Key ID' -- part: 'contents' - regex: "((\\\"|'|`)?((?i)aws)?_?((?i)account)_?((?i)id)?(\\\"|'|`)?(\\\\s{0,50})?(:|=>|=)(\\\\s{0,50})?(\\\"|'|`)?[0-9]{4}-?[0-9]{4}-?[0-9]{4}(\\\"|'|`)?)" - regextype: 'large' - name: 'AWS Account ID' -- part: 'contents' - regex: "((\\\"|'|`)?((?i)aws)?_?((?i)secret)_?((?i)access)?_?((?i)key)?_?((?i)id)?(\\\"|'|`)?(\\\\s{0,50})?(:|=>|=)(\\\\s{0,50})?(\\\"|'|`)?[A-Za-z0-9/+=]{40}(\\\"|'|`)?)" - regextype: 'large' - name: 'AWS Secret Access Key' -- part: 'contents' - regex: "((\\\"|'|`)?((?i)aws)?_?((?i)session)?_?((?i)token)?(\\\"|'|`)?(\\\\s{0,50})?(:|=>|=)(\\\\s{0,50})?(\\\"|'|`)?[A-Za-z0-9/+=]{100,400}(\\\"|'|`)?)" - regextype: 'large' - name: 'AWS Session Token' -- part: 'contents' - regex: "(?i)artifactory.{0,50}(\\\"|'|`)?[a-zA-Z0-9=]{112}(\\\"|'|`)?" - regextype: 'large' - name: 'Artifactory' -- part: 'contents' - regex: "(?i)codeclima.{0,50}(\\\"|'|`)?[0-9a-f]{64}(\\\"|'|`)?" - regextype: 'large' - name: 'CodeClimate' -- part: 'contents' - regex: 'EAACEdEose0cBA[0-9A-Za-z]+' - name: 'Facebook access token' -- part: 'contents' - regex: "((\\\"|'|`)?type(\\\"|'|`)?\\\\s{0,50}(:|=>|=)\\\\s{0,50}(\\\"|'|`)?service_account(\\\"|'|`)?,?)" - regextype: 'large' - name: 'Google (GCM) Service account' -- part: 'contents' - regex: '(?:r|s)k_(live|test)_[0-9a-zA-Z]{24}' - name: 'Stripe API key' -- part: 'contents' - regex: '[0-9]+-[0-9A-Za-z_]{32}\.apps\.googleusercontent\.com' - name: 'Google OAuth Key' -- part: 'contents' - regex: 'AIza[0-9A-Za-z\\-_]{35}' - name: 'Google Cloud API Key' -- part: 'contents' - regex: 'ya29\\.[0-9A-Za-z\\-_]+' - name: 'Google OAuth Access Token' -- part: 'contents' - regex: 'sk_[live|test]_[0-9a-z]{32}' - name: 'Picatic API key' -- part: 'contents' - regex: 'sq0atp-[0-9A-Za-z\-_]{22}' - name: 'Square Access Token' -- part: 'contents' - regex: 'sq0csp-[0-9A-Za-z\-_]{43}' - name: 'Square OAuth Secret' -- part: 'contents' - regex: 'access_token\$production\$[0-9a-z]{16}\$[0-9a-f]{32}' - name: 'PayPal/Braintree Access Token' -- part: 'contents' - regex: 'amzn\.mws\.[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}' - name: 'Amazon MWS Auth Token' -- part: 'contents' - regex: 'SK[0-9a-fA-F]{32}' - name: 'Twilo API Key' -- part: 'contents' - regex: 'SG\.[0-9A-Za-z\-_]{22}\.[0-9A-Za-z\-_]{43}' - name: 'SendGrid API Key' -- part: 'contents' - regex: 'key-[0-9a-zA-Z]{32}' - name: 'MailGun API Key' -- part: 'contents' - regex: '[0-9a-f]{32}-us[0-9]{12}' - name: 'MailChimp API Key' -- part: 'contents' - regex: "sshpass -p.*['|\\\"]" - regextype: 'large' - name: 'SSH Password' -- part: 'contents' - regex: '(https\\://outlook\\.office.com/webhook/[0-9a-f-]{36}\\@)' - name: 'Outlook team' -- part: 'contents' - regex: "(?i)sauce.{0,50}(\\\"|'|`)?[0-9a-f-]{36}(\\\"|'|`)?" - name: 'Sauce Token' -- part: 'contents' - regex: '(xox[pboa]-[0-9]{12}-[0-9]{12}-[0-9]{12}-[a-z0-9]{32})' - name: 'Slack Token' -- part: 'contents' - regex: 'https://hooks.slack.com/services/T[a-zA-Z0-9_]{8}/B[a-zA-Z0-9_]{8}/[a-zA-Z0-9_]{24}' - name: 'Slack Webhook' -- part: 'contents' - regex: "(?i)sonar.{0,50}(\\\"|'|`)?[0-9a-f]{40}(\\\"|'|`)?" - name: 'SonarQube Docs API Key' -- part: 'contents' - regex: "(?i)hockey.{0,50}(\\\"|'|`)?[0-9a-f]{32}(\\\"|'|`)?" - name: 'HockeyApp' -- part: 'contents' - regex: '([\w+]{1,24})(://)([^$<]{1})([^\s";]{1,}):([^$<]{1})([^\s";/]{1,})@[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-zA-Z0-9()]{1,24}([^\s]+)' - regextype: 'large' - name: 'Username and password in URI' -- part: 'contents' - regex: '(username|user)=?([^$<]{1})([^\s\\\";]{1,})(;|,|:)?(password|pwd|passwd)=' - regextype: 'large' - name: 'Username and password in file' -- part: 'contents' - regex: 'oy2[a-z0-9]{43}' - name: 'NuGet API Key' -- part: 'contents' - regex: "(?i)appid=(\\\"|'|`)?[0-9a-f]{32}(\\\"|'|`)?" - name: 'OpenWeather API Key' -- part: 'contents' - regex: 'hawk\.[0-9A-Za-z\-_]{20}\.[0-9A-Za-z\-_]{20}' - regextype: 'large' - name: 'StackHawk API Key' - -- part: 'extension' - match: '.ppk' - name: 'Potential PuTTYgen private key' -- part: 'filename' - match: 'heroku.json' - name: 'Heroku config file' -- part: 'extension' - match: '.sqldump' - name: 'SQL Data dump file' -- part: 'filename' - match: 'dump.sql' - name: 'MySQL dump w/ bcrypt hashes' -- part: 'filename' - match: 'id_rsa_pub' - name: 'Public ssh key' -- part: 'filename' - match: 'mongoid.yml' - name: 'Mongoid config file' -- part: 'filename' - match: 'salesforce.js' - name: 'Salesforce credentials in a nodejs project' -- part: 'extension' - match: '.netrc' - name: 'netrc with SMTP credentials' - -- part: 'filename' - regex: '.remote-sync.json$' - name: 'Created by remote-sync for Atom, contains FTP and/or SCP/SFTP/SSH server details and credentials' -- part: 'filename' - regex: '.esmtprc$' - name: 'esmtp configuration' -- part: 'filename' - regex: '^deployment-config.json?$' - name: 'Created by sftp-deployment for Atom, contains server details and credentials' -- part: 'filename' - regex: '.ftpconfig$' - name: 'Created by sftp-deployment for Atom, contains server details and credentials' - -- part: 'contents' - regex: '-----BEGIN (EC|RSA|DSA|OPENSSH|PGP) PRIVATE KEY' - name: 'Contains a private key' -- part: 'contents' - regex: 'define(.{0,20})?(DB_CHARSET|NONCE_SALT|LOGGED_IN_SALT|AUTH_SALT|NONCE_KEY|DB_HOST|DB_PASSWORD|AUTH_KEY|SECURE_AUTH_KEY|LOGGED_IN_KEY|DB_NAME|DB_USER)(.{0,20})?[''|"].{10,120}[''|"]' - regextype: 'large' - name: 'WP-Config' -- part: 'contents' - regex: '(?i)(aws_access_key_id|aws_secret_access_key)(.{0,20})?=.[0-9a-zA-Z\/+]{20,40}' - name: 'AWS cred file info' -- part: 'contents' - regex: '(?i)(facebook|fb)(.{0,20})?(?-i)[''\"][0-9a-f]{32}[''\"]' - name: 'Facebook Secret Key' -- part: 'contents' - regex: '(?i)(facebook|fb)(.{0,20})?[''\"][0-9]{13,17}[''\"]' - name: 'Facebook Client ID' -- part: 'contents' - regex: '(?i)twitter(.{0,20})?[''\"][0-9a-z]{35,44}[''\"]' - name: 'Twitter Secret Key' -- part: 'contents' - regex: '(?i)twitter(.{0,20})?[''\"][0-9a-z]{18,25}[''\"]' - name: 'Twitter Client ID' -- part: 'contents' - regex: '(?i)github(.{0,20})?(?-i)[''\"][0-9a-zA-Z]{35,40}[''\"]' - name: 'Github Key' -- part: 'contents' - regex: '(?i)heroku(.{0,20})?[''"][0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}[''"]' - name: 'Heroku API key' -- part: 'contents' - regex: '(?i)linkedin(.{0,20})?(?-i)[''\"][0-9a-z]{12}[''\"]' - name: 'Linkedin Client ID' -- part: 'contents' - regex: '(?i)linkedin(.{0,20})?[''\"][0-9a-z]{16}[''\"]' - name: 'LinkedIn Secret Key' - -- part: 'path' - regex: '\.?idea[\\\/]WebServers.xml$' - name: 'Created by Jetbrains IDEs, contains webserver credentials with encoded passwords (not encrypted!)' -- part: 'path' - regex: '\.?vscode[\\\/]sftp.json$' - name: 'Created by vscode-sftp for VSCode, contains SFTP/SSH server details and credentials' -- part: 'path' - regex: 'web[\\\/]ruby[\\\/]secrets.yml' - name: 'Ruby on rails secrets.yml file (contains passwords)' -- part: 'path' - regex: '\.?docker[\\\/]config.json$' - name: 'Docker registry authentication file' -- part: 'path' - regex: 'ruby[\\\/]config[\\\/]master.key$' - name: 'Rails master key (used for decrypting credentials.yml.enc for Rails 5.2+)' -- part: 'path' - regex: '\.?mozilla[\\\/]firefox[\\\/]logins.json$' - name: 'Firefox saved password collection (can be decrypted using keys4.db)' - -- part: 'filename' - match: 'wallet.dat' - name: 'Bitcoin Core wallet' -- part: 'filename' - match: 'onion_v3_private_key' - name: 'Private key for Bitcoin Core onion service' -- part: 'filename' - match: 'bitcoin.conf' - name: 'Bitcoin Core config' +exclude_extensions: [ ".exe", ".jpg", ".jpeg", ".png", ".gif", ".bmp", ".tiff", ".tif", ".psd", ".xcf", ".zip", ".tar", ".tar.gz", ".ttf", ".lock", ".so", ".gz" ] +exclude_paths: ["/var/lib/docker", "/var/lib/containerd", "/dev", "/proc", "/usr/lib", "/sys", "/boot", "/run", ".home/kubernetes", "/snap", "/usr/bin", "/usr/local"] +max_file_size: 1073741824 \ No newline at end of file diff --git a/main.go b/main.go index e33c628..e4bdb15 100644 --- a/main.go +++ b/main.go @@ -72,99 +72,6 @@ type SecretsWriter interface { AddSecret(output.SecretFound) } -func runOnce(ctx context.Context, filters config.Filters, format string) { - var result SecretsWriter - var err error - node_type := "" - node_id := "" - var nodeType scan.ScanType - - // Scan container image for secrets - if len(*session.Options.ImageName) > 0 { - node_type = "image" - node_id = *session.Options.ImageName - nodeType = scan.ImageScan - log.Infof("Scanning image %s for secrets...", *session.Options.ImageName) - result = &output.JSONImageSecretsOutput{ - ImageName: *session.Options.ImageName, - Secrets: []output.SecretFound{}, - } - } else if len(*session.Options.Local) > 0 { // Scan local directory for secrets - node_id = *session.Options.Local - nodeType = scan.DirScan - result = &output.JSONDirSecretsOutput{ - DirName: *session.Options.Local, - Secrets: []output.SecretFound{}, - } - } else if len(*session.Options.ContainerID) > 0 { // Scan existing container for secrets - node_type = "container_image" - node_id = *session.Options.ContainerID - nodeType = scan.ContainerScan - result = &output.JSONImageSecretsOutput{ - ContainerID: *session.Options.ContainerID, - Secrets: []output.SecretFound{}, - } - } - - scanCtx := tasks.ScanContext{ - Context: ctx, - IsAlive: atomic.Bool{}, - } - - scan.Scan(&scanCtx, nodeType, filters, "", node_id, "", func(sf output.SecretFound, s string) { - result.AddSecret(sf) - }) - - if result == nil { - log.Error("set either -local or -image-name flag") - return - } - - if len(*core.GetSession().Options.ConsoleURL) != 0 && len(*core.GetSession().Options.DeepfenceKey) != 0 { - pub, err := output.NewPublisher( - *core.GetSession().Options.ConsoleURL, - strconv.Itoa(*core.GetSession().Options.ConsolePort), - *core.GetSession().Options.DeepfenceKey, - ) - if err != nil { - log.Error(err.Error()) - } - - pub.SendReport(output.GetHostname(), *session.Options.ImageName, *session.Options.ContainerID, node_type) - scanId := pub.StartScan(node_id, node_type) - if len(scanId) == 0 { - scanId = fmt.Sprintf("%s-%d", node_id, time.Now().UnixMilli()) - } - pub.IngestSecretScanResults(scanId, result.GetSecrets()) - log.Info("scan id %s", scanId) - } - - counts := output.CountBySeverity(result.GetSecrets()) - log.Infof("result severity counts: %+v", counts) - - if format == core.JSONOutput { - err = result.WriteJSON() - if err != nil { - log.Fatal("main: error while writing secrets: %s", err) - } - } else { - fmt.Println("summary:") - fmt.Printf(" total=%d high=%d medium=%d low=%d\n", counts.Total, counts.High, counts.Medium, counts.Low) - err = result.WriteTable() - if err != nil { - log.Fatal("main: error while writing secrets: %s", err) - } - } - - output.FailOn( - counts, - *core.GetSession().Options.FailOnHighCount, - *core.GetSession().Options.FailOnMediumCount, - *core.GetSession().Options.FailOnLowCount, - *core.GetSession().Options.FailOnCount, - ) -} - func main() { log.SetOutput(os.Stderr) log.SetLevel(log.InfoLevel)