From c086379d005566f837b03e55daf6873831278a07 Mon Sep 17 00:00:00 2001 From: Sean Kelly Date: Mon, 13 Nov 2023 16:57:15 -0600 Subject: [PATCH] Resolve software-issues-repo#55 --- .github/workflows/secrets-detection.yaml | 76 +++++++++++ .pre-commit-config.yaml | 35 ++--- .secrets.baseline | 164 +++++++++++++++++++++++ README.md | 32 ++++- 4 files changed, 288 insertions(+), 19 deletions(-) create mode 100644 .github/workflows/secrets-detection.yaml create mode 100644 .secrets.baseline diff --git a/.github/workflows/secrets-detection.yaml b/.github/workflows/secrets-detection.yaml new file mode 100644 index 0000000..3dafe5b --- /dev/null +++ b/.github/workflows/secrets-detection.yaml @@ -0,0 +1,76 @@ +name: Secret Detection Workflow +on: + push: + branches: + - main + pull_request: + branches: + - main + +jobs: + secret-detection: + runs-on: ubuntu-latest + steps: + - + name: Checkout code + uses: actions/checkout@v2 + - + name: Install necessary packages + run: | + pip install git+https://github.com/NASA-AMMOS/slim-detect-secrets.git@exp + pip install jq + + - + name: Create an initial .secrets.baseline if .secrets.baseline does not exist + run: | + if [ ! -f .secrets.baseline ]; then + # This generated baseline file will only be temporarily available on the GitHub side and will not appear in the user's local files. + # Scanning an empty folder to generate an initial .secrets.baseline without secrets in the results. + echo "⚠️ No existing .secrets.baseline file detected. Creating a new blank baseline file." + mkdir empty-dir + detect-secrets scan empty-dir > .secrets.baseline + echo "✅ Blank .secrets.baseline file created successfully." + rm -r empty-dir + else + echo "✅ Existing .secrets.baseline file detected. No new baseline file will be created." + fi + + - + name: Scan repository for secrets + run: | + # scripts to scan repository for new secrets + + # backup the list of known secrets + cp .secrets.baseline .secrets.new + + # find the secrets in the repository + detect-secrets scan --disable-plugin AbsolutePathDetectorExperimental --baseline .secrets.new \ + --exclude-files '\.secrets..*' \ + --exclude-files '\.git.*' \ + --exclude-files '\.mypy_cache' \ + --exclude-files '\.pytest_cache' \ + --exclude-files '\.tox' \ + --exclude-files '\.venv' \ + --exclude-files 'venv' \ + --exclude-files 'dist' \ + --exclude-files 'build' \ + --exclude-files '.*\.egg-info' + + # if there is any difference between the known and newly detected secrets, break the build + # Function to compare secrets without listing them + compare_secrets() { diff <(jq -r '.results | keys[] as $key | "\($key),\(.[$key] | .[] | .hashed_secret)"' "$1" | sort) <(jq -r '.results | keys[] as $key | "\($key),\(.[$key] | .[] | .hashed_secret)"' "$2" | sort) >/dev/null; } + + # Check if there's any difference between the known and newly detected secrets + if ! compare_secrets .secrets.baseline .secrets.new; then + echo "⚠️ Attention Required! ⚠️" >&2 + echo "New secrets have been detected in your recent commit. Due to security concerns, we cannot display detailed information here and we cannot proceed until this issue is resolved." >&2 + echo "" >&2 + echo "Please follow the steps below on your local machine to reveal and handle the secrets:" >&2 + echo "" >&2 + echo "1️⃣ Run the 'detect-secrets' tool on your local machine. This tool will identify and clean up the secrets. You can find detailed instructions at this link: https://nasa-ammos.github.io/slim/continuous-testing/starter-kits/#detect-secrets" >&2 + echo "" >&2 + echo "2️⃣ After cleaning up the secrets, commit your changes and re-push your update to the repository." >&2 + echo "" >&2 + echo "Your efforts to maintain the security of our codebase are greatly appreciated!" >&2 + exit 1 + fi diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c0a8e72..f7f1521 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v4.0.1 hooks: - id: trailing-whitespace exclude: REQUIREMENTS\.md$ @@ -11,17 +11,6 @@ repos: - id: debug-statements - id: check-yaml files: .*\.(yaml|yml)$ - - id: check-json - - id: pretty-format-json - -- repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.81.0 - hooks: - - id: terraform_fmt - - id: terraform_validate - # - id: terraform_tflint - # - id: terrascan - # - id: terraform_tfsec - repo: https://github.com/asottile/reorder_python_imports rev: v2.6.0 @@ -63,9 +52,21 @@ repos: stages: [push] pass_filenames: false -# #65: support for git-secrets -- repo: https://github.com/awslabs/git-secrets.git - # We have to use an sha here instead of a tag because of awslabs/git-secrets#182 - rev: b9e96b3212fa06aea65964ff0d5cda84ce935f38 +- repo: https://github.com/NASA-AMMOS/slim-detect-secrets + # using commit id for now, will change to tag when official version is released + rev: 91e097ad4559ae6ab785c883dc5ed989202c7fbe hooks: - - id: git-secrets + - id: detect-secrets + args: + - '--baseline' + - '.secrets.baseline' + - --exclude-files '\.secrets..*' + - --exclude-files '\.git.*' + - --exclude-files '\.mypy_cache' + - --exclude-files '\.pytest_cache' + - --exclude-files '\.tox' + - --exclude-files '\.venv' + - --exclude-files 'venv' + - --exclude-files 'dist' + - --exclude-files 'build' + - --exclude-files '.*\.egg-info' diff --git a/.secrets.baseline b/.secrets.baseline new file mode 100644 index 0000000..01faf0c --- /dev/null +++ b/.secrets.baseline @@ -0,0 +1,164 @@ +{ + "version": "1.4.0", + "plugins_used": [ + { + "name": "AbsolutePathDetectorExperimental" + }, + { + "name": "ArtifactoryDetector" + }, + { + "name": "AWSKeyDetector" + }, + { + "name": "AWSSensitiveInfoDetectorExperimental" + }, + { + "name": "AzureStorageKeyDetector" + }, + { + "name": "Base64HighEntropyString", + "limit": 4.5 + }, + { + "name": "BasicAuthDetector" + }, + { + "name": "CloudantDetector" + }, + { + "name": "DiscordBotTokenDetector" + }, + { + "name": "EmailAddressDetector" + }, + { + "name": "GitHubTokenDetector" + }, + { + "name": "HexHighEntropyString", + "limit": 3.0 + }, + { + "name": "IbmCloudIamDetector" + }, + { + "name": "IbmCosHmacDetector" + }, + { + "name": "IPPublicDetector" + }, + { + "name": "JwtTokenDetector" + }, + { + "name": "KeywordDetector", + "keyword_exclude": "" + }, + { + "name": "MailchimpDetector" + }, + { + "name": "NpmDetector" + }, + { + "name": "PrivateKeyDetector" + }, + { + "name": "SendGridDetector" + }, + { + "name": "SlackDetector" + }, + { + "name": "SoftlayerDetector" + }, + { + "name": "SquareOAuthDetector" + }, + { + "name": "StripeDetector" + }, + { + "name": "TwilioKeyDetector" + } + ], + "filters_used": [ + { + "path": "detect_secrets.filters.allowlist.is_line_allowlisted" + }, + { + "path": "detect_secrets.filters.common.is_baseline_file", + "filename": ".secrets.baseline" + }, + { + "path": "detect_secrets.filters.common.is_ignored_due_to_verification_policies", + "min_level": 2 + }, + { + "path": "detect_secrets.filters.heuristic.is_indirect_reference" + }, + { + "path": "detect_secrets.filters.heuristic.is_likely_id_string" + }, + { + "path": "detect_secrets.filters.heuristic.is_lock_file" + }, + { + "path": "detect_secrets.filters.heuristic.is_not_alphanumeric_string" + }, + { + "path": "detect_secrets.filters.heuristic.is_potential_uuid" + }, + { + "path": "detect_secrets.filters.heuristic.is_prefixed_with_dollar_sign" + }, + { + "path": "detect_secrets.filters.heuristic.is_sequential_string" + }, + { + "path": "detect_secrets.filters.heuristic.is_swagger_file" + }, + { + "path": "detect_secrets.filters.heuristic.is_templated_secret" + }, + { + "path": "detect_secrets.filters.regex.should_exclude_file", + "pattern": [ + "\\.secrets..*", + "\\.git.*", + "\\.mypy_cache", + "\\.pytest_cache", + "\\.tox", + "\\.venv", + "venv", + "dist", + "build", + ".*\\.egg-info" + ] + } + ], + "results": { + ".pre-commit-config.yaml": [ + { + "type": "Hex High Entropy String", + "filename": ".pre-commit-config.yaml", + "hashed_secret": "63076edad0be7557f5fbd9b60bc1ffa664fce207", + "is_verified": false, + "line_number": 57, + "is_secret": false + } + ], + "setup.cfg": [ + { + "type": "Email Address", + "filename": "setup.cfg", + "hashed_secret": "3a6d7aa49a8e4a2fe32a5cd0e53da9cb96bd8d29", + "is_verified": false, + "line_number": 22, + "is_secret": false + } + ] + }, + "generated_at": "2023-11-09T15:22:01Z" +} diff --git a/README.md b/README.md index 21bcf6d..682271b 100644 --- a/README.md +++ b/README.md @@ -68,14 +68,41 @@ Install in editable mode and with extra developer dependencies into your virtual pip install --editable '.[dev]' -Configure the `pre-commit` hooks: +Make a baseline for any secrets (email addresses, passwords, API keys, etc.) in the repository: + + detect-secrets scan . \ + --all-files \ + --disable-plugin AbsolutePathDetectorExperimental \ + --exclude-files '\.secrets..*' \ + --exclude-files '\.git.*' \ + --exclude-files '\.mypy_cache' \ + --exclude-files '\.pytest_cache' \ + --exclude-files '\.tox' \ + --exclude-files '\.venv' \ + --exclude-files 'venv' \ + --exclude-files 'dist' \ + --exclude-files 'build' \ + --exclude-files '.*\.egg-info' > .secrets.baseline + +Review the secrets to determine which should be allowed and which are false positives: + + detect-secrets audit .secrets.baseline + +Please remove any secrets that should not be seen by the public. You can then add the baseline file to the commit: + + git add .secrets.baseline + +Then, configure the `pre-commit` hooks: pre-commit install pre-commit install -t pre-push pre-commit install -t prepare-commit-msg pre-commit install -t commit-msg -These hooks check code formatting and also aborts commits that contain secrets such as passwords or API keys. However, a one time setup is required in your global Git configuration. See [the wiki entry on Git Secrets](https://github.com/NASA-PDS/nasa-pds.github.io/wiki/Git-and-Github-Guide#git-secrets) to learn how. +These hooks then will check for any future commits that might contain secrets. They also check code formatting, PEP8 compliance, type hints, etc. + +👉 **Note:** A one time setup is required both to support `detect-secrets` and in your global Git configuration. See [the wiki entry on Secrets](https://github.com/NASA-PDS/nasa-pds.github.io/wiki/Git-and-Github-Guide#detect-secrets) to learn how. + ### Packaging @@ -127,6 +154,7 @@ You should not use `print()`vin the purpose of logging information on the execut To make that work, start each Python file with: ```python +"""My module.""" import logging logger = logging.getLogger(__name__)