diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..50c8ea340 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,3 @@ +* +!.dockerignore +!Dockerfile diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 000000000..74f9834a3 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,14 @@ +root = true + +[*] +indent_style = space +indent_size = 2 +charset = utf-8 +trim_trailing_whitespace = true +insert_final_newline = true + +[{*.{sh,py,md},Dockerfile}] +indent_size = 4 + +[*.md] +trim_trailing_whitespace = false diff --git a/.github/ISSUE_TEMPLATE/bug_report_docker.md b/.github/ISSUE_TEMPLATE/bug_report_docker.md new file mode 100644 index 000000000..a47e30657 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report_docker.md @@ -0,0 +1,81 @@ +--- +name: Docker bug report +about: Create a bug report +labels: +- bug +- area/docker +--- + + + +### Describe the bug + + + + +### How can we reproduce it? + + + + +### Environment information + +* OS: + + + +* `docker info`: + +
command output + +```bash +INSERT_OUTPUT_HERE +``` + +
+ +* Docker image tag/git commit: + +* Tools versions. Don't forget to specify right tag in command - + `TAG=latest && docker run --entrypoint cat pre-commit:$TAG /usr/bin/tools_versions_info` + +```bash +INSERT_OUTPUT_HERE +``` + +* `.pre-commit-config.yaml`: + +
file content + +```bash +INSERT_FILE_CONTENT_HERE +``` + +
diff --git a/.github/ISSUE_TEMPLATE/bug_report_local_install.md b/.github/ISSUE_TEMPLATE/bug_report_local_install.md new file mode 100644 index 000000000..329a3ae88 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report_local_install.md @@ -0,0 +1,106 @@ +--- +name: Local installation bug report +about: Create a bug report +labels: +- bug +- area/local_installation +--- + + + +### Describe the bug + + + + +### How can we reproduce it? + + + + +### Environment information + +* OS: + + +* `uname -a` and/or `systeminfo | Select-String "^OS"` output: + +```bash +INSERT_OUTPUT_HERE +``` + + + +* Tools availability and versions: + + + +```bash +INSERT_TOOLS_VERSIONS_HERE +``` + + +* `.pre-commit-config.yaml`: + +
file content + +```bash +INSERT_FILE_CONTENT_HERE +``` + +
diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 000000000..d1b4b6424 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,29 @@ +--- +name: Feature request +about: Suggest an idea for this project +labels: +- feature +--- + + + +### What problem are you facing? + + + + +### How could pre-commit-terraform help solve your problem? + + diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 000000000..a7af18c5b --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,31 @@ + + +Put an `x` into the box if that apply: + +- [ ] This PR introduces breaking change. +- [ ] This PR fixes a bug. +- [ ] This PR adds new functionality. +- [ ] This PR enhances existing functionality. + +### Description of your changes + + + + + +### How has this code been tested + + diff --git a/.github/workflows/pre-commit.yaml b/.github/workflows/pre-commit.yaml new file mode 100644 index 000000000..9672e077f --- /dev/null +++ b/.github/workflows/pre-commit.yaml @@ -0,0 +1,38 @@ +name: Common issues check + +on: [pull_request] + +jobs: + pre-commit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - run: | + git fetch --no-tags --prune --depth=1 origin +refs/heads/*:refs/remotes/origin/* + + - name: Get changed files + id: file_changes + run: | + export DIFF=$(git diff --name-only origin/${{ github.base_ref }} ${{ github.sha }}) + echo "Diff between ${{ github.base_ref }} and ${{ github.sha }}" + echo "::set-output name=files::$( echo "$DIFF" | xargs echo )" + + - name: Install shfmt + run: | + curl -L "$(curl -s https://api.github.com/repos/mvdan/sh/releases/latest | grep -o -E -m 1 "https://.+?linux_amd64")" > shfmt \ + && chmod +x shfmt && sudo mv shfmt /usr/bin/ + # Need to success pre-commit fix push + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - uses: actions/setup-python@v2 + with: + python-version: '3.9' + - name: Execute pre-commit + uses: pre-commit/action@v2.0.0 + env: + SKIP: no-commit-to-branch + with: + token: ${{ secrets.GITHUB_TOKEN }} + extra_args: --color=always --show-diff-on-failure --files ${{ steps.file_changes.outputs.files }} diff --git a/.github/workflows/stale-actions.yaml b/.github/workflows/stale-actions.yaml new file mode 100644 index 000000000..f769925dc --- /dev/null +++ b/.github/workflows/stale-actions.yaml @@ -0,0 +1,31 @@ +name: "Mark or close stale issues and PRs" +on: + schedule: + - cron: "0 0 * * *" + +jobs: + stale: + runs-on: ubuntu-latest + steps: + - uses: actions/stale@v3 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + # Staling issues and PR's + days-before-stale: 30 + stale-issue-label: stale + stale-pr-label: stale + stale-issue-message: | + This issue has been automatically marked as stale because it has been open 30 days + with no activity. Remove stale label or comment or this issue will be closed in 10 days + stale-pr-message: | + This PR has been automatically marked as stale because it has been open 30 days + with no activity. Remove stale label or comment or this PR will be closed in 10 days + # Not stale if have this labels + exempt-issue-labels: bug,wip,on-hold + exempt-pr-labels: bug,wip,on-hold + # Close issue operations + # Label will be automatically removed if the issues are no longer closed nor locked. + days-before-close: 10 + delete-branch: true + close-issue-message: This issue was automatically closed because of stale in 10 days + close-pr-message: This PR was automatically closed because of stale in 10 days diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a0304e23e..020f65f4c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,13 +1,33 @@ repos: - repo: git://github.com/pre-commit/pre-commit-hooks - rev: v3.4.0 + rev: v4.0.1 hooks: - - id: check-yaml + # Git style + - id: check-added-large-files + - id: check-merge-conflict + - id: check-vcs-permalinks + - id: forbid-new-submodules + - id: no-commit-to-branch + + # Common errors - id: end-of-file-fixer - id: trailing-whitespace - - id: check-case-conflict + args: [--markdown-linebreak-ext=md] + - id: check-yaml - id: check-merge-conflict - id: check-executables-have-shebangs + + # Cross platform + - id: check-case-conflict + - id: mixed-line-ending + args: [--fix=lf] + + # Security + - id: detect-aws-credentials + args: ['--allow-missing-credentials'] + - id: detect-private-key + + - repo: git://github.com/jumanjihouse/pre-commit-hooks rev: 2.1.5 hooks: diff --git a/.pre-commit-hooks.yaml b/.pre-commit-hooks.yaml index 76339abcc..746a676ac 100644 --- a/.pre-commit-hooks.yaml +++ b/.pre-commit-hooks.yaml @@ -12,7 +12,7 @@ require_serial: true entry: terraform_docs.sh language: script - files: (\.tf)$ + files: (\.tf|\.terraform\.lock\.hcl)$ exclude: \.terraform\/.*$ - id: terraform_docs_without_aggregate_type_defaults @@ -54,6 +54,7 @@ - id: terraform_tflint name: Terraform validate with tflint description: Validates all Terraform configuration files with TFLint. + require_serial: true entry: terraform_tflint.sh language: script files: (\.tf|\.tfvars)$ @@ -91,3 +92,9 @@ files: \.tf$ exclude: \.+.terraform\/.*$ require_serial: true + +- id: terrascan + name: terrascan + description: Runs terrascan on Terraform templates. + language: script + entry: terrascan.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index 8170d93fa..84166f8e3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,47 @@ All notable changes to this project will be documented in this file. + +## [v1.51.0] - 2021-09-17 + +- fix: trigger terraform-docs on changes in lock files ([#228](https://github.com/antonbabenko/pre-commit-terraform/issues/228)) +- fix: label auto-adding after label rename ([#226](https://github.com/antonbabenko/pre-commit-terraform/issues/226)) +- chore: Updated GH stale action config ([#223](https://github.com/antonbabenko/pre-commit-terraform/issues/223)) +- feat: Add GH checks and templates ([#222](https://github.com/antonbabenko/pre-commit-terraform/issues/222)) +- feat: Add mixed line ending check to prevent possible errors ([#221](https://github.com/antonbabenko/pre-commit-terraform/issues/221)) +- fix: Dockerized pre-commit-terraform ([#219](https://github.com/antonbabenko/pre-commit-terraform/issues/219)) +- docs: Initial docs improvement ([#218](https://github.com/antonbabenko/pre-commit-terraform/issues/218)) +- chore: Update Ubuntu install method ([#198](https://github.com/antonbabenko/pre-commit-terraform/issues/198)) + + + +## [v1.50.0] - 2021-04-22 + +- feat: Adds support for Terrascan ([#195](https://github.com/antonbabenko/pre-commit-terraform/issues/195)) + + + +## [v1.49.0] - 2021-04-20 + +- fix: Fix and pin versions in Dockerfile ([#193](https://github.com/antonbabenko/pre-commit-terraform/issues/193)) +- chore: Fix mistake on command ([#185](https://github.com/antonbabenko/pre-commit-terraform/issues/185)) +- Update README.md + + + +## [v1.48.0] - 2021-03-12 + +- chore: add dockerfile ([#183](https://github.com/antonbabenko/pre-commit-terraform/issues/183)) +- docs: Added checkov install ([#182](https://github.com/antonbabenko/pre-commit-terraform/issues/182)) + + + +## [v1.47.0] - 2021-02-25 + +- fix: remove sed postprocessing from the terraform_docs_replace hook to fix compatibility with terraform-docs 0.11.0+ ([#176](https://github.com/antonbabenko/pre-commit-terraform/issues/176)) +- docs: updates installs for macOS and ubuntu ([#175](https://github.com/antonbabenko/pre-commit-terraform/issues/175)) + + ## [v1.46.0] - 2021-02-20 @@ -387,7 +428,12 @@ https://github.com/antonbabenko/pre-commit-terraform/commit/35e0356188b64a4c5af9 - Initial commit -[Unreleased]: https://github.com/antonbabenko/pre-commit-terraform/compare/v1.46.0...HEAD +[Unreleased]: https://github.com/antonbabenko/pre-commit-terraform/compare/v1.51.0...HEAD +[v1.51.0]: https://github.com/antonbabenko/pre-commit-terraform/compare/v1.50.0...v1.51.0 +[v1.50.0]: https://github.com/antonbabenko/pre-commit-terraform/compare/v1.49.0...v1.50.0 +[v1.49.0]: https://github.com/antonbabenko/pre-commit-terraform/compare/v1.48.0...v1.49.0 +[v1.48.0]: https://github.com/antonbabenko/pre-commit-terraform/compare/v1.47.0...v1.48.0 +[v1.47.0]: https://github.com/antonbabenko/pre-commit-terraform/compare/v1.46.0...v1.47.0 [v1.46.0]: https://github.com/antonbabenko/pre-commit-terraform/compare/v1.45.0...v1.46.0 [v1.45.0]: https://github.com/antonbabenko/pre-commit-terraform/compare/v1.44.0...v1.45.0 [v1.44.0]: https://github.com/antonbabenko/pre-commit-terraform/compare/v1.43.1...v1.44.0 diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..18e1c7c63 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,168 @@ +FROM ubuntu:20.04 as builder + +# Install general dependencies +RUN apt update && \ + DEBIAN_FRONTEND=noninteractive apt install -y \ + # Needed for pre-commit in next build stage + git \ + libpcre2-8-0 \ + # Builder deps + unzip \ + software-properties-common \ + curl \ + python3 \ + python3-pip && \ + # Upgrade pip for be able get latest Checkov + python3 -m pip install --upgrade pip && \ + # Cleanup + rm -rf /var/lib/apt/lists/* + +ARG PRE_COMMIT_VERSION=${PRE_COMMIT_VERSION:-latest} +ARG TERRAFORM_VERSION=${TERRAFORM_VERSION:-latest} + +# Install pre-commit +RUN [ ${PRE_COMMIT_VERSION} = "latest" ] && pip3 install --no-cache-dir pre-commit \ + || pip3 install --no-cache-dir pre-commit==${PRE_COMMIT_VERSION} + +# Install terraform because pre-commit needs it +RUN curl -fsSL https://apt.releases.hashicorp.com/gpg | apt-key add - && \ + apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main" && \ + apt update && \ + ( \ + [ "$TERRAFORM_VERSION" = "latest" ] && apt install -y terraform \ + || apt install -y terraform=${TERRAFORM_VERSION} \ + ) && \ + # Cleanup + rm -rf /var/lib/apt/lists/* + +# +# Install tools +# +WORKDIR /bin_dir + +ARG CHECKOV_VERSION=${CHECKOV_VERSION:-false} +ARG TERRAFORM_DOCS_VERSION=${TERRAFORM_DOCS_VERSION:-false} +ARG TERRAGRUNT_VERSION=${TERRAGRUNT_VERSION:-false} +ARG TERRASCAN_VERSION=${TERRASCAN_VERSION:-false} +ARG TFLINT_VERSION=${TFLINT_VERSION:-false} +ARG TFSEC_VERSION=${TFSEC_VERSION:-false} + + +# Tricky thing to install all tools by set only one arg. +# In RUN command below used `. /.env` <- this is sourcing vars that +# specified in step below +ARG INSTALL_ALL=${INSTALL_ALL:-false} +RUN if [ "$INSTALL_ALL" != "false" ]; then \ + echo "export CHECKOV_VERSION=latest" >> /.env && \ + echo "export TERRAFORM_DOCS_VERSION=latest" >> /.env && \ + echo "export TERRAGRUNT_VERSION=latest" >> /.env && \ + echo "export TERRASCAN_VERSION=latest" >> /.env && \ + echo "export TFLINT_VERSION=latest" >> /.env && \ + echo "export TFSEC_VERSION=latest" >> /.env \ + ; else \ + touch /.env \ + ; fi + + +# Checkov +RUN . /.env && \ + if [ "$CHECKOV_VERSION" != "false" ]; then \ + ( \ + [ "$CHECKOV_VERSION" = "latest" ] && pip3 install --no-cache-dir checkov \ + || pip3 install --no-cache-dir checkov==${CHECKOV_VERSION} \ + ) \ + ; fi + +# Terraform docs +RUN . /.env && \ + if [ "$TERRAFORM_DOCS_VERSION" != "false" ]; then \ + ( \ + TERRAFORM_DOCS_RELEASES="https://api.github.com/repos/terraform-docs/terraform-docs/releases" && \ + [ "$TERRAFORM_DOCS_VERSION" = "latest" ] && curl -L "$(curl -s ${TERRAFORM_DOCS_RELEASES}/latest | grep -o -E -m 1 "https://.+?-linux-amd64.tar.gz")" > terraform-docs.tgz \ + || curl -L "$(curl -s ${TERRAFORM_DOCS_RELEASES} | grep -o -E "https://.+?v${TERRAFORM_DOCS_VERSION}-linux-amd64.tar.gz")" > terraform-docs.tgz \ + ) && tar -xzf terraform-docs.tgz terraform-docs && rm terraform-docs.tgz && chmod +x terraform-docs \ + ; fi + +# Terragrunt +RUN . /.env \ + && if [ "$TERRAGRUNT_VERSION" != "false" ]; then \ + ( \ + TERRAGRUNT_RELEASES="https://api.github.com/repos/gruntwork-io/terragrunt/releases" && \ + [ "$TERRAGRUNT_VERSION" = "latest" ] && curl -L "$(curl -s ${TERRAGRUNT_RELEASES}/latest | grep -o -E -m 1 "https://.+?/terragrunt_linux_amd64")" > terragrunt \ + || curl -L "$(curl -s ${TERRAGRUNT_RELEASES} | grep -o -E -m 1 "https://.+?v${TERRAGRUNT_VERSION}/terragrunt_linux_amd64")" > terragrunt \ + ) && chmod +x terragrunt \ + ; fi + + +# Terrascan +RUN . /.env && \ + if [ "$TERRASCAN_VERSION" != "false" ]; then \ + ( \ + TERRASCAN_RELEASES="https://api.github.com/repos/accurics/terrascan/releases" && \ + [ "$TERRASCAN_VERSION" = "latest" ] && curl -L "$(curl -s ${TERRASCAN_RELEASES}/latest | grep -o -E -m 1 "https://.+?_Linux_x86_64.tar.gz")" > terrascan.tar.gz \ + || curl -L "$(curl -s ${TERRASCAN_RELEASES} | grep -o -E "https://.+?${TERRASCAN_VERSION}_Linux_x86_64.tar.gz")" > terrascan.tar.gz \ + ) && tar -xzf terrascan.tar.gz terrascan && rm terrascan.tar.gz && \ + ./terrascan init \ + ; fi + +# TFLint +RUN . /.env && \ + if [ "$TFLINT_VERSION" != "false" ]; then \ + ( \ + TFLINT_RELEASES="https://api.github.com/repos/terraform-linters/tflint/releases" && \ + [ "$TFLINT_VERSION" = "latest" ] && curl -L "$(curl -s ${TFLINT_RELEASES}/latest | grep -o -E -m 1 "https://.+?_linux_amd64.zip")" > tflint.zip \ + || curl -L "$(curl -s ${TFLINT_RELEASES} | grep -o -E "https://.+?/v${TFLINT_VERSION}/tflint_linux_amd64.zip")" > tflint.zip \ + ) && unzip tflint.zip && rm tflint.zip \ + ; fi + +# TFSec +RUN . /.env && \ + if [ "$TFSEC_VERSION" != "false" ]; then \ + ( \ + TFSEC_RELEASES="https://api.github.com/repos/aquasecurity/tfsec/releases" && \ + [ "$TFSEC_VERSION" = "latest" ] && curl -L "$(curl -s ${TFSEC_RELEASES}/latest | grep -o -E -m 1 "https://.+?/tfsec-linux-amd64")" > tfsec \ + || curl -L "$(curl -s ${TFSEC_RELEASES} | grep -o -E -m 1 "https://.+?v${TFSEC_VERSION}/tfsec-linux-amd64")" > tfsec \ + ) && chmod +x tfsec \ + ; fi + +# Checking binaries versions and write it to debug file +RUN . /.env && \ + F=tools_versions_info && \ + pre-commit --version >> $F && \ + terraform --version | head -n 1 >> $F && \ + (if [ "$CHECKOV_VERSION" != "false" ]; then echo "checkov $(checkov --version)" >> $F; else echo "checkov SKIPPED" >> $F ; fi) && \ + (if [ "$TERRAFORM_DOCS_VERSION" != "false" ]; then ./terraform-docs --version >> $F; else echo "terraform-docs SKIPPED" >> $F; fi) && \ + (if [ "$TERRAGRUNT_VERSION" != "false" ]; then ./terragrunt --version >> $F; else echo "terragrunt SKIPPED" >> $F ; fi) && \ + (if [ "$TERRASCAN_VERSION" != "false" ]; then echo "terrascan $(./terrascan version)" >> $F; else echo "terrascan SKIPPED" >> $F ; fi) && \ + (if [ "$TFLINT_VERSION" != "false" ]; then ./tflint --version >> $F; else echo "tflint SKIPPED" >> $F ; fi) && \ + (if [ "$TFSEC_VERSION" != "false" ]; then echo "tfsec $(./tfsec --version)" >> $F; else echo "tfsec SKIPPED" >> $F ; fi) && \ + echo "\n\n" && cat $F && echo "\n\n" + +# based on debian:buster-slim +# https://github.com/docker-library/python/blob/master/3.9/buster/slim/Dockerfile +FROM python:3.9-slim-buster + +# Python 3.8 (ubuntu 20.04) -> Python3.9 hacks +COPY --from=builder /usr/local/lib/python3.8/dist-packages/ /usr/local/lib/python3.9/site-packages/ +COPY --from=builder /usr/lib/python3/dist-packages /usr/local/lib/python3.9/site-packages +RUN mkdir /usr/lib/python3 && \ + ln -s /usr/local/lib/python3.9/site-packages /usr/lib/python3/site-packages && \ + ln -s /usr/local/bin/python3 /usr/bin/python3 +# Copy binaries needed for pre-commit +COPY --from=builder /usr/lib/git-core/ /usr/lib/git-core/ +COPY --from=builder /usr/lib/x86_64-linux-gnu/libpcre2-8.so.0 /usr/lib/x86_64-linux-gnu/ +# Copy tools +COPY --from=builder \ + /bin_dir/ \ + /usr/bin/terraform \ + /usr/local/bin/checkov* \ + /usr/local/bin/pre-commit \ + /usr/bin/git \ + /usr/bin/git-shell \ + /usr/bin/ +# Copy terrascan policies +COPY --from=builder /root/ /root/ + +ENV PRE_COMMIT_COLOR=${PRE_COMMIT_COLOR:-always} + +ENTRYPOINT [ "pre-commit" ] diff --git a/README.md b/README.md index 4881f249e..4c7efdbb6 100644 --- a/README.md +++ b/README.md @@ -1,37 +1,134 @@ # Collection of git hooks for Terraform to be used with [pre-commit framework](http://pre-commit.com/) -[![Github tag](https://img.shields.io/github/tag/antonbabenko/pre-commit-terraform.svg)](https://github.com/antonbabenko/pre-commit-terraform/releases) ![](https://img.shields.io/maintenance/yes/2020.svg) [![Help Contribute to Open Source](https://www.codetriage.com/antonbabenko/pre-commit-terraform/badges/users.svg)](https://www.codetriage.com/antonbabenko/pre-commit-terraform) +[![Github tag](https://img.shields.io/github/tag/antonbabenko/pre-commit-terraform.svg)](https://github.com/antonbabenko/pre-commit-terraform/releases) ![maintenance status](https://img.shields.io/maintenance/yes/2021.svg) [![Help Contribute to Open Source](https://www.codetriage.com/antonbabenko/pre-commit-terraform/badges/users.svg)](https://www.codetriage.com/antonbabenko/pre-commit-terraform) + +* [How to install](#how-to-install) + * [1. Install dependencies](#1-install-dependencies) + * [2. Install the pre-commit hook globally](#2-install-the-pre-commit-hook-globally) + * [3. Add configs and hooks](#3-add-configs-and-hooks) + * [4. Run](#4-run) +* [Available Hooks](#available-hooks) +* [Hooks usage notes and examples](#hooks-usage-notes-and-examples) + * [checkov](#checkov) + * [terraform_docs](#terraform_docs) + * [terraform_docs_replace](#terraform_docs_replace) + * [terraform_providers_lock](#terraform_providers_lock) + * [terraform_tflint](#terraform_tflint) + * [terraform_tfsec](#terraform_tfsec) + * [terraform_validate](#terraform_validate) +* [Notes for contributors](#notes-for-contributors) + * [Run and debug hooks locally](#run-and-debug-hooks-locally) +* [Authors](#authors) +* [License](#license) ## How to install ### 1. Install dependencies -* [`pre-commit`][1] -* [`terraform-docs`][2] required for `terraform_docs` hooks. `GNU awk` is required if using `terraform-docs` older than 0.8.0 with Terraform 0.12. -* [`TFLint`][3] required for `terraform_tflint` hook. -* [`TFSec`][4] required for `terraform_tfsec` hook. -* [`coreutils`][5] required for `terraform_validate` hook on macOS (due to use of `realpath`). -* [`checkov`][6] required for `checkov` hook. + + +* [`pre-commit`](https://pre-commit.com/#install), + [`terraform`](https://www.terraform.io/downloads.html), + [`git`](https://git-scm.com/downloads), + POSIX compatible shell, + Internet connection (on first run), + x86_64 compatible operation system, + Some hardware where this OS will run, + Electricity for hardware and internet connection, + Some basic physical laws, + Hope that it all will works. +

+* [`checkov`](https://github.com/bridgecrewio/checkov) required for `checkov` hook. +* [`terraform-docs`](https://github.com/terraform-docs/terraform-docs) required for `terraform_docs` hooks. +* [`terragrunt`](https://terragrunt.gruntwork.io/docs/getting-started/install/) required for `terragrunt_validate` hook. +* [`terrascan`](https://github.com/accurics/terrascan) required for `terrascan` hook. +* [`TFLint`](https://github.com/terraform-linters/tflint) required for `terraform_tflint` hook. +* [`TFSec`](https://github.com/liamg/tfsec) required for `terraform_tfsec` hook. + +
Docker
+ +If no `--build-arg` is specified, then the latest versions of `pre-commit` and `terraform` will be installed. -##### MacOS +```bash +git clone git@github.com:antonbabenko/pre-commit-terraform.git +cd pre-commit-terraform +# Install all tools with latest versions: +docker build -t pre-commit --build-arg INSTALL_ALL=true . +``` + +You can specify needed tool versions by providing `--build-arg`'s. +If you'd like you can use the `latest` versions: ```bash -brew tap liamg/tfsec -brew install pre-commit gawk terraform-docs tflint tfsec coreutils +docker build -t pre-commit \ + --build-arg PRE_COMMIT_VERSION=latest \ + --build-arg TERRAFORM_VERSION=latest \ + --build-arg CHECKOV_VERSION=2.0.405 \ + --build-arg TERRAFORM_DOCS_VERSION=0.15.0 \ + --build-arg TERRAGRUNT_VERSION=latest \ + --build-arg TERRASCAN_VERSION=1.10.0 \ + --build-arg TFLINT_VERSION=0.31.0 \ + --build-arg TFSEC_VERSION=latest \ + . ``` -##### Ubuntu +To disable pre-commit color output set `-e PRE_COMMIT_COLOR=never`. + +
+ + +
MacOS
+ +[`coreutils`](https://formulae.brew.sh/formula/coreutils) required for `terraform_validate` hook on macOS (due to use of `realpath`). ```bash -sudo apt install python3-pip gawk &&\ -pip3 install pre-commit -curl -L "$(curl -s https://api.github.com/repos/terraform-docs/terraform-docs/releases/latest | grep -o -E "https://.+?-linux-amd64")" > terraform-docs && chmod +x terraform-docs && sudo mv terraform-docs /usr/bin/ -curl -L "$(curl -s https://api.github.com/repos/terraform-linters/tflint/releases/latest | grep -o -E "https://.+?_linux_amd64.zip")" > tflint.zip && unzip tflint.zip && rm tflint.zip && sudo mv tflint /usr/bin/ -env GO111MODULE=on go get -u github.com/liamg/tfsec/cmd/tfsec +brew install pre-commit terraform-docs tflint tfsec coreutils checkov terrascan +terrascan init ``` +
+ +
Ubuntu 18.04
+ +```bash +sudo apt update +sudo apt install -y unzip software-properties-common +sudo add-apt-repository ppa:deadsnakes/ppa +sudo apt install -y python3.7 python3-pip +python3 -m pip install --upgrade pip +pip3 install --no-cache-dir pre-commit +python3.7 -m pip install -U checkov +curl -L "$(curl -s https://api.github.com/repos/terraform-docs/terraform-docs/releases/latest | grep -o -E -m 1 "https://.+?-linux-amd64.tar.gz")" > terraform-docs.tgz && tar -xzf terraform-docs.tgz && rm terraform-docs.tgz && chmod +x terraform-docs && sudo mv terraform-docs /usr/bin/ +curl -L "$(curl -s https://api.github.com/repos/terraform-linters/tflint/releases/latest | grep -o -E -m 1 "https://.+?_linux_amd64.zip")" > tflint.zip && unzip tflint.zip && rm tflint.zip && sudo mv tflint /usr/bin/ +curl -L "$(curl -s https://api.github.com/repos/aquasecurity/tfsec/releases/latest | grep -o -E -m 1 "https://.+?tfsec-linux-amd64")" > tfsec && chmod +x tfsec && sudo mv tfsec /usr/bin/ +curl -L "$(curl -s https://api.github.com/repos/accurics/terrascan/releases/latest | grep -o -E -m 1"https://.+?_Linux_x86_64.tar.gz")" > terrascan.tar.gz && tar -xzf terrascan.tar.gz terrascan && rm terrascan.tar.gz && sudo mv terrascan /usr/bin/ && terrascan init +``` + +
+ + +
Ubuntu 20.04
+ +```bash +sudo apt update +sudo apt install -y unzip software-properties-common python3 python3-pip +python3 -m pip install --upgrade pip +pip3 install --no-cache-dir pre-commit +pip3 install --no-cache-dir checkov +curl -L "$(curl -s https://api.github.com/repos/terraform-docs/terraform-docs/releases/latest | grep -o -E -m 1 "https://.+?-linux-amd64.tar.gz")" > terraform-docs.tgz && tar -xzf terraform-docs.tgz terraform-docs && rm terraform-docs.tgz && chmod +x terraform-docs && sudo mv terraform-docs /usr/bin/ +curl -L "$(curl -s https://api.github.com/repos/accurics/terrascan/releases/latest | grep -o -E -m 1"https://.+?_Linux_x86_64.tar.gz")" > terrascan.tar.gz && tar -xzf terrascan.tar.gz terrascan && rm terrascan.tar.gz && sudo mv terrascan /usr/bin/ && terrascan init +curl -L "$(curl -s https://api.github.com/repos/terraform-linters/tflint/releases/latest | grep -o -E -m 1 "https://.+?_linux_amd64.zip")" > tflint.zip && unzip tflint.zip && rm tflint.zip && sudo mv tflint /usr/bin/ +curl -L "$(curl -s https://api.github.com/repos/aquasecurity/tfsec/releases/latest | grep -o -E -m 1 "https://.+?tfsec-linux-amd64")" > tfsec && chmod +x tfsec && sudo mv tfsec /usr/bin/ +``` + +
+ + + ### 2. Install the pre-commit hook globally +> Note: not needed if you use the Docker image + ```bash DIR=~/.git-template git config --global init.templateDir ${DIR} @@ -56,103 +153,141 @@ EOF ### 4. Run -After pre-commit hook has been installed you can run it manually on all files in the repository +After pre-commit hook has been installed you can run it manually on all files in the repository. + +Local installation: ```bash pre-commit run -a ``` +Docker: + +```bash +docker run -v $(pwd):/lint -w /lint pre-commit run -a +``` + +> You be able list tools versions when needed +> +> ```bash +> TAG=latest && docker run --entrypoint cat pre-commit:$TAG /usr/bin/tools_versions_info +> ``` + ## Available Hooks There are several [pre-commit](https://pre-commit.com/) hooks to keep Terraform configurations (both `*.tf` and `*.tfvars`) and Terragrunt configurations (`*.hcl`) in a good shape: -| Hook name | Description | -| ------------------------------------------------ | -------------------------------------------------------------------------------------------------------------------------- | -| `terraform_fmt` | Rewrites all Terraform configuration files to a canonical format. | -| `terraform_validate` | Validates all Terraform configuration files. | -| `terraform_providers_lock` | Updates provider signatures in [dependency lock files][7]. | -| `terraform_docs` | Inserts input and output documentation into `README.md`. Recommended. | -| `terraform_docs_without_aggregate_type_defaults` | Inserts input and output documentation into `README.md` without aggregate type defaults. | -| `terraform_docs_replace` | Runs `terraform-docs` and pipes the output directly to README.md (requires [terraform-docs][2] v0.10.0 or later) | -| `terraform_tflint` | Validates all Terraform configuration files with [TFLint][3]. | -| `terragrunt_fmt` | Rewrites all [Terragrunt][8] configuration files (`*.hcl`) to a canonical format. | -| `terragrunt_validate` | Validates all [Terragrunt][8] configuration files (`*.hcl`) | -| `terraform_tfsec` | [TFSec][4] static analysis of terraform templates to spot potential security issues. | -| `checkov` | [checkov][6] static analysis of terraform templates to spot potential security issues. | +| Hook name | Description | +| ------------------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `checkov` | [checkov](https://github.com/bridgecrewio/checkov) static analysis of terraform templates to spot potential security issues. [Hook notes](#checkov) | +| `terraform_docs_replace` | Runs `terraform-docs` and pipes the output directly to README.md | +| `terraform_docs_without_aggregate_type_defaults` | Inserts input and output documentation into `README.md` without aggregate type defaults. Hook notes same as for [terraform_docs](#terraform_docs) | +| `terraform_docs` | Inserts input and output documentation into `README.md`. Recommended. [Hook notes](#terraform_docs) | +| `terraform_fmt` | Rewrites all Terraform configuration files to a canonical format. [Hook notes](#terraform_docs) | +| `terraform_providers_lock` | Updates provider signatures in [dependency lock files](https://www.terraform.io/docs/cli/commands/providers/lock.html). [Hook notes](#terraform_providers_lock) +| `terraform_tflint` | Validates all Terraform configuration files with [TFLint](https://github.com/terraform-linters/tflint). [Available TFLint rules](https://github.com/terraform-linters/tflint/tree/master/docs/rules#rules). [Hook notes](#terraform_tflint). | +| `terraform_tfsec` | [TFSec](https://github.com/liamg/tfsec) static analysis of terraform templates to spot potential security issues. [Hook notes](#terraform_tfsec) | +| `terraform_validate` | Validates all Terraform configuration files. [Hook notes](#terraform_validate) | +| `terragrunt_fmt` | Rewrites all [Terragrunt](https://github.com/gruntwork-io/terragrunt) configuration files (`*.hcl`) to a canonical format. | +| `terragrunt_validate` | Validates all [Terragrunt](https://github.com/gruntwork-io/terragrunt) configuration files (`*.hcl`) | +| `terrascan` | [terrascan](https://github.com/accurics/terrascan) Detect compliance and security violations. | Check the [source file](https://github.com/antonbabenko/pre-commit-terraform/blob/master/.pre-commit-hooks.yaml) to know arguments used for each hook. -## Notes about terraform_docs hooks +## Hooks usage notes and examples + +### checkov -1. `terraform_docs` and `terraform_docs_without_aggregate_type_defaults` will insert/update documentation generated by [terraform-docs][2] framed by markers: -```txt - +For [checkov](https://github.com/bridgecrewio/checkov) you need to specify each argument separately: - +```yaml +- id: checkov + args: [ + "-d", ".", + "--skip-check", "CKV2_AWS_8", + ] ``` -if they are present in `README.md`. -1. `terraform_docs_replace` replaces the entire README.md rather than doing string replacement between markers. Put your additional documentation at the top of your `main.tf` for it to be pulled in. The optional `--dest` argument lets you change the name of the file that gets created/modified. This hook requires terraform-docs v0.10.0 or later. +### terraform_docs - 1. Example: - ```yaml - hooks: - - id: terraform_docs_replace - args: ['--sort-by-required', '--dest=TEST.md'] +1. `terraform_docs` and `terraform_docs_without_aggregate_type_defaults` will insert/update documentation generated by [terraform-docs](https://github.com/terraform-docs/terraform-docs) framed by markers: + + ```txt + + + ``` -1. It is possible to pass additional arguments to shell scripts when using `terraform_docs` and `terraform_docs_without_aggregate_type_defaults`. Send pull-request with the new hook if there is something missing. + if they are present in `README.md`. + +2. It is possible to pass additional arguments to shell scripts when using `terraform_docs` and `terraform_docs_without_aggregate_type_defaults`. Send pull-request with the new hook if there is something missing. + +For these hooks you need to specify all arguments as one: + +```yaml +- id: terraform_docs + args: + - tfvars hcl --output-file terraform.tfvars.model . +``` + +### terraform_docs_replace + +`terraform_docs_replace` replaces the entire README.md rather than doing string replacement between markers. Put your additional documentation at the top of your `main.tf` for it to be pulled in. The optional `--dest` argument lets you change the name of the file that gets created/modified. + +Example: + +```yaml +- id: terraform_docs_replace + args: + - --sort-by-required + - --dest=TEST.md +``` -## Notes about terraform_tflint hooks +### terraform_tflint 1. `terraform_tflint` supports custom arguments so you can enable module inspection, deep check mode etc. - 1. Example: - ```yaml - hooks: - - id: terraform_tflint - args: ['--args=--deep'] - ``` + Example: - In order to pass multiple args, try the following: ```yaml - - id: terraform_tflint - args: - - '--args=--deep' - - '--args=--enable-rule=terraform_documented_variables' + - id: terraform_tflint + args: + - --args=--deep + - --args=--enable-rule=terraform_documented_variables ``` -1. When you have multiple directories and want to run `tflint` in all of them and share single config file it is impractical to hard-code the path to `.tflint.hcl` file. The solution is to use `__GIT_WORKING_DIR__` placeholder which will be replaced by `terraform_tflint` hooks with Git working directory (repo root) at run time. For example: +2. When you have multiple directories and want to run `tflint` in all of them and share single config file it is impractical to hard-code the path to `.tflint.hcl` file. The solution is to use `__GIT_WORKING_DIR__` placeholder which will be replaced by `terraform_tflint` hooks with Git working directory (repo root) at run time. For example: - ```yaml - hooks: - - id: terraform_tflint - args: - - '--args=--config=__GIT_WORKING_DIR__/.tflint.hcl' - ``` + ```yaml + - id: terraform_tflint + args: + - --args=--config=__GIT_WORKING_DIR__/.tflint.hcl + ``` -## Notes about terraform_tfsec hooks +### terraform_tfsec 1. `terraform_tfsec` will consume modified files that pre-commit passes to it, so you can perform whitelisting of directories or files to run against via [files](https://pre-commit.com/#config-files) pre-commit flag - 1. Example: + Example: + ```yaml - hooks: - - id: terraform_tfsec - files: ^prd-infra/ + - id: terraform_tfsec + files: ^prd-infra/ ``` The above will tell pre-commit to pass down files from the `prd-infra/` folder only such that the underlying `tfsec` tool can run against changed files in this directory, ignoring any other folders at the root level -1. To ignore specific warnings, follow the convention from the +2. To ignore specific warnings, follow the convention from the [documentation](https://github.com/liamg/tfsec#ignoring-warnings). - 1. Example: + + Example: + ```hcl resource "aws_security_group_rule" "my-rule" { type = "ingress" @@ -160,51 +295,44 @@ if they are present in `README.md`. } ``` -## Notes about terraform_validate hooks - -1. `terraform_validate` supports custom arguments so you can pass supported no-color or json flags. +### terraform_validate - 1. Example: - ```yaml - hooks: - - id: terraform_validate - args: ['--args=-json'] - ``` +1. `terraform_validate` supports custom arguments so you can pass supported no-color or json flags: - In order to pass multiple args, try the following: ```yaml - id: terraform_validate args: - - '--args=-json' - - '--args=-no-color' + - --args=-json + - --args=-no-color ``` -1. `terraform_validate` also supports custom environment variables passed to the pre-commit runtime - 1. Example: - ```yaml - hooks: - - id: terraform_validate - args: ['--envs=AWS_DEFAULT_REGION="us-west-2"'] - ``` +2. `terraform_validate` also supports custom environment variables passed to the pre-commit runtime: - In order to pass multiple args, try the following: ```yaml - - id: terraform_validate - args: - - '--envs=AWS_DEFAULT_REGION="us-west-2"' - - '--envs=AWS_ACCESS_KEY_ID="anaccesskey"' - - '--envs=AWS_SECRET_ACCESS_KEY="asecretkey"' + - id: terraform_validate + args: + - --envs=AWS_DEFAULT_REGION="us-west-2" + - --envs=AWS_ACCESS_KEY_ID="anaccesskey" + - --envs=AWS_SECRET_ACCESS_KEY="asecretkey" ``` -1. It may happen that Terraform working directory (`.terraform`) already exists but not in the best condition (eg, not initialized modules, wrong version of Terraform, etc). To solve this problem you can find and delete all `.terraform` directories in your repository using this command: +3. It may happen that Terraform working directory (`.terraform`) already exists but not in the best condition (eg, not initialized modules, wrong version of Terraform, etc). To solve this problem you can find and delete all `.terraform` directories in your repository: - ```shell - find . -type d -name ".terraform" -print0 | xargs -0 rm -r + ```bash + echo " + function rm_terraform { + find . -name ".terraform*" -print0 | xargs -0 rm -r + } + " >>~/.bashrc + + # Reload shell and use `rm_terraform` command in repo root ``` `terraform_validate` hook will try to reinitialize them before running `terraform validate` command. -## Notes about terraform_providers_lock hook + **Warning:** If you use Terraform workspaces, DO NOT use this workaround ([details](https://github.com/antonbabenko/pre-commit-terraform/issues/203#issuecomment-918791847)). Wait to [`force-init`](https://github.com/antonbabenko/pre-commit-terraform/issues/224) option implementation + +### terraform_providers_lock 1. The hook requires Terraform 0.14 or later. @@ -245,12 +373,29 @@ if they are present in `README.md`. ## Notes for developers +## Notes for contributors + 1. Python hooks are supported now too. All you have to do is: 1. add a line to the `console_scripts` array in `entry_points` in `setup.py` - 1. Put your python script in the `pre_commit_hooks` folder + 2. Put your python script in the `pre_commit_hooks` folder Enjoy the clean, valid, and documented code! +### Run and debug hooks locally + +```bash +pre-commit try-repo {-a} /path/to/local/pre-commit-terraform/repo {hook_name} +``` + +I.e. + +```bash +pre-commit try-repo /mnt/c/Users/tf/pre-commit-terraform terraform_fmt # Run only `terraform_fmt` check +pre-commit try-repo -a ~/pre-commit-terraform # run all existing checks from repo +``` + +Running `pre-commit` with `try-repo` ignores all arguments specified in `.pre-commit-config.yaml`. + ## Authors This repository is managed by [Anton Babenko](https://github.com/antonbabenko) with help from [these awesome contributors](https://github.com/antonbabenko/pre-commit-terraform/graphs/contributors). @@ -258,12 +403,3 @@ This repository is managed by [Anton Babenko](https://github.com/antonbabenko) w ## License MIT licensed. See LICENSE for full details. - -[1]: https://pre-commit.com/#install -[2]: https://github.com/terraform-docs/terraform-docs -[3]: https://github.com/terraform-linters/tflint -[4]: https://github.com/liamg/tfsec -[5]: https://formulae.brew.sh/formula/coreutils -[6]: https://github.com/bridgecrewio/checkov -[7]: https://www.terraform.io/docs/cli/commands/providers/lock.html -[8]: https://github.com/gruntwork-io/terragrunt diff --git a/pre_commit_hooks/terraform_docs_replace.py b/pre_commit_hooks/terraform_docs_replace.py index e1777b306..a9cf6c9bc 100644 --- a/pre_commit_hooks/terraform_docs_replace.py +++ b/pre_commit_hooks/terraform_docs_replace.py @@ -29,7 +29,7 @@ def main(argv=None): dirs = [] for filename in args.filenames: - if (os.path.realpath(filename) not in dirs and \ + if (os.path.realpath(filename) not in dirs and (filename.endswith(".tf") or filename.endswith(".tfvars"))): dirs.append(os.path.dirname(filename)) @@ -43,9 +43,8 @@ def main(argv=None): procArgs.append('--sort-by-required') procArgs.append('md') procArgs.append("./{dir}".format(dir=dir)) - procArgs.append("| sed -e '$ d' -e 'N;/^\\n$/D;P;D'") procArgs.append('>') - procArgs.append("./{dir}/{dest}".format(dir=dir,dest=args.dest)) + procArgs.append("./{dir}/{dest}".format(dir=dir, dest=args.dest)) subprocess.check_call(" ".join(procArgs), shell=True) except subprocess.CalledProcessError as e: print(e) diff --git a/terraform_docs.sh b/terraform_docs.sh index 8c7076bcf..1eb3c0526 100755 --- a/terraform_docs.sh +++ b/terraform_docs.sh @@ -93,7 +93,6 @@ terraform_docs() { local -a -r files=("$@") declare -a paths - declare -a tfvars_files local index=0 local file_with_path @@ -102,10 +101,6 @@ terraform_docs() { paths[index]=$(dirname "$file_with_path") - if [[ "$file_with_path" == *".tfvars" ]]; then - tfvars_files+=("$file_with_path") - fi - ((index += 1)) done diff --git a/terraform_tflint.sh b/terraform_tflint.sh index 670e860ad..6da3b93ac 100755 --- a/terraform_tflint.sh +++ b/terraform_tflint.sh @@ -63,7 +63,18 @@ tflint_() { path_uniq="${path_uniq//__REPLACED__SPACE__/ }" pushd "$path_uniq" > /dev/null - tflint "${ARGS[@]}" + TFLINT_MSG=$( + tflint "${ARGS[@]}" 2>&1 || + echo >&2 -e "\033[1;31m\nERROR in ./$path_uniq/:\033[0m" && + tflint "${ARGS[@]}" # Print TFLint error with PATH + ) + + # Print checked PATH if TFLint have any messages + if [ ! -z "$TFLINT_MSG" ]; then + echo -e "\n./$path_uniq/:" + echo "$TFLINT_MSG" + fi + popd > /dev/null done } diff --git a/terrascan.sh b/terrascan.sh new file mode 100755 index 000000000..d8233068b --- /dev/null +++ b/terrascan.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +set -eo pipefail + +main() { + initialize_ + parse_cmdline_ "$@" + + # propagate $FILES to custom function + terrascan_ "$ARGS" "$FILES" +} + +terrascan_() { + # consume modified files passed from pre-commit so that + # terrascan runs against only those relevant directories + for file_with_path in $FILES; do + file_with_path="${file_with_path// /__REPLACED__SPACE__}" + paths[index]=$(dirname "$file_with_path") + + let "index+=1" + done + + for path_uniq in $(echo "${paths[*]}" | tr ' ' '\n' | sort -u); do + path_uniq="${path_uniq//__REPLACED__SPACE__/ }" + pushd "$path_uniq" > /dev/null + terrascan scan -i terraform $ARGS + popd > /dev/null + done +} + +initialize_() { + # get directory containing this script + local dir + local source + source="${BASH_SOURCE[0]}" + while [[ -L $source ]]; do # resolve $source until the file is no longer a symlink + dir="$(cd -P "$(dirname "$source")" > /dev/null && pwd)" + source="$(readlink "$source")" + # if $source was a relative symlink, we need to resolve it relative to the path where the symlink file was located + [[ $source != /* ]] && source="$dir/$source" + done + _SCRIPT_DIR="$(dirname "$source")" + + # source getopt function + # shellcheck source=lib_getopt + . "$_SCRIPT_DIR/lib_getopt" +} + +parse_cmdline_() { + declare argv + argv=$(getopt -o a: --long args: -- "$@") || return + eval "set -- $argv" + + for argv; do + case $argv in + -a | --args) + shift + ARGS+=("$1") + shift + ;; + --) + shift + FILES+=("$@") + break + ;; + esac + done +} + +# global arrays +declare -a ARGS=() +declare -a FILES=() + +[[ ${BASH_SOURCE[0]} != "$0" ]] || main "$@"