From 5f10fa1df16d0ecda7f759bd30d4970f3489ca5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wiktor=20Niesiob=C4=99dzki?= Date: Thu, 30 May 2024 12:36:38 +0000 Subject: [PATCH 1/3] Pre-commit config Run following linters on commit: Terraform: - terraform fmt - terraform tflint Python specific: - yapf Shell scripts - shellcheck - shfmt YAML files: - yamllint (disabled as of now) - check-yaml Other: - end-of-file-fixer - trailing-whitespace fixer Fabric specific - tools/tfdoc.py - tools/check_boilerplate.py --- .pre-commit-config.yaml | 80 +++++++++++++++++++++++++++++++++++++++ .yamllint | 16 ++++++++ tools/pre-commit-tfdoc.sh | 20 ++++++++++ tools/tflint-fast.py | 7 +++- 4 files changed, 121 insertions(+), 2 deletions(-) create mode 100644 .pre-commit-config.yaml create mode 100644 .yamllint create mode 100755 tools/pre-commit-tfdoc.sh diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000000..54d149a167 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,80 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +repos: +- repo: https://github.com/antonbabenko/pre-commit-terraform + rev: v1.86.0 + hooks: + - id: terraform_fmt + - id: terraform_tflint + args: + - --args=--config=__GIT_WORKING_DIR__/.tflint.hcl + files: ^modules/ + exclude: (tests/fixtures/|tools/lockfile/) + # - id: terraform_validate + # args: + # - --hook-config=--retry-once-with-cleanup=true +- repo: local + hooks: + - id: cff-readme + name: Regenerate README.md with tfdoc.py + entry: tools/pre-commit-tfdoc.sh + language: script + #types: [terraform] + files: ^(modules|fast).*(tf|README.md)$ + pass_filenames: true + require_serial: true + - id: licenese + name: Check license presence and other boilerplate checks + language: system + entry: tools/check_boilerplate.py + pass_filenames: true + - id: tflint-fast + name: Checking FAST code with tflint + entry: tools/tflint-fast.py + language: system + pass_filenames: false + require_serial: true + files: ^fast/.*tf + +# - repo: https://github.com/adrienverge/yamllint +# rev: v1.34.0 +# hooks: +# - id: yamllint +# args: [-c=.yamllint, --no-warnings] +# exclude: (/templates/|modules/cloud-config-container/) + +- repo: https://github.com/jumanjihouse/pre-commit-hooks + rev: "3.0.0" + hooks: + - id: script-must-have-extension + - id: shellcheck + - id: shfmt + exclude: ".*tpl" + +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.5.0 + hooks: + - id: end-of-file-fixer + - id: trailing-whitespace + - id: check-yaml + args: + - --allow-multiple-documents + exclude: (/templates/|modules/cloud-config-container/) + +- repo: https://github.com/google/yapf/ + rev: v0.40.2 + hooks: + - id: yapf diff --git a/.yamllint b/.yamllint new file mode 100644 index 0000000000..e5f87654f7 --- /dev/null +++ b/.yamllint @@ -0,0 +1,16 @@ +--- + +yaml-files: +- '*.yaml' +- '*.yml' +- '.yamllint' + +extends: default + +rules: + indentation: + indent-sequences: consistent + line-length: + max: 120 + level: warning + braces: disable diff --git a/tools/pre-commit-tfdoc.sh b/tools/pre-commit-tfdoc.sh new file mode 100755 index 0000000000..2f894a6087 --- /dev/null +++ b/tools/pre-commit-tfdoc.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +set -ev + +SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) + +files=("$@") +declare -A directories + +for file in "${files[@]}"; do + dir=$(dirname "${file}") + if [ -f "${dir}/README.md" ] && [ -f "${dir}/main.tf" ]; then + directories["${dir}"]=1 + fi +done + +for dir in "${!directories[@]}"; do # iterate over keys in directories + echo python "${SCRIPT_DIR}/tfdoc.py" "${dir}" + python "${SCRIPT_DIR}/tfdoc.py" "${dir}" +done diff --git a/tools/tflint-fast.py b/tools/tflint-fast.py index dc02ffdf5e..e8bb1919ab 100755 --- a/tools/tflint-fast.py +++ b/tools/tflint-fast.py @@ -42,8 +42,11 @@ def main(junit): args += ['--format=junit'] args += [ '--chdir', - str((BASEDIR / module_path).absolute()), '--var-file', - str((BASEDIR / var_path).absolute()) + str((BASEDIR / module_path).absolute()), + '--var-file', + str((BASEDIR / var_path).absolute()), + '--config', + str((BASEDIR / ".tflint.hcl").absolute()), ] print(' '.join(args)) if junit: From 625e92f46a844e46a28dd74fb777f116ba86aa82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wiktor=20Niesiob=C4=99dzki?= Date: Thu, 30 May 2024 13:32:57 +0000 Subject: [PATCH 2/3] linting fixes --- .../apigee/bigquery-analytics/send-requests.sh | 16 ++++++++-------- .../workload-identity-federation/setup.sh | 6 +++--- .../examples/cleanup-policies.yaml | 3 +-- .../modules/gcs/examples/iam-authoritative.yaml | 3 +-- .../gcs/examples/iam-bindings-additive.yaml | 3 +-- tests/modules/gcs/examples/iam-bindings.yaml | 3 +-- .../secret_manager/examples/secret-cmek.yaml | 2 -- 7 files changed, 15 insertions(+), 21 deletions(-) diff --git a/blueprints/apigee/bigquery-analytics/send-requests.sh b/blueprints/apigee/bigquery-analytics/send-requests.sh index 325cad78ac..7b6e772386 100755 --- a/blueprints/apigee/bigquery-analytics/send-requests.sh +++ b/blueprints/apigee/bigquery-analytics/send-requests.sh @@ -1,3 +1,5 @@ +#!/bin/bash + # Copyright 2023 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -11,18 +13,16 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -#!/bin/bash if [ $# -lt 2 ]; then - echo "Usage: $0 ENVGROUP_HOSTNAME NUM_REQUESTS" - exit 1 -fi + echo "Usage: $0 ENVGROUP_HOSTNAME NUM_REQUESTS" + exit 1 +fi ENVGROUP_HOSTNAME=$1 NUM_REQUESTS=$2 -for i in $(seq 1 $NUM_REQUESTS) -do -curl -v https://$ENVGROUP_HOSTNAME/httpbin/headers +# shellcheck disable=SC2034 +for i in $(seq 1 "$NUM_REQUESTS"); do + curl -v "https://$ENVGROUP_HOSTNAME/httpbin/headers" done diff --git a/blueprints/cloud-operations/workload-identity-federation/setup.sh b/blueprints/cloud-operations/workload-identity-federation/setup.sh index 7fe3a7103c..6943b648c6 100644 --- a/blueprints/cloud-operations/workload-identity-federation/setup.sh +++ b/blueprints/cloud-operations/workload-identity-federation/setup.sh @@ -1,3 +1,5 @@ +#!/bin/bash + # Copyright 2023 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -11,11 +13,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -#!/bin/bash sudo apt -y update sudo apt -y install apt-transport-https ca-certificates gnupg jq echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" | sudo tee -a /etc/apt/sources.list.d/google-cloud-sdk.list curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key --keyring /usr/share/keyrings/cloud.google.gpg add - -sudo apt-get update && sudo apt-get install google-cloud-sdk \ No newline at end of file +sudo apt-get update && sudo apt-get install google-cloud-sdk diff --git a/tests/modules/artifact_registry/examples/cleanup-policies.yaml b/tests/modules/artifact_registry/examples/cleanup-policies.yaml index f39fb8a132..94617df9a5 100644 --- a/tests/modules/artifact_registry/examples/cleanup-policies.yaml +++ b/tests/modules/artifact_registry/examples/cleanup-policies.yaml @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -values: values: module.registry-docker.google_artifact_registry_repository.registry: cleanup_policies: @@ -38,7 +37,7 @@ values: mode: STANDARD_REPOSITORY project: project-id repository_id: docker-cleanup-policies - + counts: google_artifact_registry_repository: 1 diff --git a/tests/modules/gcs/examples/iam-authoritative.yaml b/tests/modules/gcs/examples/iam-authoritative.yaml index 0be40cfb1c..84b93b1c5e 100644 --- a/tests/modules/gcs/examples/iam-authoritative.yaml +++ b/tests/modules/gcs/examples/iam-authoritative.yaml @@ -14,7 +14,6 @@ values: module.bucket.google_storage_bucket.bucket: - autoclass: [] cors: [] custom_placement_config: [] default_event_based_hold: null @@ -46,4 +45,4 @@ counts: google_storage_bucket: 1 google_storage_bucket_iam_binding: 1 modules: 1 - resources: 2 \ No newline at end of file + resources: 2 diff --git a/tests/modules/gcs/examples/iam-bindings-additive.yaml b/tests/modules/gcs/examples/iam-bindings-additive.yaml index 0c5f23112b..3419114550 100644 --- a/tests/modules/gcs/examples/iam-bindings-additive.yaml +++ b/tests/modules/gcs/examples/iam-bindings-additive.yaml @@ -14,7 +14,6 @@ values: module.bucket.google_storage_bucket.bucket: - autoclass: [] cors: [] custom_placement_config: [] default_event_based_hold: null @@ -48,4 +47,4 @@ counts: google_storage_bucket: 1 google_storage_bucket_iam_member: 1 modules: 1 - resources: 2 \ No newline at end of file + resources: 2 diff --git a/tests/modules/gcs/examples/iam-bindings.yaml b/tests/modules/gcs/examples/iam-bindings.yaml index 3111f57b3e..23d98f7290 100644 --- a/tests/modules/gcs/examples/iam-bindings.yaml +++ b/tests/modules/gcs/examples/iam-bindings.yaml @@ -14,7 +14,6 @@ values: module.bucket.google_storage_bucket.bucket: - autoclass: [] cors: [] custom_placement_config: [] default_event_based_hold: null @@ -49,4 +48,4 @@ counts: google_storage_bucket: 1 google_storage_bucket_iam_binding: 1 modules: 1 - resources: 2 \ No newline at end of file + resources: 2 diff --git a/tests/modules/secret_manager/examples/secret-cmek.yaml b/tests/modules/secret_manager/examples/secret-cmek.yaml index bfeec67f4a..c793b782f4 100644 --- a/tests/modules/secret_manager/examples/secret-cmek.yaml +++ b/tests/modules/secret_manager/examples/secret-cmek.yaml @@ -71,5 +71,3 @@ counts: resources: 11 outputs: {} - -outputs: {} \ No newline at end of file From fbdcbee0825cafa43ad8cb416514a529879539e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wiktor=20Niesiob=C4=99dzki?= Date: Thu, 30 May 2024 14:55:03 +0000 Subject: [PATCH 3/3] Fix boilerplate check --- .pre-commit-config.yaml | 2 ++ tools/check_boilerplate.py | 50 +++++++++++++++++++++++--------------- tools/pre-commit-tfdoc.sh | 16 +++++++++++- 3 files changed, 47 insertions(+), 21 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 54d149a167..e4b2263e78 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -41,6 +41,8 @@ repos: language: system entry: tools/check_boilerplate.py pass_filenames: true + args: + - --scan-files - id: tflint-fast name: Checking FAST code with tflint entry: tools/tflint-fast.py diff --git a/tools/check_boilerplate.py b/tools/check_boilerplate.py index a0f3ad8730..6cd222c28e 100755 --- a/tools/check_boilerplate.py +++ b/tools/check_boilerplate.py @@ -24,11 +24,12 @@ triggered by pull requests. ''' -import glob import os import re import sys +import click + _EXCLUDE_DIRS = ('.git', '.terraform') _EXCLUDE_RE = re.compile(r'# skip boilerplate check') _MATCH_FILES = ('Dockerfile', '.py', '.sh', '.tf', '.yaml', '.yml') @@ -38,23 +39,34 @@ _MATCH_RE = re.compile(_MATCH_STRING, re.M) -def main(base_dirs): - "Cycle through files in base_dirs and check for the Apache 2.0 boilerplate." +def check_files(root, files, errors, warnings): + for fname in files: + if fname in _MATCH_FILES or os.path.splitext(fname)[1] in _MATCH_FILES: + fpath = os.path.abspath(os.path.join(root, fname)) + content = open(fpath).read() + if _EXCLUDE_RE.search(content): + continue + try: + if not _MATCH_RE.search(content): + errors.append(fpath) + except (IOError, OSError): + warnings.append(fpath) + + +@click.command() +@click.argument('paths', type=str, nargs=-1) +@click.option('--scan-files', default=False, is_flag=True) +def main(paths, scan_files=False): + "Cycle through files in paths and check for the Apache 2.0 boilerplate." errors, warnings = [], [] - for dir in base_dirs: - for root, dirs, files in os.walk(dir): - dirs[:] = [d for d in dirs if d not in _EXCLUDE_DIRS] - for fname in files: - if fname in _MATCH_FILES or os.path.splitext(fname)[1] in _MATCH_FILES: - fpath = os.path.abspath(os.path.join(root, fname)) - content = open(fpath).read() - if _EXCLUDE_RE.search(content): - continue - try: - if not _MATCH_RE.search(content): - errors.append(fpath) - except (IOError, OSError): - warnings.append(fpath) + if scan_files: + check_files("./", paths, errors, warnings) + else: + for dir in paths: + for root, dirs, files in os.walk(dir): + dirs[:] = [d for d in dirs if d not in _EXCLUDE_DIRS] + check_files(root, files, errors, warnings) + if warnings: print('The following files cannot be accessed:') print('\n'.join(' - {}'.format(s) for s in warnings)) @@ -65,6 +77,4 @@ def main(base_dirs): if __name__ == '__main__': - if len(sys.argv) < 2: - raise SystemExit('No directory to check.') - main(sys.argv[1:]) + main() diff --git a/tools/pre-commit-tfdoc.sh b/tools/pre-commit-tfdoc.sh index 2f894a6087..21d1876744 100755 --- a/tools/pre-commit-tfdoc.sh +++ b/tools/pre-commit-tfdoc.sh @@ -1,6 +1,20 @@ #!/bin/bash -set -ev +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)