From 09ab582dd64e53cc008dcbe40b4346b16c0f7dd1 Mon Sep 17 00:00:00 2001 From: Nicolas Vuillamy Date: Sat, 23 Nov 2024 01:08:17 +0100 Subject: [PATCH] Env variable replacement for PRE_COMMIT + command in log (#4298) * Env variable replacement for PRE_COMMIT Allow to replace an ENV var value with the value of another ENV var before calling a PRE_COMMAND (helps for tflint run from GitHub Enterprise) Fixes https://github.com/oxsecurity/megalinter/issues/2947 * [MegaLinter] Apply linters fixes * fix quick build * [MegaLinter] Apply linters fixes quick build * Display command log (truncated to 200 chars) even when LOG_LEVEL is not DEBUG * fixxxx quick build * fix quick build * again * [MegaLinter] Apply linters fixes quick build * grumf quick build * [MegaLinter] Apply linters fixes quick build * last fix ^^ quick build * change error color --------- Co-authored-by: nvuillam --- .automation/test/pre-post-test/.mega-linter.yml | 6 ++++++ CHANGELOG.md | 2 ++ megalinter/Linter.py | 6 +++--- .../descriptors/terraform.megalinter-descriptor.yml | 5 +++++ megalinter/linters/TfLintLinter.py | 4 ++++ megalinter/pre_post_factory.py | 7 +++++++ megalinter/reporters/ConsoleLinterReporter.py | 13 +++++++++++++ megalinter/reporters/UpdatedSourcesReporter.py | 6 +++--- megalinter/tests/test_megalinter/pre_post_test.py | 7 +++++++ megalinter/utils_reporter.py | 2 +- 10 files changed, 51 insertions(+), 7 deletions(-) diff --git a/.automation/test/pre-post-test/.mega-linter.yml b/.automation/test/pre-post-test/.mega-linter.yml index 4b8b09b113e..2787dea71aa 100644 --- a/.automation/test/pre-post-test/.mega-linter.yml +++ b/.automation/test/pre-post-test/.mega-linter.yml @@ -20,6 +20,12 @@ PRE_COMMANDS: - command: export MY_OUTPUT_VARIABLE="my output variable value" && export MY_OUTPUT_VARIABLE2="my output variable value2" output_variables: ["MY_OUTPUT_VARIABLE", "MY_OUTPUT_VARIABLE2"] cwd: "root" + - command: export MY_OUTPUT_VARIABLE_REPLACED="${MY_INPUT_VARIABLE}" + replacement_env_vars: + - var_src: MY_INPUT_VARIABLE_REPLACEMENT + var_dest: MY_INPUT_VARIABLE + output_variables: ["MY_OUTPUT_VARIABLE_REPLACED"] + cwd: "root" POST_COMMANDS: - command: npm run test cwd: "workspace" diff --git a/CHANGELOG.md b/CHANGELOG.md index ecc4a7a14d8..da999386989 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), Note: Can be used with `oxsecurity/megalinter@beta` in your GitHub Action mega-linter.yml file, or with `oxsecurity/megalinter:beta` docker image - Core + - Display command log (truncated to 250 chars) even when LOG_LEVEL is not DEBUG + - Allow to replace an ENV var value with the value of another ENV var before calling a PRE_COMMAND (helps for tflint run from GitHub Enterprise) - New linters diff --git a/megalinter/Linter.py b/megalinter/Linter.py index 99ba844d8b9..c21f6f0fc2d 100644 --- a/megalinter/Linter.py +++ b/megalinter/Linter.py @@ -146,7 +146,7 @@ def __init__(self, params=None, linter_config=None): self.report_folder = "" self.reporters = [] - self.lint_command_log: list(str) | str | None = None + self.lint_command_log: list(str) = [] # Initialize parameters default_params = { @@ -986,7 +986,6 @@ def process_linter(self, file=None): command = self.build_lint_command(file) # Output command if debug mode logging.debug(f"[{self.linter_name}] command: {str(command)}") - self.lint_command_log = command # Run command via CLI return_code, return_output = self.execute_lint_command(command) logging.debug( @@ -1004,6 +1003,7 @@ def execute_lint_command(self, command): "FORCE_COLOR": "0", } if isinstance(command, str): + self.lint_command_log.append(command) # Call linter with a sub-process process = subprocess.run( command, @@ -1028,7 +1028,7 @@ def execute_lint_command(self, command): msg = "Unable to find command: " + command[0] logging.error(msg) return errno.ESRCH, msg - + self.lint_command_log.append(" ".join(command)) # Call linter with a sub-process (RECOMMENDED: with a list of strings corresponding to the command) try: process = subprocess.run( diff --git a/megalinter/descriptors/terraform.megalinter-descriptor.yml b/megalinter/descriptors/terraform.megalinter-descriptor.yml index 31a12d164a7..854a398a833 100644 --- a/megalinter/descriptors/terraform.megalinter-descriptor.yml +++ b/megalinter/descriptors/terraform.megalinter-descriptor.yml @@ -24,6 +24,8 @@ linters: linter_text: | > If you are using the GitHub action please use the `TERRAFORM_TFLINT_UNSECURED_ENV_VARIABLES: GITHUB_TOKEN` to prevent plugin download issues + > If you have issues with tflint --init, create a GitHub Personal Access Token and set its value to PAT_GITHUB_COM variable. + Note: It's recommended to create your own `.tflint.hcl` custom config file tailored to your project's specific needs. The default configuration enables all supported languages and rules, which may not be optimal for every project. linter_icon_png_url: https://raw.githubusercontent.com/oxsecurity/megalinter/main/docs/assets/icons/linters/tflint.png @@ -38,6 +40,9 @@ linters: - name: TERRAFORM_TFLINT_SECURED_ENV default_value: true description: Allows to send the full env to **tflint --init**. Initialized with default value `true`. Set to `false` to allow `tflint --init` to access your env vars. + - name: PAT_GITHUB_COM + default_value: "" + description: If you have issues with tflint --init, create a GitHub Personal Access Token and set its value to PAT_GITHUB_COM variable. examples: - "tflint" - "tflint -c .tflint.hcl" diff --git a/megalinter/linters/TfLintLinter.py b/megalinter/linters/TfLintLinter.py index a8168356526..620199afa5e 100644 --- a/megalinter/linters/TfLintLinter.py +++ b/megalinter/linters/TfLintLinter.py @@ -24,10 +24,14 @@ def before_lint_files(self): == "false" else True ) + replacement_def = dict( + {"var_dest": "GITHUB_TOKEN", "var_src": "PAT_GITHUB_COM"} + ) tflint_pre_command = { "command": tflint_init_command, "cwd": self.workspace, "secured_env": tflint_secured_env, + "replacement_env_vars": [replacement_def], } if self.pre_commands is None: self.pre_commands = [] diff --git a/megalinter/pre_post_factory.py b/megalinter/pre_post_factory.py index ae13e294764..076dc2b0b60 100644 --- a/megalinter/pre_post_factory.py +++ b/megalinter/pre_post_factory.py @@ -98,6 +98,13 @@ def run_command(command_info, log_key, mega_linter, linter=None): mega_linter.request_id, command_info["secured_env"], unsecured_env_variables ) } + # Complete with replacement variables if necessary + if "replacement_env_vars" in command_info: + for replacement in command_info["replacement_env_vars"]: + if replacement["var_src"] in subprocess_env: + var_src_name = replacement["var_src"] + var_dest_name = replacement["var_dest"] + subprocess_env[var_dest_name] = subprocess_env[var_src_name] add_in_logs( linter, log_key, diff --git a/megalinter/reporters/ConsoleLinterReporter.py b/megalinter/reporters/ConsoleLinterReporter.py index e1b56ae7e12..e53b34e4389 100644 --- a/megalinter/reporters/ConsoleLinterReporter.py +++ b/megalinter/reporters/ConsoleLinterReporter.py @@ -102,6 +102,19 @@ def produce_report(self): msg += [ f"- Number of files analyzed: [{len(self.master.files_lint_results)}]" ] + # Command + if len(self.master.lint_command_log) == 1: + end = "" if len(self.master.lint_command_log[0]) < 250 else "...(truncated)" + msg += [f"- Command: [{self.master.lint_command_log[0][:250]}{end}]"] + elif len(self.master.lint_command_log) > 1: + msg += ["- Commands:"] + for command_log in self.master.lint_command_log: + end = ( + "" + if len(self.master.lint_command_log[0]) < 250 + else "...(truncated)" + ) + msg += [f" [{command_log[:250]}{end}]"] logging.info("\n".join(msg)) # Pre-commands logs if len(self.master.log_lines_pre) > 0: diff --git a/megalinter/reporters/UpdatedSourcesReporter.py b/megalinter/reporters/UpdatedSourcesReporter.py index abf5c4c347c..abbb03627f0 100644 --- a/megalinter/reporters/UpdatedSourcesReporter.py +++ b/megalinter/reporters/UpdatedSourcesReporter.py @@ -86,9 +86,9 @@ def produce_report(self): if BITBUCKET_BRANCH != "": remote_branch = BITBUCKET_BRANCH if remote_branch == "": - logging.error( - c.red( - "❌ [Updated Sources Reporter] Failed to retrieve git source branch" + logging.warning( + c.yellow( + "⚠️ [Updated Sources Reporter] Failed to retrieve git source branch" ) ) else: diff --git a/megalinter/tests/test_megalinter/pre_post_test.py b/megalinter/tests/test_megalinter/pre_post_test.py index cfcd1be06af..8661a9f3797 100644 --- a/megalinter/tests/test_megalinter/pre_post_test.py +++ b/megalinter/tests/test_megalinter/pre_post_test.py @@ -32,6 +32,8 @@ def test_pre_post_success(self): "GITHUB_COMMENT_REPORTER": "false", "LOG_LEVEL": "DEBUG", "request_id": self.request_id, + "MY_INPUT_VARIABLE": "SHOULD_BE_REPLACED", + "MY_INPUT_VARIABLE_REPLACEMENT": "HAS_BEEN_REPLACED", } ) self.assertTrue( @@ -53,6 +55,11 @@ def test_pre_post_success(self): == "my output variable value2", "MY_OUTPUT_VARIABLE2 should be found", ) + replaced_val = config.get(self.request_id, "MY_OUTPUT_VARIABLE_REPLACED", "") + self.assertTrue( + replaced_val == "HAS_BEEN_REPLACED", + f"MY_OUTPUT_VARIABLE_REPLACED has not been replaced (value: {replaced_val})", + ) self.assertTrue( config.get(self.request_id, "MY_OUTPUT_LINTER_VARIABLE", "") == "my output linter variable value", diff --git a/megalinter/utils_reporter.py b/megalinter/utils_reporter.py index d905a14ff18..50e2d318da2 100644 --- a/megalinter/utils_reporter.py +++ b/megalinter/utils_reporter.py @@ -289,7 +289,7 @@ def build_linter_reporter_external_result(reporter, redis_stream=False) -> dict: "linterStatusMessage": status_message, "linterElapsedTime": round(reporter.master.elapsed_time_s, 2), } - if reporter.master.lint_command_log is not None: + if len(reporter.master.lint_command_log) > 0: result["linterCliCommand"] = reporter.master.lint_command_log result = result | get_linter_infos(reporter.master) if (