Env variable replacement for PRE_COMMIT + command in log (#4298)

* Env variable replacement for PRE_COMMIT Allow to replace an ENV var value with the value of another ENV var before calling a PRE_COMMAND (helps for tflint run from GitHub Enterprise) Fixes #2947 * [MegaLinter] Apply linters fixes * fix quick build * [MegaLinter] Apply linters fixes quick build * Display command log (truncated to 200 chars) even when LOG_LEVEL is not DEBUG * fixxxx quick build * fix quick build * again * [MegaLinter] Apply linters fixes quick build * grumf quick build * [MegaLinter] Apply linters fixes quick build * last fix ^^ quick build * change error color --------- Co-authored-by: nvuillam <[email protected]>
oxsecurity · Nov 23, 2024 · 09ab582 · 09ab582
1 parent e33c1c7
commit 09ab582
Show file tree

Hide file tree

Showing 10 changed files with 51 additions and 7 deletions.
diff --git a/.automation/test/pre-post-test/.mega-linter.yml b/.automation/test/pre-post-test/.mega-linter.yml
@@ -20,6 +20,12 @@ PRE_COMMANDS:
   - command: export MY_OUTPUT_VARIABLE="my output variable value" && export MY_OUTPUT_VARIABLE2="my output variable value2"
     output_variables: ["MY_OUTPUT_VARIABLE", "MY_OUTPUT_VARIABLE2"]
     cwd: "root"
+  - command: export MY_OUTPUT_VARIABLE_REPLACED="${MY_INPUT_VARIABLE}"
+    replacement_env_vars: 
+      - var_src: MY_INPUT_VARIABLE_REPLACEMENT
+        var_dest: MY_INPUT_VARIABLE
+    output_variables: ["MY_OUTPUT_VARIABLE_REPLACED"]
+    cwd: "root"
 POST_COMMANDS:
   - command: npm run test
     cwd: "workspace"

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -9,6 +9,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 Note: Can be used with `oxsecurity/megalinter@beta` in your GitHub Action mega-linter.yml file, or with `oxsecurity/megalinter:beta` docker image
 
 - Core
+  - Display command log (truncated to 250 chars) even when LOG_LEVEL is not DEBUG
+  - Allow to replace an ENV var value with the value of another ENV var before calling a PRE_COMMAND (helps for tflint run from GitHub Enterprise)
 
 - New linters
 

diff --git a/megalinter/Linter.py b/megalinter/Linter.py
@@ -146,7 +146,7 @@ def __init__(self, params=None, linter_config=None):
 
         self.report_folder = ""
         self.reporters = []
-        self.lint_command_log: list(str) | str | None = None
+        self.lint_command_log: list(str) = []
 
         # Initialize parameters
         default_params = {
@@ -986,7 +986,6 @@ def process_linter(self, file=None):
         command = self.build_lint_command(file)
         # Output command if debug mode
         logging.debug(f"[{self.linter_name}] command: {str(command)}")
-        self.lint_command_log = command
         # Run command via CLI
         return_code, return_output = self.execute_lint_command(command)
         logging.debug(
@@ -1004,6 +1003,7 @@ def execute_lint_command(self, command):
             "FORCE_COLOR": "0",
         }
         if isinstance(command, str):
+            self.lint_command_log.append(command)
             # Call linter with a sub-process
             process = subprocess.run(
                 command,
@@ -1028,7 +1028,7 @@ def execute_lint_command(self, command):
                     msg = "Unable to find command: " + command[0]
                     logging.error(msg)
                     return errno.ESRCH, msg
-
+            self.lint_command_log.append(" ".join(command))
             # Call linter with a sub-process (RECOMMENDED: with a list of strings corresponding to the command)
             try:
                 process = subprocess.run(

diff --git a/megalinter/descriptors/terraform.megalinter-descriptor.yml b/megalinter/descriptors/terraform.megalinter-descriptor.yml
@@ -24,6 +24,8 @@ linters:
     linter_text: |
       > If you are using the GitHub action please use the `TERRAFORM_TFLINT_UNSECURED_ENV_VARIABLES: GITHUB_TOKEN` to prevent plugin download issues
 
+      > If you have issues with tflint --init, create a GitHub Personal Access Token and set its value to PAT_GITHUB_COM variable.
+
       Note: It's recommended to create your own `.tflint.hcl` custom config file tailored to your project's specific needs.
       The default configuration enables all supported languages and rules, which may not be optimal for every project.
     linter_icon_png_url: https://raw.githubusercontent.com/oxsecurity/megalinter/main/docs/assets/icons/linters/tflint.png
@@ -38,6 +40,9 @@ linters:
       - name: TERRAFORM_TFLINT_SECURED_ENV
         default_value: true
         description: Allows to send the full env to **tflint --init**. Initialized with default value `true`. Set to `false` to allow `tflint --init` to access your env vars.
+      - name: PAT_GITHUB_COM
+        default_value: ""
+        description: If you have issues with tflint --init, create a GitHub Personal Access Token and set its value to PAT_GITHUB_COM variable.
     examples:
       - "tflint"
       - "tflint -c .tflint.hcl"

diff --git a/megalinter/linters/TfLintLinter.py b/megalinter/linters/TfLintLinter.py
@@ -24,10 +24,14 @@ def before_lint_files(self):
             == "false"
             else True
         )
+        replacement_def = dict(
+            {"var_dest": "GITHUB_TOKEN", "var_src": "PAT_GITHUB_COM"}
+        )
         tflint_pre_command = {
             "command": tflint_init_command,
             "cwd": self.workspace,
             "secured_env": tflint_secured_env,
+            "replacement_env_vars": [replacement_def],
         }
         if self.pre_commands is None:
             self.pre_commands = []

diff --git a/megalinter/pre_post_factory.py b/megalinter/pre_post_factory.py
@@ -98,6 +98,13 @@ def run_command(command_info, log_key, mega_linter, linter=None):
             mega_linter.request_id, command_info["secured_env"], unsecured_env_variables
         )
     }
+    # Complete with replacement variables if necessary
+    if "replacement_env_vars" in command_info:
+        for replacement in command_info["replacement_env_vars"]:
+            if replacement["var_src"] in subprocess_env:
+                var_src_name = replacement["var_src"]
+                var_dest_name = replacement["var_dest"]
+                subprocess_env[var_dest_name] = subprocess_env[var_src_name]
     add_in_logs(
         linter,
         log_key,

diff --git a/megalinter/reporters/ConsoleLinterReporter.py b/megalinter/reporters/ConsoleLinterReporter.py
@@ -102,6 +102,19 @@ def produce_report(self):
             msg += [
                 f"- Number of files analyzed: [{len(self.master.files_lint_results)}]"
             ]
+        # Command
+        if len(self.master.lint_command_log) == 1:
+            end = "" if len(self.master.lint_command_log[0]) < 250 else "...(truncated)"
+            msg += [f"- Command: [{self.master.lint_command_log[0][:250]}{end}]"]
+        elif len(self.master.lint_command_log) > 1:
+            msg += ["- Commands:"]
+            for command_log in self.master.lint_command_log:
+                end = (
+                    ""
+                    if len(self.master.lint_command_log[0]) < 250
+                    else "...(truncated)"
+                )
+                msg += [f"  [{command_log[:250]}{end}]"]
         logging.info("\n".join(msg))
         # Pre-commands logs
         if len(self.master.log_lines_pre) > 0:

diff --git a/megalinter/reporters/UpdatedSourcesReporter.py b/megalinter/reporters/UpdatedSourcesReporter.py
@@ -86,9 +86,9 @@ def produce_report(self):
                     if BITBUCKET_BRANCH != "":
                         remote_branch = BITBUCKET_BRANCH
                     if remote_branch == "":
-                        logging.error(
-                            c.red(
-                                "❌ [Updated Sources Reporter] Failed to retrieve git source branch"
+                        logging.warning(
+                            c.yellow(
+                                "⚠️ [Updated Sources Reporter] Failed to retrieve git source branch"
                             )
                         )
                     else:

diff --git a/megalinter/tests/test_megalinter/pre_post_test.py b/megalinter/tests/test_megalinter/pre_post_test.py
@@ -32,6 +32,8 @@ def test_pre_post_success(self):
                 "GITHUB_COMMENT_REPORTER": "false",
                 "LOG_LEVEL": "DEBUG",
                 "request_id": self.request_id,
+                "MY_INPUT_VARIABLE": "SHOULD_BE_REPLACED",
+                "MY_INPUT_VARIABLE_REPLACEMENT": "HAS_BEEN_REPLACED",
             }
         )
         self.assertTrue(
@@ -53,6 +55,11 @@ def test_pre_post_success(self):
             == "my output variable value2",
             "MY_OUTPUT_VARIABLE2 should be found",
         )
+        replaced_val = config.get(self.request_id, "MY_OUTPUT_VARIABLE_REPLACED", "")
+        self.assertTrue(
+            replaced_val == "HAS_BEEN_REPLACED",
+            f"MY_OUTPUT_VARIABLE_REPLACED has not been replaced (value: {replaced_val})",
+        )
         self.assertTrue(
             config.get(self.request_id, "MY_OUTPUT_LINTER_VARIABLE", "")
             == "my output linter variable value",

diff --git a/megalinter/utils_reporter.py b/megalinter/utils_reporter.py
@@ -289,7 +289,7 @@ def build_linter_reporter_external_result(reporter, redis_stream=False) -> dict:
         "linterStatusMessage": status_message,
         "linterElapsedTime": round(reporter.master.elapsed_time_s, 2),
     }
-    if reporter.master.lint_command_log is not None:
+    if len(reporter.master.lint_command_log) > 0:
         result["linterCliCommand"] = reporter.master.lint_command_log
     result = result | get_linter_infos(reporter.master)
     if (