Even more scrubbing (#5152)

.changes/unreleased/Fixes-20220425-203924.yaml

@@ -0,0 +1,7 @@
+kind: Fixes
+body: Scrub secret env vars from CommandError in exception stacktrace
+time: 2022-04-25T20:39:24.365495+02:00
+custom:
+  Author: jtcohen6
+  Issue: "5151"
+  PR: "5152"

core/dbt/clients/git.py

@@ -4,13 +4,20 @@
 from dbt.clients.system import run_cmd, rmdir
 from dbt.events.functions import fire_event
 from dbt.events.types import (
-    GitSparseCheckoutSubdirectory, GitProgressCheckoutRevision,
-    GitProgressUpdatingExistingDependency, GitProgressPullingNewDependency,
-    GitNothingToDo, GitProgressUpdatedCheckoutRange, GitProgressCheckedOutAt
+    GitSparseCheckoutSubdirectory,
+    GitProgressCheckoutRevision,
+    GitProgressUpdatingExistingDependency,
+    GitProgressPullingNewDependency,
+    GitNothingToDo,
+    GitProgressUpdatedCheckoutRange,
+    GitProgressCheckedOutAt,
 )
 from dbt.exceptions import (
-    CommandResultError, RuntimeException, bad_package_spec, raise_git_cloning_error,
-    raise_git_cloning_problem
+    CommandResultError,
+    RuntimeException,
+    bad_package_spec,
+    raise_git_cloning_error,
+    raise_git_cloning_problem,
 )
 from packaging import version
 
@@ -21,9 +28,9 @@ def _is_commit(revision: str) -> bool:
 
 
 def _raise_git_cloning_error(repo, revision, error):
-    stderr = error.stderr.decode('utf-8').strip()
-    if 'usage: git' in stderr:
-        stderr = stderr.split('\nusage: git')[0]
+    stderr = error.stderr.strip()
+    if "usage: git" in stderr:
+        stderr = stderr.split("\nusage: git")[0]
     if re.match("fatal: destination path '(.+)' already exists", stderr):
         raise_git_cloning_error(error)
 
@@ -34,48 +41,48 @@ def clone(repo, cwd, dirname=None, remove_git_dir=False, revision=None, subdirec
     has_revision = revision is not None
     is_commit = _is_commit(revision or "")
 
-    clone_cmd = ['git', 'clone', '--depth', '1']
+    clone_cmd = ["git", "clone", "--depth", "1"]
     if subdirectory:
         fire_event(GitSparseCheckoutSubdirectory(subdir=subdirectory))
-        out, _ = run_cmd(cwd, ['git', '--version'], env={'LC_ALL': 'C'})
+        out, _ = run_cmd(cwd, ["git", "--version"], env={"LC_ALL": "C"})
         git_version = version.parse(re.search(r"\d+\.\d+\.\d+", out.decode("utf-8")).group(0))
         if not git_version >= version.parse("2.25.0"):
             # 2.25.0 introduces --sparse
             raise RuntimeError(
                 "Please update your git version to pull a dbt package "
                 "from a subdirectory: your version is {}, >= 2.25.0 needed".format(git_version)
             )
-        clone_cmd.extend(['--filter=blob:none', '--sparse'])
+        clone_cmd.extend(["--filter=blob:none", "--sparse"])
 
     if has_revision and not is_commit:
-        clone_cmd.extend(['--branch', revision])
+        clone_cmd.extend(["--branch", revision])
 
     clone_cmd.append(repo)
 
     if dirname is not None:
         clone_cmd.append(dirname)
     try:
-        result = run_cmd(cwd, clone_cmd, env={'LC_ALL': 'C'})
+        result = run_cmd(cwd, clone_cmd, env={"LC_ALL": "C"})
     except CommandResultError as exc:
         _raise_git_cloning_error(repo, revision, exc)
 
     if subdirectory:
-        cwd_subdir = os.path.join(cwd, dirname or '')
-        clone_cmd_subdir = ['git', 'sparse-checkout', 'set', subdirectory]
+        cwd_subdir = os.path.join(cwd, dirname or "")
+        clone_cmd_subdir = ["git", "sparse-checkout", "set", subdirectory]
         try:
             run_cmd(cwd_subdir, clone_cmd_subdir)
         except CommandResultError as exc:
             _raise_git_cloning_error(repo, revision, exc)
 
     if remove_git_dir:
-        rmdir(os.path.join(dirname, '.git'))
+        rmdir(os.path.join(dirname, ".git"))
 
     return result
 
 
 def list_tags(cwd):
-    out, err = run_cmd(cwd, ['git', 'tag', '--list'], env={'LC_ALL': 'C'})
-    tags = out.decode('utf-8').strip().split("\n")
+    out, err = run_cmd(cwd, ["git", "tag", "--list"], env={"LC_ALL": "C"})
+    tags = out.decode("utf-8").strip().split("\n")
     return tags
 
 
@@ -87,44 +94,44 @@ def _checkout(cwd, repo, revision):
     if _is_commit(revision):
         run_cmd(cwd, fetch_cmd + [revision])
     else:
-        run_cmd(cwd, ['git', 'remote', 'set-branches', 'origin', revision])
+        run_cmd(cwd, ["git", "remote", "set-branches", "origin", revision])
         run_cmd(cwd, fetch_cmd + ["--tags", revision])
 
     if _is_commit(revision):
         spec = revision
     # Prefer tags to branches if one exists
     elif revision in list_tags(cwd):
-        spec = 'tags/{}'.format(revision)
+        spec = "tags/{}".format(revision)
     else:
-        spec = 'origin/{}'.format(revision)
+        spec = "origin/{}".format(revision)
 
-    out, err = run_cmd(cwd, ['git', 'reset', '--hard', spec],
-                       env={'LC_ALL': 'C'})
+    out, err = run_cmd(cwd, ["git", "reset", "--hard", spec], env={"LC_ALL": "C"})
     return out, err
 
 
 def checkout(cwd, repo, revision=None):
     if revision is None:
-        revision = 'HEAD'
+        revision = "HEAD"
     try:
         return _checkout(cwd, repo, revision)
     except CommandResultError as exc:
-        stderr = exc.stderr.decode('utf-8').strip()
-    bad_package_spec(repo, revision, stderr)
+        stderr = exc.stderr.strip()
+        bad_package_spec(repo, revision, stderr)
 
 
 def get_current_sha(cwd):
-    out, err = run_cmd(cwd, ['git', 'rev-parse', 'HEAD'], env={'LC_ALL': 'C'})
+    out, err = run_cmd(cwd, ["git", "rev-parse", "HEAD"], env={"LC_ALL": "C"})
 
-    return out.decode('utf-8')
+    return out.decode("utf-8")
 
 
 def remove_remote(cwd):
-    return run_cmd(cwd, ['git', 'remote', 'rm', 'origin'], env={'LC_ALL': 'C'})
+    return run_cmd(cwd, ["git", "remote", "rm", "origin"], env={"LC_ALL": "C"})
 
 
-def clone_and_checkout(repo, cwd, dirname=None, remove_git_dir=False,
-                       revision=None, subdirectory=None):
+def clone_and_checkout(
+    repo, cwd, dirname=None, remove_git_dir=False, revision=None, subdirectory=None
+):
     exists = None
     try:
         _, err = clone(
@@ -135,7 +142,7 @@ def clone_and_checkout(repo, cwd, dirname=None, remove_git_dir=False,
             subdirectory=subdirectory,
         )
     except CommandResultError as exc:
-        err = exc.stderr.decode('utf-8')
+        err = exc.stderr
         exists = re.match("fatal: destination path '(.+)' already exists", err)
         if not exists:
             raise_git_cloning_problem(repo)
@@ -146,11 +153,9 @@ def clone_and_checkout(repo, cwd, dirname=None, remove_git_dir=False,
         directory = exists.group(1)
         fire_event(GitProgressUpdatingExistingDependency(dir=directory))
     else:
-        matches = re.match("Cloning into '(.+)'", err.decode('utf-8'))
+        matches = re.match("Cloning into '(.+)'", err.decode("utf-8"))
         if matches is None:
-            raise RuntimeException(
-                f'Error cloning {repo} - never saw "Cloning into ..." from git'
-            )
+            raise RuntimeException(f'Error cloning {repo} - never saw "Cloning into ..." from git')
         directory = matches.group(1)
         fire_event(GitProgressPullingNewDependency(dir=directory))
     full_path = os.path.join(cwd, directory)
@@ -161,9 +166,9 @@ def clone_and_checkout(repo, cwd, dirname=None, remove_git_dir=False,
         if start_sha == end_sha:
             fire_event(GitNothingToDo(sha=start_sha[:7]))
         else:
-            fire_event(GitProgressUpdatedCheckoutRange(
-                start_sha=start_sha[:7], end_sha=end_sha[:7]
-            ))
+            fire_event(
+                GitProgressUpdatedCheckoutRange(start_sha=start_sha[:7], end_sha=end_sha[:7])
+            )
     else:
         fire_event(GitProgressCheckedOutAt(end_sha=end_sha[:7]))
-    return os.path.join(directory, subdirectory or '')
+    return os.path.join(directory, subdirectory or "")