multifile grep: perform greps in series

Passing a very long argument list to git-grep can cause it to fail;
indeed, it's possible for the list of paths passed by git-secrets to
either grep or git-grep to exceed the maximum number of arguments
allowed in a user's environment (`getconf ARG_MAX`). Instead, let xargs
check that the number of arguments won't exceed the system limit.

Signed-off-by: Emily Shaffer <[email protected]>

git-secrets

@@ -111,18 +111,63 @@ scan_history() {
 git_grep() {
   local options="$1"; shift
   local files=("${@}") combined_patterns=$(load_combined_patterns)
+  local status=0
 
   [ -z "${combined_patterns}" ] && return 1
-  GREP_OPTIONS= LC_ALL=C git grep -nwHEI ${options} "${combined_patterns}" -- "${files[@]}"
+
+  if [ ${#files[@]} -eq 0 ]; then
+    GREP_OPTIONS= LC_ALL=C git grep -nwHEI ${options} "${combined_patterns}"
+    return $?
+  fi
+
+  # let xargs watch for system limit on arg count for us. xargs returns 123 if
+  # any call returned 1, but we care that all calls returned 1, so invert the
+  # output - xargs will return 0 if every call returned 0
+  printf "%s\n" "${files[@]}" |
+      GREP_OPTIONS= LC_ALL=C xargs -P "$(nproc)" -d'\n' sh -c \
+      'git grep "$@"; [ $? -eq 1 ]' - \
+	-nwHEI "${options}" "${combined_patterns}" --
+
+  # uninvert xargs's 0, which means all git grep invocations returned 1.
+  [ "$?" -ne 0 ]
 }
 
 # Performs a regular grep, taking into account patterns and recursion.
 # Note: this function returns 1 on success, 0 on error.
 regular_grep() {
   local files=("${@}") patterns=$(load_patterns) action='skip'
+  local status=0
   [ -z "${patterns}" ] && return 1
   [ ${RECURSIVE} -eq 1 ] && action="recurse"
-  GREP_OPTIONS= LC_ALL=C grep -d "${action}" -nwHEI "${patterns}" "${files[@]}"
+
+  if [ "${files[@]}" = "-" ]; then
+    GREP_OPTIONS= LC_ALL=C grep -d "${action}" -nwHEI "${patterns}" -
+    return $?
+  fi
+
+  # let xargs watch for system limit on arg count for us. xargs returns 123 if
+  # any call returned 1-128, and 124 if any call returned 255. we care that all
+  # calls returned 1 (no result found), and grep returns 2 in an error; convert
+  # the error to one xargs can recognize and invert the result otherwise.
+  printf "%s\n" "${files[@]}" |
+      GREP_OPTIONS= LC_ALL=C xargs -P "$(nproc)" -d'\n' sh -c \
+      'grep "$@"
+       rc=$?
+       if [ $rc -eq 2 ]; then
+	 exit 255
+       fi
+       [ $rc -eq 1 ]' - \
+      -d "${action}" -nwHEI "${patterns}"
+  status=$?
+
+  # we fudged the output of grep to return 255 on error instead of 2; xargs
+  # reports a 255 as 124
+  if [ $status -eq 124 ]; then
+    return 2
+  fi
+
+  # xargs returns 0 if all invocations returned 0, and we inverted the output
+  [ $status -ne 0 ]
 }
 
 # Process the given status ($1) and output variables ($2).

test/pre-commit.bats

@@ -60,3 +60,20 @@ load test_helper
   [ "${lines[1]}" == "failure1.txt:1:another line... forbidden" ]
   [ "${lines[2]}" == "failure2.txt:1:me" ]
 }
+
+@test "Runs safely with args beyond the system argument length limit" {
+  setup_good_repo
+  repo_run git-secrets --install $TEST_REPO
+  cd $TEST_REPO
+
+  FILENAME_LENGTH="$(getconf NAME_MAX .)"
+  (( FILE_COUNT = "$(getconf ARG_MAX)" / "$FILENAME_LENGTH" ))
+
+  for (( i = 0; i < "$FILE_COUNT"; i++ )); do
+    >"$(printf "%0${FILENAME_LENGTH}d" "$i")"
+  done
+
+  run git add .
+  run git commit -m 'This is fine'
+  [ $status -eq 0 ]
+}