diff --git a/pw_presubmit/py/BUILD.gn b/pw_presubmit/py/BUILD.gn index 88c65fa8f6..81714d5d26 100644 --- a/pw_presubmit/py/BUILD.gn +++ b/pw_presubmit/py/BUILD.gn @@ -24,6 +24,7 @@ pw_python_package("py") { ] sources = [ "pw_presubmit/__init__.py", + "pw_presubmit/bazel_parser.py", "pw_presubmit/build.py", "pw_presubmit/cli.py", "pw_presubmit/cpp_checks.py", @@ -44,6 +45,7 @@ pw_python_package("py") { "pw_presubmit/tools.py", ] tests = [ + "bazel_parser_test.py", "git_repo_test.py", "gitmodules_test.py", "keep_sorted_test.py", diff --git a/pw_presubmit/py/bazel_parser_test.py b/pw_presubmit/py/bazel_parser_test.py new file mode 100644 index 0000000000..1b8ddbf41d --- /dev/null +++ b/pw_presubmit/py/bazel_parser_test.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 +# Copyright 2022 The Pigweed Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +"""Tests for bazel_parser.""" + +from pathlib import Path +import tempfile +import unittest + +from pw_presubmit import bazel_parser + +# This is a real Bazel failure, trimmed slightly. +_REAL_TEST_INPUT = """ +Starting local Bazel server and connecting to it... +WARNING: --verbose_explanations has no effect when --explain= is not enabled +Loading: +Loading: 0 packages loaded +Analyzing: 1362 targets (197 packages loaded) +Analyzing: 1362 targets (197 packages loaded, 0 targets configured) +INFO: Analyzed 1362 targets (304 packages loaded, 15546 targets configured). + +INFO: Found 1362 targets... +[6 / 124] [Prepa] BazelWorkspaceStatusAction stable-status.txt +[747 / 1,548] Compiling pw_kvs/entry.cc; 0s linux-sandbox ... (3 actions, ... +ERROR: /usr/local/google/home/mohrr/pigweed/pigweed/pw_kvs/BUILD.bazel:25:14: Compiling pw_kvs/entry.cc failed: (Exit 1): gcc failed: error executing command + (cd /usr/local/google/home/mohrr/.cache/bazel/_bazel_mohrr/7e133e1f95b61... \ + exec env - \ + CPP_TOOL_PATH=external/clang_llvm_12_00_x86_64_linux_gnu_ubuntu_16_04/... \ + GCC_TOOL_PATH=external/clang_llvm_12_00_x86_64_linux_gnu_ubuntu_16_04/... \ + LD_TOOL_PATH=external/clang_llvm_12_00_x86_64_linux_gnu_ubuntu_16_04/... \ + NM_TOOL_PATH=external/clang_llvm_12_00_x86_64_linux_gnu_ubuntu_16_04/... \ + OBJDUMP_TOOL_PATH=external/clang_llvm_12_00_x86_64_linux_gnu_ubuntu_16_... \ + PATH=/usr/local/google/home/mohrr/pigweed/pigweed/out/host/host_tools:... \ + PWD=/proc/self/cwd \ + external/rules_cc_toolchain/cc_toolchain/wrappers/posix/gcc -MD -MF bazel-out/k8-fastbuild/bin/pw_kvs/_objs/pw_kvs/entry.pic.d '-frandom-seed=bazel-out/k8-fastbuild/bin/pw_kvs/_objs/pw_kvs/entry.pic.o' -fPIC -iquote . -iquote bazel-out/k8-fastbuild/bin -isystem pw_kvs/public -isystem bazel-out/k8-fastbuild/bin/pw_kvs/public -isystem pw_assert/assert_compatibility_public_overrides -isystem bazel-out/k8-fastbuild/bin/pw_assert/assert_compatibility_public_overrides -isystem pw_assert/public -isystem bazel-out/k8-fastbuild/bin/pw_assert/public -isystem pw_preprocessor/public -isystem bazel-out/k8-fastbuild/bin/pw_preprocessor/public -isystem pw_assert_basic/public -isystem bazel-out/k8-fastbuild/bin/pw_assert_basic/public -isystem pw_assert_basic/public_overrides -isystem bazel-out/k8-fastbuild/bin/pw_assert_basic/public_overrides -isystem pw_string/public -isystem bazel-out/k8-fastbuild/bin/pw_string/public -isystem pw_span/public -isystem bazel-out/k8-fastbuild/bin/pw_span/public -isystem pw_polyfill/public -isystem bazel-out/k8-fastbuild/bin/pw_polyfill/public -isystem pw_status/public -isystem bazel-out/k8-fastbuild/bin/pw_status/public -isystem pw_result/public -isystem bazel-out/k8-fastbuild/bin/pw_result/public -isystem pw_sys_io/public -isystem bazel-out/k8-fastbuild/bin/pw_sys_io/public -isystem pw_bytes/public -isystem bazel-out/k8-fastbuild/bin/pw_bytes/public -isystem pw_containers/public -isystem bazel-out/k8-fastbuild/bin/pw_containers/public -isystem pw_checksum/public -isystem bazel-out/k8-fastbuild/bin/pw_checksum/public -isystem pw_compilation_testing/public -isystem bazel-out/k8-fastbuild/bin/pw_compilation_testing/public -isystem pw_log/public -isystem bazel-out/k8-fastbuild/bin/pw_log/public -isystem pw_log_basic/public -isystem bazel-out/k8-fastbuild/bin/pw_log_basic/public -isystem pw_log_basic/public_overrides -isystem bazel-out/k8-fastbuild/bin/pw_log_basic/public_overrides -isystem pw_stream/public -isystem bazel-out/k8-fastbuild/bin/pw_stream/public -nostdinc++ -nostdinc -isystemexternal/clang_llvm_12_00_x86_64_linux_gnu_ubuntu_16_04/include/c++/v1 -isystemexternal/debian_stretch_amd64_sysroot/usr/local/include -isystemexternal/debian_stretch_amd64_sysroot/usr/include/x86_64-linux-gnu -isystemexternal/debian_stretch_amd64_sysroot/usr/include -isystemexternal/clang_llvm_12_00_x86_64_linux_gnu_ubuntu_16_04/lib/clang/12.0.0 -isystemexternal/clang_llvm_12_00_x86_64_linux_gnu_ubuntu_16_04/lib/clang/12.0.0/include -fdata-sections -ffunction-sections -no-canonical-prefixes -Wno-builtin-macro-redefined '-D__DATE__="redacted"' '-D__TIMESTAMP__="redacted"' '-D__TIME__="redacted"' -xc++ --sysroot external/debian_stretch_amd64_sysroot -O0 -fPIC -g -fno-common -fno-exceptions -ffunction-sections -fdata-sections -Wall -Wextra -Werror '-Wno-error=cpp' '-Wno-error=deprecated-declarations' -Wno-private-header '-std=c++17' -fno-rtti -Wnon-virtual-dtor -Wno-register -c pw_kvs/entry.cc -o bazel-out/k8-fastbuild/bin/pw_kvs/_objs/pw_kvs/entry.pic.o) +# Configuration: 752863e407a197a5b9da05cfc572e7013efd6958e856cee61d2fa474ed... +# Execution platform: @local_config_platform//:host + +Use --sandbox_debug to see verbose messages from the sandbox and retain the sandbox build root for debugging +pw_kvs/entry.cc:49:20: error: no member named 'Dat' in 'pw::Status' + return Status::Dat aLoss(); + ~~~~~~~~^ +pw_kvs/entry.cc:49:23: error: expected ';' after return statement + return Status::Dat aLoss(); + ^ + ; +2 errors generated. +INFO: Elapsed time: 5.662s, Critical Path: 1.01s +INFO: 12 processes: 12 internal. +FAILED: Build did NOT complete successfully +FAILED: Build did NOT complete successfully +""" + +_REAL_TEST_SUMMARY = """ +ERROR: /usr/local/google/home/mohrr/pigweed/pigweed/pw_kvs/BUILD.bazel:25:14: Compiling pw_kvs/entry.cc failed: (Exit 1): gcc failed: error executing command +# Configuration: 752863e407a197a5b9da05cfc572e7013efd6958e856cee61d2fa474ed... +# Execution platform: @local_config_platform//:host + +Use --sandbox_debug to see verbose messages from the sandbox and retain the sandbox build root for debugging +pw_kvs/entry.cc:49:20: error: no member named 'Dat' in 'pw::Status' + return Status::Dat aLoss(); + ~~~~~~~~^ +pw_kvs/entry.cc:49:23: error: expected ';' after return statement + return Status::Dat aLoss(); + ^ + ; +2 errors generated. +""" + +_STOP = 'INFO:\n' + +# pylint: disable=attribute-defined-outside-init + + +class TestBazelParser(unittest.TestCase): + """Test bazel_parser.""" + + def _run(self, contents: str) -> None: + with tempfile.TemporaryDirectory() as tempdir: + path = Path(tempdir) / 'foo' + + with path.open('w') as outs: + outs.write(contents) + + self.output = bazel_parser.parse_bazel_stdout(path) + + def test_simple(self) -> None: + error = 'ERROR: abc\nerror 1\nerror2\n' + self._run('[0/10] foo\n[1/10] bar\n' + error + _STOP) + self.assertEqual(error.strip(), self.output.strip()) + + def test_path(self) -> None: + error_in = 'ERROR: abc\n PATH=... \\\nerror 1\nerror2\n' + error_out = 'ERROR: abc\nerror 1\nerror2\n' + self._run('[0/10] foo\n[1/10] bar\n' + error_in + _STOP) + self.assertEqual(error_out.strip(), self.output.strip()) + + def test_unterminated(self) -> None: + error = 'ERROR: abc\nerror 1\nerror 2\n' + self._run('[0/10] foo\n[1/10] bar\n' + error) + self.assertEqual(error.strip(), self.output.strip()) + + def test_failure(self) -> None: + self._run(_REAL_TEST_INPUT) + self.assertEqual(_REAL_TEST_SUMMARY.strip(), self.output.strip()) + + +if __name__ == '__main__': + unittest.main() diff --git a/pw_presubmit/py/pw_presubmit/bazel_parser.py b/pw_presubmit/py/pw_presubmit/bazel_parser.py new file mode 100644 index 0000000000..865e8a50e3 --- /dev/null +++ b/pw_presubmit/py/pw_presubmit/bazel_parser.py @@ -0,0 +1,56 @@ +# Copyright 2022 The Pigweed Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +"""Extracts a concise error from a bazel log.""" + +from pathlib import Path +import re +import sys + + +def parse_bazel_stdout(bazel_stdout: Path) -> str: + """Extracts a concise error from a bazel log.""" + seen_error = False + error_lines = [] + + with bazel_stdout.open() as ins: + for line in ins: + # Trailing whitespace isn't significant, as it doesn't affect the + # way the line shows up in the logs. However, leading whitespace may + # be significant, especially for compiler error messages. + line = line.rstrip() + + if re.match(r'^ERROR:', line): + seen_error = True + + if seen_error: + # Ignore long lines that just show the environment. + if re.search(r' +[\w_]*(PATH|PWD)=.* \\', line): + continue + + # Ignore lines that only show bazel sandboxing. + if line.strip().startswith(('(cd /', 'exec env')): + continue + + # An ":" line usually means compiler output is done + # and useful compiler errors are complete. + if re.match(r'^(?!ERROR)[A-Z]+:', line): + break + + error_lines.append(line) + + return '\n'.join(error_lines) + + +if __name__ == '__main__': + print(parse_bazel_stdout(Path(sys.argv[1]))) diff --git a/pw_presubmit/py/pw_presubmit/build.py b/pw_presubmit/py/pw_presubmit/build.py index 3480fd56e3..0f46e9d55a 100644 --- a/pw_presubmit/py/pw_presubmit/build.py +++ b/pw_presubmit/py/pw_presubmit/build.py @@ -38,6 +38,7 @@ from pw_package import package_manager from pw_presubmit import ( + bazel_parser, call, Check, FileFilter, @@ -68,19 +69,31 @@ def bazel(ctx: PresubmitContext, cmd: str, *args: str) -> None: if ctx.continue_after_build_error: keep_going.append('--keep_going') - call( - 'bazel', - cmd, - '--verbose_failures', - '--verbose_explanations', - '--worker_verbose', - f'--symlink_prefix={ctx.output_dir / ".bazel-"}', - *num_jobs, - *keep_going, - *args, - cwd=ctx.root, - env=env_with_clang_vars(), - ) + bazel_stdout = ctx.output_dir / 'bazel.stdout' + try: + with bazel_stdout.open('w') as outs: + call( + 'bazel', + cmd, + '--verbose_failures', + '--verbose_explanations', + '--worker_verbose', + f'--symlink_prefix={ctx.output_dir / ".bazel-"}', + *num_jobs, + *keep_going, + *args, + cwd=ctx.root, + env=env_with_clang_vars(), + tee=outs, + ) + + except PresubmitFailure as exc: + failure = bazel_parser.parse_bazel_stdout(bazel_stdout) + if failure: + with ctx.failure_summary_log.open('w') as outs: + outs.write(failure) + + raise exc def install_package(ctx: PresubmitContext, name: str) -> None: