From 41b6486c9b24ef2582a74ae60eef9089ac8c7fb8 Mon Sep 17 00:00:00 2001 From: Yu Watanabe Date: Mon, 9 Dec 2024 04:18:17 +0900 Subject: [PATCH 1/5] mkosi: move drop-in config for sanitizers --- .../usr/lib/systemd/system/iscsi-init.service.d/10-asan.conf | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename mkosi.extra/usr/lib/systemd/system/iscsi-init.service.d/asan.conf => mkosi.sanitizers/mkosi.extra/usr/lib/systemd/system/iscsi-init.service.d/10-asan.conf (100%) diff --git a/mkosi.extra/usr/lib/systemd/system/iscsi-init.service.d/asan.conf b/mkosi.sanitizers/mkosi.extra/usr/lib/systemd/system/iscsi-init.service.d/10-asan.conf similarity index 100% rename from mkosi.extra/usr/lib/systemd/system/iscsi-init.service.d/asan.conf rename to mkosi.sanitizers/mkosi.extra/usr/lib/systemd/system/iscsi-init.service.d/10-asan.conf From 1bdb9e808f3c88ef4094fe8d7d36107d189f8ef8 Mon Sep 17 00:00:00 2001 From: Yu Watanabe Date: Sat, 7 Dec 2024 13:36:39 +0900 Subject: [PATCH 2/5] test: extract sanitizer reports from journal --- test/TEST-64-UDEV-STORAGE/meson.build | 3 + test/integration-test-wrapper.py | 134 +++++++++++++++++++++++++- test/meson.build | 2 + 3 files changed, 137 insertions(+), 2 deletions(-) diff --git a/test/TEST-64-UDEV-STORAGE/meson.build b/test/TEST-64-UDEV-STORAGE/meson.build index 15981ce35f167..a91c23ae170a9 100644 --- a/test/TEST-64-UDEV-STORAGE/meson.build +++ b/test/TEST-64-UDEV-STORAGE/meson.build @@ -38,6 +38,9 @@ foreach testcase : [ ], 'priority' : 10, 'vm' : true, + # Suppress ASan error + # 'multipathd[1820]: ==1820==ERROR: AddressSanitizer: Joining already joined thread, aborting.' + 'sanitizer-exclude-regex' : 'multipathd' }, ] endforeach diff --git a/test/integration-test-wrapper.py b/test/integration-test-wrapper.py index c08f77043c436..d44e8f8bd674e 100755 --- a/test/integration-test-wrapper.py +++ b/test/integration-test-wrapper.py @@ -122,6 +122,127 @@ def process_coredumps(args: argparse.Namespace, journal_file: Path) -> bool: return True +def process_sanitizer_report(args: argparse.Namespace, journal_file: Path) -> bool: + # Collect sanitizer reports from the journal file. + + if args.sanitizer_exclude_regex: + exclude_regex = re.compile(args.sanitizer_exclude_regex) + else: + exclude_regex = None + + total = 0 + fatal = 0 + asan = 0 + ubsan = 0 + msan = 0 + + # Internal errors: + # ==2554==LeakSanitizer has encountered a fatal error. + # ==2554==HINT: For debugging, try setting environment variable LSAN_OPTIONS=verbosity=1:log_threads=1 + # ==2554==HINT: LeakSanitizer does not work under ptrace (strace, gdb, etc) + fatal_begin = re.compile(r'==[0-9]+==.+?\w+Sanitizer has encountered a fatal error') + fatal_end = re.compile(r'==[0-9]+==HINT:\s+\w+Sanitizer') + + # 'Standard' errors: + standard_begin = re.compile(r'([0-9]+: runtime error|==[0-9]+==.+?\w+Sanitizer)') + standard_end = re.compile(r'SUMMARY:\s+(\w+)Sanitizer') + + # extract COMM + find_comm = re.compile(r'^\[[.0-9 ]+?\]\s(.*?:)\s') + + with subprocess.Popen( + sandbox(args) + [ + 'journalctl', + '--output', 'short-monotonic', + '--no-hostname', + '--quiet', + '--priority', 'info', + '--file', journal_file, + ], + stdout=subprocess.PIPE, + text=True, + ) as p: # fmt: skip + assert p.stdout + + is_fatal = False + is_standard = False + comm = None + + while True: + line = p.stdout.readline() + if not line and p.poll() is not None: + break + + if not is_standard and fatal_begin.search(line): + m = find_comm.search(line) + if m: + if exclude_regex and exclude_regex.search(m.group(1)): + continue + comm = m.group(1) + + sys.stderr.write(line) + + is_fatal = True + total += 1 + fatal += 1 + continue + + if is_fatal: + if comm and comm not in line: + continue + + sys.stderr.write(line) + + if fatal_end.search(line): + print(file=sys.stderr) + is_fatal = False + comm = None + continue + + if standard_begin.search(line): + m = find_comm.search(line) + if m: + if exclude_regex and exclude_regex.search(m.group(1)): + continue + comm = m.group(1) + + sys.stderr.write(line) + + is_standard = True + total += 1 + continue + + if is_standard: + if comm and comm not in line: + continue + + sys.stderr.write(line) + + kind = standard_end.search(line) + if kind: + print(file=sys.stderr) + is_standard = False + comm = None + + t = kind.group(1) + if t == 'Address': + asan += 1 + elif t == 'UndefinedBehavior': + ubsan += 1 + elif t == 'Memory': + msan += 1 + + if total > 0: + print( + f'Found {total} sanitizer issues ({fatal} internal, {asan} asan, {ubsan} ubsan, {msan} msan).', + file=sys.stderr, + ) + else: + print('No sanitizer issues found.', file=sys.stderr) + + return total > 0 + + def process_coverage(args: argparse.Namespace, summary: Summary, name: str, journal_file: Path) -> None: coverage = subprocess.run( sandbox(args) + [ @@ -248,6 +369,7 @@ def main() -> None: parser.add_argument('--vm', action=argparse.BooleanOptionalAction) parser.add_argument('--exit-code', required=True, type=int) parser.add_argument('--coredump-exclude-regex', required=True) + parser.add_argument('--sanitizer-exclude-regex', required=True) parser.add_argument('mkosi_args', nargs='*') args = parser.parse_args() @@ -401,19 +523,27 @@ def main() -> None: coredumps = process_coredumps(args, journal_file) + sanitizer = False + if summary.environment.get('SANITIZERS'): + sanitizer = process_sanitizer_report(args, journal_file) + if ( summary.environment.get('COVERAGE', '0') == '1' and result.returncode in (args.exit_code, 77) and not coredumps + and not sanitizer ): process_coverage(args, summary, name, journal_file) if keep_journal == '0' or ( - keep_journal == 'fail' and result.returncode in (args.exit_code, 77) and not coredumps + keep_journal == 'fail' + and result.returncode in (args.exit_code, 77) + and not coredumps + and not sanitizer ): journal_file.unlink(missing_ok=True) - if shell or (result.returncode in (args.exit_code, 77) and not coredumps): + if shell or (result.returncode in (args.exit_code, 77) and not coredumps and not sanitizer): exit(0 if shell or result.returncode == args.exit_code else 77) ops = [] diff --git a/test/meson.build b/test/meson.build index 6d3fdef1fc41c..5545a56c23abf 100644 --- a/test/meson.build +++ b/test/meson.build @@ -298,6 +298,7 @@ integration_test_template = { 'exit-code' : 123, 'vm' : false, 'coredump-exclude-regex' : '', + 'sanitizer-exclude-regex' : '', } testdata_subdirs = [ 'auxv', @@ -393,6 +394,7 @@ foreach integration_test : integration_tests '--firmware', integration_test['firmware'], '--exit-code', integration_test['exit-code'].to_string(), '--coredump-exclude-regex', integration_test['coredump-exclude-regex'], + '--sanitizer-exclude-regex', integration_test['sanitizer-exclude-regex'], ] if 'unit' in integration_test From 91ef65784e98eb83ef46faffbaa3df6512ca4662 Mon Sep 17 00:00:00 2001 From: Yu Watanabe Date: Fri, 6 Dec 2024 03:42:09 +0900 Subject: [PATCH 3/5] mkosi/sanitizers: add more ASAN options This adds the following three options: - detect_invalid_pointer_pairs=2 - handle_ioctl=1 - print_cmdline=1 Note, these options were used in the CentOS CI job. --- mkosi.sanitizers/mkosi.conf | 2 +- .../usr/lib/systemd/system.conf.d/10-sanitizers.conf | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/mkosi.sanitizers/mkosi.conf b/mkosi.sanitizers/mkosi.conf index 0492716ec110c..22ae2028f2a9c 100644 --- a/mkosi.sanitizers/mkosi.conf +++ b/mkosi.sanitizers/mkosi.conf @@ -17,6 +17,6 @@ Environment=ASAN_OPTIONS=verify_asan_link_order=0:intercept_tls_get_addr=0 # systemd.setenv here as there's a size limit on the kernel command line and we don't want to trigger it. We # don't use ManagerEnvironment= either as we want these to be set for pid1 from the earliest possible moment. KernelCommandLine= - ASAN_OPTIONS=strict_string_checks=1:detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1:disable_coredump=0:use_madv_dontdump=1 + ASAN_OPTIONS=strict_string_checks=1:detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1:detect_invalid_pointer_pairs=2:handle_ioctl=1:print_cmdline=1:disable_coredump=0:use_madv_dontdump=1 UBSAN_OPTIONS=print_stacktrace=1:print_summary=1:halt_on_error=1 LSAN_OPTIONS=suppressions=/usr/lib/systemd/leak-sanitizer-suppressions diff --git a/mkosi.sanitizers/mkosi.extra/usr/lib/systemd/system.conf.d/10-sanitizers.conf b/mkosi.sanitizers/mkosi.extra/usr/lib/systemd/system.conf.d/10-sanitizers.conf index a7152a3abe473..1798226b6f8d4 100644 --- a/mkosi.sanitizers/mkosi.extra/usr/lib/systemd/system.conf.d/10-sanitizers.conf +++ b/mkosi.sanitizers/mkosi.extra/usr/lib/systemd/system.conf.d/10-sanitizers.conf @@ -1,6 +1,7 @@ # SPDX-License-Identifier: LGPL-2.1-or-later [Manager] -DefaultEnvironment=ASAN_OPTIONS=strict_string_checks=1:detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1:disable_coredump=0:use_madv_dontdump=1 \ - UBSAN_OPTIONS=print_stacktrace=1:print_summary=1:halt_on_error=1 \ - LSAN_OPTIONS=suppressions=/usr/lib/systemd/leak-sanitizer-suppressions +DefaultEnvironment= \ + ASAN_OPTIONS=strict_string_checks=1:detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1:detect_invalid_pointer_pairs=2:handle_ioctl=1:print_cmdline=1:disable_coredump=0:use_madv_dontdump=1 \ + UBSAN_OPTIONS=print_stacktrace=1:print_summary=1:halt_on_error=1 \ + LSAN_OPTIONS=suppressions=/usr/lib/systemd/leak-sanitizer-suppressions From 456727b5d4aefa5c5da25985e4c718f9d481c9ff Mon Sep 17 00:00:00 2001 From: Yu Watanabe Date: Fri, 6 Dec 2024 08:42:41 +0900 Subject: [PATCH 4/5] test-network: check status of networkd after everything cleared on tear down Otherwise, if networkd is failed, e.g. .network files that triggered the failure will remain, and the next test case will start with previous .network files. So, most subsequent test will fail. --- test/test-network/systemd-networkd-tests.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/test/test-network/systemd-networkd-tests.py b/test/test-network/systemd-networkd-tests.py index 1fd1b2290f668..000645c916e26 100755 --- a/test/test-network/systemd-networkd-tests.py +++ b/test/test-network/systemd-networkd-tests.py @@ -929,17 +929,25 @@ def read_networkd_log(invocation_id=None, since=None): def networkd_is_failed(): return call_quiet('systemctl is-failed -q systemd-networkd.service') != 1 -def stop_networkd(show_logs=True): +def stop_networkd(show_logs=True, check_failed=True): global show_journal show_logs = show_logs and show_journal if show_logs: invocation_id = networkd_invocation_id() - check_output('systemctl stop systemd-networkd.socket') - check_output('systemctl stop systemd-networkd.service') + + if check_failed: + check_output('systemctl stop systemd-networkd.socket') + check_output('systemctl stop systemd-networkd.service') + else: + call('systemctl stop systemd-networkd.socket') + call('systemctl stop systemd-networkd.service') + if show_logs: print(read_networkd_log(invocation_id)) + # Check if networkd exits cleanly. - assert not networkd_is_failed() + if check_failed: + assert not networkd_is_failed() def start_networkd(): check_output('systemctl start systemd-networkd') @@ -1024,7 +1032,7 @@ def tear_down_common(): flush_links() # 5. stop networkd - stop_networkd() + stop_networkd(check_failed=False) # 6. remove configs clear_network_units() @@ -1041,6 +1049,9 @@ def tear_down_common(): sys.stdout.flush() check_output('journalctl --sync') + # 9. check the status of networkd + assert not networkd_is_failed() + def setUpModule(): rm_rf(networkd_ci_temp_dir) cp_r(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'conf'), networkd_ci_temp_dir) From d2d006cc8cbc6fb5c0a30ae5a7b192cf53cc864a Mon Sep 17 00:00:00 2001 From: Yu Watanabe Date: Sun, 8 Dec 2024 04:01:48 +0900 Subject: [PATCH 5/5] test: use systemd-asan-env environment file at more places --- .../systemd-coredump@.service.d/10-asan.conf | 6 ++++ test/units/TEST-07-PID1.exec-context.sh | 35 ++++++++++++------- test/units/TEST-07-PID1.issue-14566.sh | 2 +- 3 files changed, 29 insertions(+), 14 deletions(-) create mode 100644 mkosi.sanitizers/mkosi.extra/usr/lib/systemd/system/systemd-coredump@.service.d/10-asan.conf diff --git a/mkosi.sanitizers/mkosi.extra/usr/lib/systemd/system/systemd-coredump@.service.d/10-asan.conf b/mkosi.sanitizers/mkosi.extra/usr/lib/systemd/system/systemd-coredump@.service.d/10-asan.conf new file mode 100644 index 0000000000000..d627672307c84 --- /dev/null +++ b/mkosi.sanitizers/mkosi.extra/usr/lib/systemd/system/systemd-coredump@.service.d/10-asan.conf @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: LGPL-2.1-or-later + +[Service] +# systemd-coredump may call get_user_creds(), which may pull in instrumented +# systemd NSS modules and may trigger fatal LSAN error. +EnvironmentFile=-/usr/lib/systemd/systemd-asan-env diff --git a/test/units/TEST-07-PID1.exec-context.sh b/test/units/TEST-07-PID1.exec-context.sh index 46fb1f79e85f7..402a3b3498ff2 100755 --- a/test/units/TEST-07-PID1.exec-context.sh +++ b/test/units/TEST-07-PID1.exec-context.sh @@ -349,18 +349,18 @@ if [[ ! -v ASAN_OPTIONS ]] && systemctl --version | grep "+BPF_FRAMEWORK" && ker (! systemd-run --wait --pipe -p RestrictFileSystems="~proc devtmpfs sysfs" ls /sys) fi -if [[ ! -v ASAN_OPTIONS ]]; then - # Ensure DynamicUser=yes does not imply PrivateTmp=yes if TemporaryFileSystem=/tmp /var/tmp is set - systemd-run --unit test-07-dynamic-user-tmp.service \ - --service-type=notify \ - -p DynamicUser=yes \ - -p NotifyAccess=all \ - sh -c 'touch /tmp/a && touch /var/tmp/b && ! test -f /tmp/b && ! test -f /var/tmp/a && systemd-notify --ready && sleep infinity' - (! ls /tmp/systemd-private-"$(tr -d '-' < /proc/sys/kernel/random/boot_id)"-test-07-dynamic-user-tmp.service-* &>/dev/null) - (! ls /var/tmp/systemd-private-"$(tr -d '-' < /proc/sys/kernel/random/boot_id)"-test-07-dynamic-user-tmp.service-* &>/dev/null) - systemctl is-active test-07-dynamic-user-tmp.service - systemctl stop test-07-dynamic-user-tmp.service -fi +# Ensure DynamicUser=yes does not imply PrivateTmp=yes if TemporaryFileSystem=/tmp /var/tmp is set +systemd-run \ + --unit test-07-dynamic-user-tmp.service \ + --service-type=notify \ + -p DynamicUser=yes \ + -p EnvironmentFile=-/usr/lib/systemd/systemd-asan-env \ + -p NotifyAccess=all \ + sh -c 'touch /tmp/a && touch /var/tmp/b && ! test -f /tmp/b && ! test -f /var/tmp/a && systemd-notify --ready && sleep infinity' +(! ls /tmp/systemd-private-"$(tr -d '-' < /proc/sys/kernel/random/boot_id)"-test-07-dynamic-user-tmp.service-* &>/dev/null) +(! ls /var/tmp/systemd-private-"$(tr -d '-' < /proc/sys/kernel/random/boot_id)"-test-07-dynamic-user-tmp.service-* &>/dev/null) +systemctl is-active test-07-dynamic-user-tmp.service +systemctl stop test-07-dynamic-user-tmp.service # Make sure we properly (de)serialize various string arrays, including whitespaces # See: https://github.com/systemd/systemd/issues/31214 @@ -401,7 +401,16 @@ mkdir /tmp/root touch /tmp/root/foo chmod +x /tmp/root/foo (! systemd-run --wait --pipe false) -(! systemd-run --wait --pipe --unit "test-dynamicuser-fail" -p DynamicUser=yes -p WorkingDirectory=/nonexistent true) +if [[ ! -v ASAN_OPTIONS ]]; then + # Here, -p EnvironmentFile=-/usr/lib/systemd/systemd-asan-env does not work, + # as sd-executor loads NSS module and fails before applying the environment: + # (true)[660]: test-dynamicuser-fail.service: Changing to the requested working directory failed: No such file or directory + # (true)[660]: test-dynamicuser-fail.service: Failed at step CHDIR spawning /usr/bin/true: No such file or directory + # TEST-07-PID1.sh[660]: ==660==LeakSanitizer has encountered a fatal error. + # TEST-07-PID1.sh[660]: ==660==HINT: For debugging, try setting environment variable LSAN_OPTIONS=verbosity=1:log_threads=1 + # TEST-07-PID1.sh[660]: ==660==HINT: LeakSanitizer does not work under ptrace (strace, gdb, etc) + (! systemd-run --wait --pipe --unit "test-dynamicuser-fail" -p DynamicUser=yes -p WorkingDirectory=/nonexistent true) +fi (! systemd-run --wait --pipe -p RuntimeDirectory=not-a-directory true) (! systemd-run --wait --pipe -p RootDirectory=/tmp/root this-shouldnt-exist) (! systemd-run --wait --pipe -p RootDirectory=/tmp/root /foo) diff --git a/test/units/TEST-07-PID1.issue-14566.sh b/test/units/TEST-07-PID1.issue-14566.sh index d4be5b53b40ed..ecc18e411bdb8 100755 --- a/test/units/TEST-07-PID1.issue-14566.sh +++ b/test/units/TEST-07-PID1.issue-14566.sh @@ -6,7 +6,7 @@ set -o pipefail # Test that KillMode=mixed does not leave left over processes with ExecStopPost= # Issue: https://github.com/systemd/systemd/issues/14566 -if [[ -n "${ASAN_OPTIONS:-}" ]]; then +if [[ -v ASAN_OPTIONS ]]; then # Temporarily skip this test when running with sanitizers due to a deadlock # See: https://bugzilla.redhat.com/show_bug.cgi?id=2098125 echo "Sanitizers detected, skipping the test..."