mkosi: several improvements for running with sanitizers (systemd#35480)

mkosi.sanitizers/mkosi.conf

@@ -17,6 +17,6 @@ Environment=ASAN_OPTIONS=verify_asan_link_order=0:intercept_tls_get_addr=0
 # systemd.setenv here as there's a size limit on the kernel command line and we don't want to trigger it. We
 # don't use ManagerEnvironment= either as we want these to be set for pid1 from the earliest possible moment.
 KernelCommandLine=
-        ASAN_OPTIONS=strict_string_checks=1:detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1:disable_coredump=0:use_madv_dontdump=1
+        ASAN_OPTIONS=strict_string_checks=1:detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1:detect_invalid_pointer_pairs=2:handle_ioctl=1:print_cmdline=1:disable_coredump=0:use_madv_dontdump=1
         UBSAN_OPTIONS=print_stacktrace=1:print_summary=1:halt_on_error=1
         LSAN_OPTIONS=suppressions=/usr/lib/systemd/leak-sanitizer-suppressions

mkosi.sanitizers/mkosi.extra/usr/lib/systemd/system.conf.d/10-sanitizers.conf

@@ -1,6 +1,7 @@
 # SPDX-License-Identifier: LGPL-2.1-or-later
 
 [Manager]
-DefaultEnvironment=ASAN_OPTIONS=strict_string_checks=1:detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1:disable_coredump=0:use_madv_dontdump=1 \
-                   UBSAN_OPTIONS=print_stacktrace=1:print_summary=1:halt_on_error=1 \
-                   LSAN_OPTIONS=suppressions=/usr/lib/systemd/leak-sanitizer-suppressions
+DefaultEnvironment= \
+        ASAN_OPTIONS=strict_string_checks=1:detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1:detect_invalid_pointer_pairs=2:handle_ioctl=1:print_cmdline=1:disable_coredump=0:use_madv_dontdump=1 \
+        UBSAN_OPTIONS=print_stacktrace=1:print_summary=1:halt_on_error=1 \
+        LSAN_OPTIONS=suppressions=/usr/lib/systemd/leak-sanitizer-suppressions

mkosi.sanitizers/mkosi.extra/usr/lib/systemd/system/systemd-coredump@.service.d/10-asan.conf

@@ -0,0 +1,6 @@
+# SPDX-License-Identifier: LGPL-2.1-or-later
+
+[Service]
+# systemd-coredump may call get_user_creds(), which may pull in instrumented
+# systemd NSS modules and may trigger fatal LSAN error.
+EnvironmentFile=-/usr/lib/systemd/systemd-asan-env

test/TEST-64-UDEV-STORAGE/meson.build

@@ -38,6 +38,9 @@ foreach testcase : [
                         ],
                         'priority' : 10,
                         'vm' : true,
+                        # Suppress ASan error
+                        # 'multipathd[1820]: ==1820==ERROR: AddressSanitizer: Joining already joined thread, aborting.'
+                        'sanitizer-exclude-regex' : 'multipathd'
                 },
         ]
 endforeach

test/integration-test-wrapper.py

@@ -122,6 +122,127 @@ def process_coredumps(args: argparse.Namespace, journal_file: Path) -> bool:
     return True
 
 
+def process_sanitizer_report(args: argparse.Namespace, journal_file: Path) -> bool:
+    # Collect sanitizer reports from the journal file.
+
+    if args.sanitizer_exclude_regex:
+        exclude_regex = re.compile(args.sanitizer_exclude_regex)
+    else:
+        exclude_regex = None
+
+    total = 0
+    fatal = 0
+    asan = 0
+    ubsan = 0
+    msan = 0
+
+    # Internal errors:
+    # ==2554==LeakSanitizer has encountered a fatal error.
+    # ==2554==HINT: For debugging, try setting environment variable LSAN_OPTIONS=verbosity=1:log_threads=1
+    # ==2554==HINT: LeakSanitizer does not work under ptrace (strace, gdb, etc)
+    fatal_begin = re.compile(r'==[0-9]+==.+?\w+Sanitizer has encountered a fatal error')
+    fatal_end = re.compile(r'==[0-9]+==HINT:\s+\w+Sanitizer')
+
+    # 'Standard' errors:
+    standard_begin = re.compile(r'([0-9]+: runtime error|==[0-9]+==.+?\w+Sanitizer)')
+    standard_end = re.compile(r'SUMMARY:\s+(\w+)Sanitizer')
+
+    # extract COMM
+    find_comm = re.compile(r'^\[[.0-9 ]+?\]\s(.*?:)\s')
+
+    with subprocess.Popen(
+        sandbox(args) + [
+            'journalctl',
+            '--output', 'short-monotonic',
+            '--no-hostname',
+            '--quiet',
+            '--priority', 'info',
+            '--file', journal_file,
+        ],
+        stdout=subprocess.PIPE,
+        text=True,
+    ) as p:  # fmt: skip
+        assert p.stdout
+
+        is_fatal = False
+        is_standard = False
+        comm = None
+
+        while True:
+            line = p.stdout.readline()
+            if not line and p.poll() is not None:
+                break
+
+            if not is_standard and fatal_begin.search(line):
+                m = find_comm.search(line)
+                if m:
+                    if exclude_regex and exclude_regex.search(m.group(1)):
+                        continue
+                    comm = m.group(1)
+
+                sys.stderr.write(line)
+
+                is_fatal = True
+                total += 1
+                fatal += 1
+                continue
+
+            if is_fatal:
+                if comm and comm not in line:
+                    continue
+
+                sys.stderr.write(line)
+
+                if fatal_end.search(line):
+                    print(file=sys.stderr)
+                    is_fatal = False
+                    comm = None
+                continue
+
+            if standard_begin.search(line):
+                m = find_comm.search(line)
+                if m:
+                    if exclude_regex and exclude_regex.search(m.group(1)):
+                        continue
+                    comm = m.group(1)
+
+                sys.stderr.write(line)
+
+                is_standard = True
+                total += 1
+                continue
+
+            if is_standard:
+                if comm and comm not in line:
+                    continue
+
+                sys.stderr.write(line)
+
+                kind = standard_end.search(line)
+                if kind:
+                    print(file=sys.stderr)
+                    is_standard = False
+                    comm = None
+
+                    t = kind.group(1)
+                    if t == 'Address':
+                        asan += 1
+                    elif t == 'UndefinedBehavior':
+                        ubsan += 1
+                    elif t == 'Memory':
+                        msan += 1
+
+    if total > 0:
+        print(
+            f'Found {total} sanitizer issues ({fatal} internal, {asan} asan, {ubsan} ubsan, {msan} msan).',
+            file=sys.stderr,
+        )
+    else:
+        print('No sanitizer issues found.', file=sys.stderr)
+
+    return total > 0
+
+
 def process_coverage(args: argparse.Namespace, summary: Summary, name: str, journal_file: Path) -> None:
     coverage = subprocess.run(
         sandbox(args) + [
@@ -248,6 +369,7 @@ def main() -> None:
     parser.add_argument('--vm', action=argparse.BooleanOptionalAction)
     parser.add_argument('--exit-code', required=True, type=int)
     parser.add_argument('--coredump-exclude-regex', required=True)
+    parser.add_argument('--sanitizer-exclude-regex', required=True)
     parser.add_argument('mkosi_args', nargs='*')
     args = parser.parse_args()
 
@@ -401,19 +523,27 @@ def main() -> None:
 
     coredumps = process_coredumps(args, journal_file)
 
+    sanitizer = False
+    if summary.environment.get('SANITIZERS'):
+        sanitizer = process_sanitizer_report(args, journal_file)
+
     if (
         summary.environment.get('COVERAGE', '0') == '1'
         and result.returncode in (args.exit_code, 77)
         and not coredumps
+        and not sanitizer
     ):
         process_coverage(args, summary, name, journal_file)
 
     if keep_journal == '0' or (
-        keep_journal == 'fail' and result.returncode in (args.exit_code, 77) and not coredumps
+        keep_journal == 'fail'
+        and result.returncode in (args.exit_code, 77)
+        and not coredumps
+        and not sanitizer
     ):
         journal_file.unlink(missing_ok=True)
 
-    if shell or (result.returncode in (args.exit_code, 77) and not coredumps):
+    if shell or (result.returncode in (args.exit_code, 77) and not coredumps and not sanitizer):
         exit(0 if shell or result.returncode == args.exit_code else 77)
 
     ops = []

test/meson.build

@@ -298,6 +298,7 @@ integration_test_template = {
         'exit-code' : 123,
         'vm' : false,
         'coredump-exclude-regex' : '',
+        'sanitizer-exclude-regex' : '',
 }
 testdata_subdirs = [
         'auxv',
@@ -393,6 +394,7 @@ foreach integration_test : integration_tests
                 '--firmware', integration_test['firmware'],
                 '--exit-code', integration_test['exit-code'].to_string(),
                 '--coredump-exclude-regex', integration_test['coredump-exclude-regex'],
+                '--sanitizer-exclude-regex', integration_test['sanitizer-exclude-regex'],
         ]
 
         if 'unit' in integration_test

test/test-network/systemd-networkd-tests.py

@@ -929,17 +929,25 @@ def read_networkd_log(invocation_id=None, since=None):
 def networkd_is_failed():
     return call_quiet('systemctl is-failed -q systemd-networkd.service') != 1
 
-def stop_networkd(show_logs=True):
+def stop_networkd(show_logs=True, check_failed=True):
     global show_journal
     show_logs = show_logs and show_journal
     if show_logs:
         invocation_id = networkd_invocation_id()
-    check_output('systemctl stop systemd-networkd.socket')
-    check_output('systemctl stop systemd-networkd.service')
+
+    if check_failed:
+        check_output('systemctl stop systemd-networkd.socket')
+        check_output('systemctl stop systemd-networkd.service')
+    else:
+        call('systemctl stop systemd-networkd.socket')
+        call('systemctl stop systemd-networkd.service')
+
     if show_logs:
         print(read_networkd_log(invocation_id))
+
     # Check if networkd exits cleanly.
-    assert not networkd_is_failed()
+    if check_failed:
+        assert not networkd_is_failed()
 
 def start_networkd():
     check_output('systemctl start systemd-networkd')
@@ -1024,7 +1032,7 @@ def tear_down_common():
     flush_links()
 
     # 5. stop networkd
-    stop_networkd()
+    stop_networkd(check_failed=False)
 
     # 6. remove configs
     clear_network_units()
@@ -1041,6 +1049,9 @@ def tear_down_common():
     sys.stdout.flush()
     check_output('journalctl --sync')
 
+    # 9. check the status of networkd
+    assert not networkd_is_failed()
+
 def setUpModule():
     rm_rf(networkd_ci_temp_dir)
     cp_r(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'conf'), networkd_ci_temp_dir)

test/units/TEST-07-PID1.exec-context.sh

@@ -349,18 +349,18 @@ if [[ ! -v ASAN_OPTIONS ]] && systemctl --version | grep "+BPF_FRAMEWORK" && ker
     (! systemd-run --wait --pipe -p RestrictFileSystems="~proc devtmpfs sysfs" ls /sys)
 fi
 
-if [[ ! -v ASAN_OPTIONS ]]; then
-    # Ensure DynamicUser=yes does not imply PrivateTmp=yes if TemporaryFileSystem=/tmp /var/tmp is set
-    systemd-run --unit test-07-dynamic-user-tmp.service \
-                --service-type=notify \
-                -p DynamicUser=yes \
-                -p NotifyAccess=all \
-                sh -c 'touch /tmp/a && touch /var/tmp/b && ! test -f /tmp/b && ! test -f /var/tmp/a && systemd-notify --ready && sleep infinity'
-    (! ls /tmp/systemd-private-"$(tr -d '-' < /proc/sys/kernel/random/boot_id)"-test-07-dynamic-user-tmp.service-* &>/dev/null)
-    (! ls /var/tmp/systemd-private-"$(tr -d '-' < /proc/sys/kernel/random/boot_id)"-test-07-dynamic-user-tmp.service-* &>/dev/null)
-    systemctl is-active test-07-dynamic-user-tmp.service
-    systemctl stop test-07-dynamic-user-tmp.service
-fi
+# Ensure DynamicUser=yes does not imply PrivateTmp=yes if TemporaryFileSystem=/tmp /var/tmp is set
+systemd-run \
+    --unit test-07-dynamic-user-tmp.service \
+    --service-type=notify \
+    -p DynamicUser=yes \
+    -p EnvironmentFile=-/usr/lib/systemd/systemd-asan-env \
+    -p NotifyAccess=all \
+    sh -c 'touch /tmp/a && touch /var/tmp/b && ! test -f /tmp/b && ! test -f /var/tmp/a && systemd-notify --ready && sleep infinity'
+(! ls /tmp/systemd-private-"$(tr -d '-' < /proc/sys/kernel/random/boot_id)"-test-07-dynamic-user-tmp.service-* &>/dev/null)
+(! ls /var/tmp/systemd-private-"$(tr -d '-' < /proc/sys/kernel/random/boot_id)"-test-07-dynamic-user-tmp.service-* &>/dev/null)
+systemctl is-active test-07-dynamic-user-tmp.service
+systemctl stop test-07-dynamic-user-tmp.service
 
 # Make sure we properly (de)serialize various string arrays, including whitespaces
 # See: https://github.com/systemd/systemd/issues/31214
@@ -401,7 +401,16 @@ mkdir /tmp/root
 touch /tmp/root/foo
 chmod +x /tmp/root/foo
 (! systemd-run --wait --pipe false)
-(! systemd-run --wait --pipe --unit "test-dynamicuser-fail" -p DynamicUser=yes -p WorkingDirectory=/nonexistent true)
+if [[ ! -v ASAN_OPTIONS ]]; then
+    # Here, -p EnvironmentFile=-/usr/lib/systemd/systemd-asan-env does not work,
+    # as sd-executor loads NSS module and fails before applying the environment:
+    # (true)[660]: test-dynamicuser-fail.service: Changing to the requested working directory failed: No such file or directory
+    # (true)[660]: test-dynamicuser-fail.service: Failed at step CHDIR spawning /usr/bin/true: No such file or directory
+    # TEST-07-PID1.sh[660]: ==660==LeakSanitizer has encountered a fatal error.
+    # TEST-07-PID1.sh[660]: ==660==HINT: For debugging, try setting environment variable LSAN_OPTIONS=verbosity=1:log_threads=1
+    # TEST-07-PID1.sh[660]: ==660==HINT: LeakSanitizer does not work under ptrace (strace, gdb, etc)
+    (! systemd-run --wait --pipe --unit "test-dynamicuser-fail" -p DynamicUser=yes -p WorkingDirectory=/nonexistent true)
+fi
 (! systemd-run --wait --pipe -p RuntimeDirectory=not-a-directory true)
 (! systemd-run --wait --pipe -p RootDirectory=/tmp/root this-shouldnt-exist)
 (! systemd-run --wait --pipe -p RootDirectory=/tmp/root /foo)

test/units/TEST-07-PID1.issue-14566.sh

@@ -6,7 +6,7 @@ set -o pipefail
 # Test that KillMode=mixed does not leave left over processes with ExecStopPost=
 # Issue: https://github.com/systemd/systemd/issues/14566
 
-if [[ -n "${ASAN_OPTIONS:-}" ]]; then
+if [[ -v ASAN_OPTIONS ]]; then
     # Temporarily skip this test when running with sanitizers due to a deadlock
     # See: https://bugzilla.redhat.com/show_bug.cgi?id=2098125
     echo "Sanitizers detected, skipping the test..."