diff --git a/mkosi.sanitizers/mkosi.conf b/mkosi.sanitizers/mkosi.conf index 0492716ec1..22ae2028f2 100644 --- a/mkosi.sanitizers/mkosi.conf +++ b/mkosi.sanitizers/mkosi.conf @@ -17,6 +17,6 @@ Environment=ASAN_OPTIONS=verify_asan_link_order=0:intercept_tls_get_addr=0 # systemd.setenv here as there's a size limit on the kernel command line and we don't want to trigger it. We # don't use ManagerEnvironment= either as we want these to be set for pid1 from the earliest possible moment. KernelCommandLine= - ASAN_OPTIONS=strict_string_checks=1:detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1:disable_coredump=0:use_madv_dontdump=1 + ASAN_OPTIONS=strict_string_checks=1:detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1:detect_invalid_pointer_pairs=2:handle_ioctl=1:print_cmdline=1:disable_coredump=0:use_madv_dontdump=1 UBSAN_OPTIONS=print_stacktrace=1:print_summary=1:halt_on_error=1 LSAN_OPTIONS=suppressions=/usr/lib/systemd/leak-sanitizer-suppressions diff --git a/mkosi.sanitizers/mkosi.extra/usr/lib/systemd/system.conf.d/10-sanitizers.conf b/mkosi.sanitizers/mkosi.extra/usr/lib/systemd/system.conf.d/10-sanitizers.conf index a7152a3abe..1798226b6f 100644 --- a/mkosi.sanitizers/mkosi.extra/usr/lib/systemd/system.conf.d/10-sanitizers.conf +++ b/mkosi.sanitizers/mkosi.extra/usr/lib/systemd/system.conf.d/10-sanitizers.conf @@ -1,6 +1,7 @@ # SPDX-License-Identifier: LGPL-2.1-or-later [Manager] -DefaultEnvironment=ASAN_OPTIONS=strict_string_checks=1:detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1:disable_coredump=0:use_madv_dontdump=1 \ - UBSAN_OPTIONS=print_stacktrace=1:print_summary=1:halt_on_error=1 \ - LSAN_OPTIONS=suppressions=/usr/lib/systemd/leak-sanitizer-suppressions +DefaultEnvironment= \ + ASAN_OPTIONS=strict_string_checks=1:detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1:detect_invalid_pointer_pairs=2:handle_ioctl=1:print_cmdline=1:disable_coredump=0:use_madv_dontdump=1 \ + UBSAN_OPTIONS=print_stacktrace=1:print_summary=1:halt_on_error=1 \ + LSAN_OPTIONS=suppressions=/usr/lib/systemd/leak-sanitizer-suppressions diff --git a/mkosi.extra/usr/lib/systemd/system/iscsi-init.service.d/asan.conf b/mkosi.sanitizers/mkosi.extra/usr/lib/systemd/system/iscsi-init.service.d/10-asan.conf similarity index 100% rename from mkosi.extra/usr/lib/systemd/system/iscsi-init.service.d/asan.conf rename to mkosi.sanitizers/mkosi.extra/usr/lib/systemd/system/iscsi-init.service.d/10-asan.conf diff --git a/mkosi.sanitizers/mkosi.extra/usr/lib/systemd/system/systemd-coredump@.service.d/10-asan.conf b/mkosi.sanitizers/mkosi.extra/usr/lib/systemd/system/systemd-coredump@.service.d/10-asan.conf new file mode 100644 index 0000000000..d627672307 --- /dev/null +++ b/mkosi.sanitizers/mkosi.extra/usr/lib/systemd/system/systemd-coredump@.service.d/10-asan.conf @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: LGPL-2.1-or-later + +[Service] +# systemd-coredump may call get_user_creds(), which may pull in instrumented +# systemd NSS modules and may trigger fatal LSAN error. +EnvironmentFile=-/usr/lib/systemd/systemd-asan-env diff --git a/test/TEST-64-UDEV-STORAGE/meson.build b/test/TEST-64-UDEV-STORAGE/meson.build index 15981ce35f..a91c23ae17 100644 --- a/test/TEST-64-UDEV-STORAGE/meson.build +++ b/test/TEST-64-UDEV-STORAGE/meson.build @@ -38,6 +38,9 @@ foreach testcase : [ ], 'priority' : 10, 'vm' : true, + # Suppress ASan error + # 'multipathd[1820]: ==1820==ERROR: AddressSanitizer: Joining already joined thread, aborting.' + 'sanitizer-exclude-regex' : 'multipathd' }, ] endforeach diff --git a/test/integration-test-wrapper.py b/test/integration-test-wrapper.py index c08f77043c..d44e8f8bd6 100755 --- a/test/integration-test-wrapper.py +++ b/test/integration-test-wrapper.py @@ -122,6 +122,127 @@ def process_coredumps(args: argparse.Namespace, journal_file: Path) -> bool: return True +def process_sanitizer_report(args: argparse.Namespace, journal_file: Path) -> bool: + # Collect sanitizer reports from the journal file. + + if args.sanitizer_exclude_regex: + exclude_regex = re.compile(args.sanitizer_exclude_regex) + else: + exclude_regex = None + + total = 0 + fatal = 0 + asan = 0 + ubsan = 0 + msan = 0 + + # Internal errors: + # ==2554==LeakSanitizer has encountered a fatal error. + # ==2554==HINT: For debugging, try setting environment variable LSAN_OPTIONS=verbosity=1:log_threads=1 + # ==2554==HINT: LeakSanitizer does not work under ptrace (strace, gdb, etc) + fatal_begin = re.compile(r'==[0-9]+==.+?\w+Sanitizer has encountered a fatal error') + fatal_end = re.compile(r'==[0-9]+==HINT:\s+\w+Sanitizer') + + # 'Standard' errors: + standard_begin = re.compile(r'([0-9]+: runtime error|==[0-9]+==.+?\w+Sanitizer)') + standard_end = re.compile(r'SUMMARY:\s+(\w+)Sanitizer') + + # extract COMM + find_comm = re.compile(r'^\[[.0-9 ]+?\]\s(.*?:)\s') + + with subprocess.Popen( + sandbox(args) + [ + 'journalctl', + '--output', 'short-monotonic', + '--no-hostname', + '--quiet', + '--priority', 'info', + '--file', journal_file, + ], + stdout=subprocess.PIPE, + text=True, + ) as p: # fmt: skip + assert p.stdout + + is_fatal = False + is_standard = False + comm = None + + while True: + line = p.stdout.readline() + if not line and p.poll() is not None: + break + + if not is_standard and fatal_begin.search(line): + m = find_comm.search(line) + if m: + if exclude_regex and exclude_regex.search(m.group(1)): + continue + comm = m.group(1) + + sys.stderr.write(line) + + is_fatal = True + total += 1 + fatal += 1 + continue + + if is_fatal: + if comm and comm not in line: + continue + + sys.stderr.write(line) + + if fatal_end.search(line): + print(file=sys.stderr) + is_fatal = False + comm = None + continue + + if standard_begin.search(line): + m = find_comm.search(line) + if m: + if exclude_regex and exclude_regex.search(m.group(1)): + continue + comm = m.group(1) + + sys.stderr.write(line) + + is_standard = True + total += 1 + continue + + if is_standard: + if comm and comm not in line: + continue + + sys.stderr.write(line) + + kind = standard_end.search(line) + if kind: + print(file=sys.stderr) + is_standard = False + comm = None + + t = kind.group(1) + if t == 'Address': + asan += 1 + elif t == 'UndefinedBehavior': + ubsan += 1 + elif t == 'Memory': + msan += 1 + + if total > 0: + print( + f'Found {total} sanitizer issues ({fatal} internal, {asan} asan, {ubsan} ubsan, {msan} msan).', + file=sys.stderr, + ) + else: + print('No sanitizer issues found.', file=sys.stderr) + + return total > 0 + + def process_coverage(args: argparse.Namespace, summary: Summary, name: str, journal_file: Path) -> None: coverage = subprocess.run( sandbox(args) + [ @@ -248,6 +369,7 @@ def main() -> None: parser.add_argument('--vm', action=argparse.BooleanOptionalAction) parser.add_argument('--exit-code', required=True, type=int) parser.add_argument('--coredump-exclude-regex', required=True) + parser.add_argument('--sanitizer-exclude-regex', required=True) parser.add_argument('mkosi_args', nargs='*') args = parser.parse_args() @@ -401,19 +523,27 @@ def main() -> None: coredumps = process_coredumps(args, journal_file) + sanitizer = False + if summary.environment.get('SANITIZERS'): + sanitizer = process_sanitizer_report(args, journal_file) + if ( summary.environment.get('COVERAGE', '0') == '1' and result.returncode in (args.exit_code, 77) and not coredumps + and not sanitizer ): process_coverage(args, summary, name, journal_file) if keep_journal == '0' or ( - keep_journal == 'fail' and result.returncode in (args.exit_code, 77) and not coredumps + keep_journal == 'fail' + and result.returncode in (args.exit_code, 77) + and not coredumps + and not sanitizer ): journal_file.unlink(missing_ok=True) - if shell or (result.returncode in (args.exit_code, 77) and not coredumps): + if shell or (result.returncode in (args.exit_code, 77) and not coredumps and not sanitizer): exit(0 if shell or result.returncode == args.exit_code else 77) ops = [] diff --git a/test/meson.build b/test/meson.build index 6d3fdef1fc..5545a56c23 100644 --- a/test/meson.build +++ b/test/meson.build @@ -298,6 +298,7 @@ integration_test_template = { 'exit-code' : 123, 'vm' : false, 'coredump-exclude-regex' : '', + 'sanitizer-exclude-regex' : '', } testdata_subdirs = [ 'auxv', @@ -393,6 +394,7 @@ foreach integration_test : integration_tests '--firmware', integration_test['firmware'], '--exit-code', integration_test['exit-code'].to_string(), '--coredump-exclude-regex', integration_test['coredump-exclude-regex'], + '--sanitizer-exclude-regex', integration_test['sanitizer-exclude-regex'], ] if 'unit' in integration_test diff --git a/test/test-network/systemd-networkd-tests.py b/test/test-network/systemd-networkd-tests.py index 1fd1b2290f..000645c916 100755 --- a/test/test-network/systemd-networkd-tests.py +++ b/test/test-network/systemd-networkd-tests.py @@ -929,17 +929,25 @@ def read_networkd_log(invocation_id=None, since=None): def networkd_is_failed(): return call_quiet('systemctl is-failed -q systemd-networkd.service') != 1 -def stop_networkd(show_logs=True): +def stop_networkd(show_logs=True, check_failed=True): global show_journal show_logs = show_logs and show_journal if show_logs: invocation_id = networkd_invocation_id() - check_output('systemctl stop systemd-networkd.socket') - check_output('systemctl stop systemd-networkd.service') + + if check_failed: + check_output('systemctl stop systemd-networkd.socket') + check_output('systemctl stop systemd-networkd.service') + else: + call('systemctl stop systemd-networkd.socket') + call('systemctl stop systemd-networkd.service') + if show_logs: print(read_networkd_log(invocation_id)) + # Check if networkd exits cleanly. - assert not networkd_is_failed() + if check_failed: + assert not networkd_is_failed() def start_networkd(): check_output('systemctl start systemd-networkd') @@ -1024,7 +1032,7 @@ def tear_down_common(): flush_links() # 5. stop networkd - stop_networkd() + stop_networkd(check_failed=False) # 6. remove configs clear_network_units() @@ -1041,6 +1049,9 @@ def tear_down_common(): sys.stdout.flush() check_output('journalctl --sync') + # 9. check the status of networkd + assert not networkd_is_failed() + def setUpModule(): rm_rf(networkd_ci_temp_dir) cp_r(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'conf'), networkd_ci_temp_dir) diff --git a/test/units/TEST-07-PID1.exec-context.sh b/test/units/TEST-07-PID1.exec-context.sh index 46fb1f79e8..402a3b3498 100755 --- a/test/units/TEST-07-PID1.exec-context.sh +++ b/test/units/TEST-07-PID1.exec-context.sh @@ -349,18 +349,18 @@ if [[ ! -v ASAN_OPTIONS ]] && systemctl --version | grep "+BPF_FRAMEWORK" && ker (! systemd-run --wait --pipe -p RestrictFileSystems="~proc devtmpfs sysfs" ls /sys) fi -if [[ ! -v ASAN_OPTIONS ]]; then - # Ensure DynamicUser=yes does not imply PrivateTmp=yes if TemporaryFileSystem=/tmp /var/tmp is set - systemd-run --unit test-07-dynamic-user-tmp.service \ - --service-type=notify \ - -p DynamicUser=yes \ - -p NotifyAccess=all \ - sh -c 'touch /tmp/a && touch /var/tmp/b && ! test -f /tmp/b && ! test -f /var/tmp/a && systemd-notify --ready && sleep infinity' - (! ls /tmp/systemd-private-"$(tr -d '-' < /proc/sys/kernel/random/boot_id)"-test-07-dynamic-user-tmp.service-* &>/dev/null) - (! ls /var/tmp/systemd-private-"$(tr -d '-' < /proc/sys/kernel/random/boot_id)"-test-07-dynamic-user-tmp.service-* &>/dev/null) - systemctl is-active test-07-dynamic-user-tmp.service - systemctl stop test-07-dynamic-user-tmp.service -fi +# Ensure DynamicUser=yes does not imply PrivateTmp=yes if TemporaryFileSystem=/tmp /var/tmp is set +systemd-run \ + --unit test-07-dynamic-user-tmp.service \ + --service-type=notify \ + -p DynamicUser=yes \ + -p EnvironmentFile=-/usr/lib/systemd/systemd-asan-env \ + -p NotifyAccess=all \ + sh -c 'touch /tmp/a && touch /var/tmp/b && ! test -f /tmp/b && ! test -f /var/tmp/a && systemd-notify --ready && sleep infinity' +(! ls /tmp/systemd-private-"$(tr -d '-' < /proc/sys/kernel/random/boot_id)"-test-07-dynamic-user-tmp.service-* &>/dev/null) +(! ls /var/tmp/systemd-private-"$(tr -d '-' < /proc/sys/kernel/random/boot_id)"-test-07-dynamic-user-tmp.service-* &>/dev/null) +systemctl is-active test-07-dynamic-user-tmp.service +systemctl stop test-07-dynamic-user-tmp.service # Make sure we properly (de)serialize various string arrays, including whitespaces # See: https://github.com/systemd/systemd/issues/31214 @@ -401,7 +401,16 @@ mkdir /tmp/root touch /tmp/root/foo chmod +x /tmp/root/foo (! systemd-run --wait --pipe false) -(! systemd-run --wait --pipe --unit "test-dynamicuser-fail" -p DynamicUser=yes -p WorkingDirectory=/nonexistent true) +if [[ ! -v ASAN_OPTIONS ]]; then + # Here, -p EnvironmentFile=-/usr/lib/systemd/systemd-asan-env does not work, + # as sd-executor loads NSS module and fails before applying the environment: + # (true)[660]: test-dynamicuser-fail.service: Changing to the requested working directory failed: No such file or directory + # (true)[660]: test-dynamicuser-fail.service: Failed at step CHDIR spawning /usr/bin/true: No such file or directory + # TEST-07-PID1.sh[660]: ==660==LeakSanitizer has encountered a fatal error. + # TEST-07-PID1.sh[660]: ==660==HINT: For debugging, try setting environment variable LSAN_OPTIONS=verbosity=1:log_threads=1 + # TEST-07-PID1.sh[660]: ==660==HINT: LeakSanitizer does not work under ptrace (strace, gdb, etc) + (! systemd-run --wait --pipe --unit "test-dynamicuser-fail" -p DynamicUser=yes -p WorkingDirectory=/nonexistent true) +fi (! systemd-run --wait --pipe -p RuntimeDirectory=not-a-directory true) (! systemd-run --wait --pipe -p RootDirectory=/tmp/root this-shouldnt-exist) (! systemd-run --wait --pipe -p RootDirectory=/tmp/root /foo) diff --git a/test/units/TEST-07-PID1.issue-14566.sh b/test/units/TEST-07-PID1.issue-14566.sh index d4be5b53b4..ecc18e411b 100755 --- a/test/units/TEST-07-PID1.issue-14566.sh +++ b/test/units/TEST-07-PID1.issue-14566.sh @@ -6,7 +6,7 @@ set -o pipefail # Test that KillMode=mixed does not leave left over processes with ExecStopPost= # Issue: https://github.com/systemd/systemd/issues/14566 -if [[ -n "${ASAN_OPTIONS:-}" ]]; then +if [[ -v ASAN_OPTIONS ]]; then # Temporarily skip this test when running with sanitizers due to a deadlock # See: https://bugzilla.redhat.com/show_bug.cgi?id=2098125 echo "Sanitizers detected, skipping the test..."