From 37ee8dc80f070311896f4a92e56a2333b8da735d Mon Sep 17 00:00:00 2001 From: Frantisek Sumsal Date: Fri, 24 May 2019 22:35:52 +0200 Subject: [PATCH 1/2] test: improve handling of ASan under clang Running integration tests with ASan is somewhat tricky to begin with, as we need to pre-load the ASan runtime DSO for certain services (like dbus), otherwise they won't start or behave as expected. In case of gcc this is pretty easy, as we need the runtime DSO during compilation, so it's already present on the host system. For clang things get more complicated, as ASan is compiled in statically by default, thus to enable the necessary dynamic-ish behavior one needs to compile with -shared-libasan and then correctly set LD_PRELOAD_PATH, as the runtime libraries are not in a standard library path. --- test/test-functions | 59 ++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 55 insertions(+), 4 deletions(-) diff --git a/test/test-functions b/test/test-functions index 47c5b2cb17..4a76dd70ea 100644 --- a/test/test-functions +++ b/test/test-functions @@ -54,6 +54,30 @@ if [[ "$IS_BUILT_WITH_ASAN" = "yes" ]]; then PATH_TO_INIT=$ROOTLIBDIR/systemd-under-asan QEMU_MEM="1536M" QEMU_SMP=4 + + # We need to correctly distinguish between gcc's and clang's ASan DSOs. + if ldd $BUILD_DIR/systemd | grep -q libasan.so; then + ASAN_COMPILER=gcc + elif ldd $BUILD_DIR/systemd | grep -q libclang_rt.asan; then + ASAN_COMPILER=clang + + # As clang's ASan DSO is usually in a non-standard path, let's check if + # the environment is set accordingly. If not, warn the user and exit. + # We're not setting the LD_LIBRARY_PATH automagically here, because + # user should encounter (and fix) the same issue when running the unit + # tests (meson test) + if ldd "$BUILD_DIR/systemd" | grep -q "libclang_rt.asan.*not found"; then + _asan_rt_name="$(ldd $BUILD_DIR/systemd | awk '/libclang_rt.asan/ {print $1; exit}')" + _asan_rt_path="$(find /usr/lib* /usr/local/lib* -type f -name "$_asan_rt_name" 2>/dev/null | sed 1q)" + echo >&2 "clang's ASan DSO ($_asan_rt_name) is not present in the runtime library path" + echo >&2 "Consider setting LD_LIBRARY_PATH=${_asan_rt_path%/*}" + exit 1 + fi + else + echo >&2 "systemd is not linked against the ASan DSO" + echo >&2 "gcc does this by default, for clang compile with -shared-libasan" + exit 1 + fi fi function find_qemu_bin() { @@ -268,7 +292,7 @@ setup_basic_environment() { install_depmod_files generate_module_dependencies if [[ "$IS_BUILT_WITH_ASAN" = "yes" ]]; then - create_asan_wrapper + create_asan_wrapper fi } @@ -348,7 +372,24 @@ EOF create_asan_wrapper() { local _asan_wrapper=$initdir/$ROOTLIBDIR/systemd-under-asan + local _asan_rt_pattern ddebug "Create $_asan_wrapper" + + case "$ASAN_COMPILER" in + gcc) + _asan_rt_pattern="*libasan*" + ;; + clang) + _asan_rt_pattern="libclang_rt.asan-*" + # Install llvm-symbolizer to generate useful reports + # See: https://clang.llvm.org/docs/AddressSanitizer.html#symbolizing-the-reports + dracut_install "llvm-symbolizer" + ;; + *) + dfail "Unsupported compiler: $ASAN_COMPILER" + exit 1 + esac + cat >$_asan_wrapper < not found" issues in the future +export PATH="/sbin:/bin:/usr/sbin:/usr/bin" + mount -t proc proc /proc mount -t sysfs sysfs /sys mount -o remount,rw / -PATH_TO_ASAN=\$(find / -name '*libasan*' | sed 1q) +PATH_TO_ASAN=\$(find / -name '$_asan_rt_pattern' | sed 1q) if [[ "\$PATH_TO_ASAN" ]]; then # A lot of services (most notably dbus) won't start without preloading libasan # See https://github.com/systemd/systemd/issues/5004 DEFAULT_ENVIRONMENT="\$DEFAULT_ENVIRONMENT LD_PRELOAD=\$PATH_TO_ASAN" + # Let's add the ASan DSO's path to the dynamic linker's cache. This is pretty + # unnecessary for gcc & libasan, however, for clang this is crucial, as its + # runtime ASan DSO is in a non-standard (library) path. + echo \${PATH_TO_ASAN%/*} > /etc/ld.so.conf.d/asan-path-override.conf + ldconfig fi echo DefaultEnvironment=\$DEFAULT_ENVIRONMENT >>/etc/systemd/system.conf echo DefaultTimeoutStartSec=180s >>/etc/systemd/system.conf +echo DefaultStandardOutput=journal+console >>/etc/systemd/system.conf # ASAN and syscall filters aren't compatible with each other. find / -name '*.service' -type f | xargs sed -i 's/^\\(MemoryDeny\\|SystemCall\\)/#\\1/' @@ -475,14 +526,14 @@ get_ldpath() { install_missing_libraries() { # install possible missing libraries for i in $initdir{,/usr}/{sbin,bin}/* $initdir{,/usr}/lib/systemd/{,tests/{,manual/,unsafe/}}*; do - LD_LIBRARY_PATH=$(get_ldpath $i) inst_libs $i + LD_LIBRARY_PATH="${LD_LIBRARY_PATH:+$LD_LIBRARY_PATH:}$(get_ldpath $i)" inst_libs $i done } create_empty_image() { local _size=500 if [[ "$STRIP_BINARIES" = "no" ]]; then - _size=$((2*_size)) + _size=$((4*_size)) fi rm -f "$TESTDIR/rootdisk.img" # Create the blank file to use as a root filesystem From c9a95378ec16b71f637628877c675e7a88be865f Mon Sep 17 00:00:00 2001 From: Frantisek Sumsal Date: Mon, 27 May 2019 14:14:26 +0200 Subject: [PATCH 2/2] docs: add documentation for sanitizers --- docs/CODE_QUALITY.md | 5 ++ docs/TESTING_WITH_SANITIZERS.md | 88 +++++++++++++++++++++++++++++++++ 2 files changed, 93 insertions(+) create mode 100644 docs/TESTING_WITH_SANITIZERS.md diff --git a/docs/CODE_QUALITY.md b/docs/CODE_QUALITY.md index 18363f0f1b..a3bdfaad8e 100644 --- a/docs/CODE_QUALITY.md +++ b/docs/CODE_QUALITY.md @@ -64,5 +64,10 @@ available functionality: of active alerts can be found [here](https://lgtm.com/projects/g/systemd/systemd/alerts/?mode=list). +15. Each PR is automatically tested with [Address Sanitizer](https://clang.llvm.org/docs/AddressSanitizer.html) + and [Undefined Behavior Sanitizer](https://clang.llvm.org/docs/UndefinedBehaviorSanitizer.html). + See [Testing systemd using sanitizers](https://systemd.io/TESTING_WITH_SANITIZERS) + for more information. + Access to Coverity and oss-fuzz reports is limited. Please reach out to the maintainers if you need access. diff --git a/docs/TESTING_WITH_SANITIZERS.md b/docs/TESTING_WITH_SANITIZERS.md new file mode 100644 index 0000000000..7cb879aaf3 --- /dev/null +++ b/docs/TESTING_WITH_SANITIZERS.md @@ -0,0 +1,88 @@ +--- +title: Testing systemd using sanitizers +--- + +# Testing systemd using sanitizers + +To catch the *nastier* kind of bugs, you can run your code with [Address Sanitizer](https://clang.llvm.org/docs/AddressSanitizer.html) +and [Undefined Behavior Sanitizer](https://clang.llvm.org/docs/UndefinedBehaviorSanitizer.html). +This is mostly done automagically by various CI systems for each PR, but you may +want to do it locally as well. The process slightly varies depending on the +compiler you want to use and which part of the test suite you want to run. + +## gcc +gcc compiles in sanitizer libraries dynamically by default, so you need to get +the shared libraries first - on Fedora these are shipped as a separate packages +(`libasan` for Address Sanitizer and `libubsan` for Undefined Behavior Sanitizer). + +The compilation itself is then a matter of simply adding `-Db_sanitize=address,undefined` +to `meson`. That's it - following executions of `meson test` and integrations tests +under `test/` subdirectory will run with sanitizers enabled. However, to get +truly useful results, you should tweak the runtime configuration of respective +sanitizers; e.g. in systemd we set the following environment variables: + +```bash +ASAN_OPTIONS=strict_string_checks=1:detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1 +UBSAN_OPTIONS=print_stacktrace=1:print_summary=1:halt_on_error=1 +``` +## clang +In case of clang things are somewhat different - the sanitizer libraries are +compiled in statically by default. This is not an issue if you plan to run +only the unit tests, but for integration tests you'll need to convince clang +to use the dynamic versions of sanitizer libraries. + +First of all, pass `-shared-libsan` to both `clang` and `clang++`: + +```bash +CFLAGS=-shared-libasan +CXXFLAGS=-shared-libasan +``` + +The `CXXFLAGS` are necessary for `src/libsystemd/sd-bus/test-bus-vtable-cc.c`. Compilation +is then the same as in case of gcc, simply add `-Db_sanitize=address,undefined` +to the `meson` call and use the same environment variables for runtime configuration. + +```bash +ASAN_OPTIONS=strict_string_checks=1:detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1 +UBSAN_OPTIONS=print_stacktrace=1:print_summary=1:halt_on_error=1 +``` + +After this, you'll probably notice that all compiled binaries complain about +missing `libclang_rt.asan*` library. To fix this, you have to install clang's +runtime libraries, usually shipped in the `compiler-rt` package. As these libraries +are installed in a non-standard location (non-standard for `ldconfig`), you'll +need to manually direct binaries to the respective runtime libraries. + +``` +# Optionally locate the respective runtime DSO +$ ldd build/systemd | grep libclang_rt.asan + libclang_rt.asan-x86_64.so => not found + libclang_rt.asan-x86_64.so => not found +$ find /usr/lib* /usr/local/lib* -type f -name libclang_rt.asan-x86_64.so 2>/dev/null +/usr/lib64/clang/7.0.1/lib/libclang_rt.asan-x86_64.so + +# Set the LD_LIBRARY_PATH accordingly +export LD_LIBRARY_PATH=/usr/lib64/clang/7.0.1/lib/ + +# If the path is correct, the "not found" message should change to an actual path +$ ldd build/systemd | grep libclang_rt.asan + libclang_rt.asan-x86_64.so => /usr/lib64/clang/7.0.1/lib/libclang_rt.asan-x86_64.so (0x00007fa9752fc000) +``` + +This should help binaries to correctly find necessary sanitizer DSOs. + +Also, to make the reports useful, `llvm-symbolizer` tool is required (usually +part of the `llvm` package). + +## Background notes +The reason why you need to force dynamic linking in case of `clang` is that some +applications make use of `libsystemd`, which is compiled with sanitizers as well. +However, if a *standard* (uninstrumented) application loads an instrumented library, +it will immediately fail due to unresolved symbols. To fix/workaround this, you +need to pre-load the ASan DSO using `LD_PRELOAD=/path/to/asan/dso`, which will +make things work as expected in most cases. This will, obviously, not work with +statically linked sanitizer libraries. + +These shenanigans are performed automatically when running the integration test +suite (i.e. `test/TEST-??-*`) and are located in `test/test-functions` (mainly, +but not only, in the `create_asan_wrapper` function).