From b5a9b601c3301b208dba23e339dd5d2f0a32c9e4 Mon Sep 17 00:00:00 2001 From: Frantisek Sumsal Date: Tue, 12 Jan 2021 22:13:33 +0100 Subject: [PATCH 1/4] ci: drop the `unbuffer` workaround It actually didn't do anything, the issue was somewhere else. --- .github/workflows/ubuntu-unit-tests.sh | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.github/workflows/ubuntu-unit-tests.sh b/.github/workflows/ubuntu-unit-tests.sh index 705b147b96..eccd048e28 100755 --- a/.github/workflows/ubuntu-unit-tests.sh +++ b/.github/workflows/ubuntu-unit-tests.sh @@ -61,12 +61,7 @@ for phase in "${PHASES[@]}"; do export ASAN_OPTIONS=strict_string_checks=1:detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1 # Never remove halt_on_error from UBSAN_OPTIONS. See https://github.com/systemd/systemd/commit/2614d83aa06592aedb. export UBSAN_OPTIONS=print_stacktrace=1:print_summary=1:halt_on_error=1 - # There's some weird stuff going on in GH Actions, where the following - # `meson test` command hangs after test #252 unless it's executed under - # unbuffer or there's something else producing output. So far it happens - # _only_ with ASAn (not with UBSan), both with gcc and clang. I'll - # need to take a closer look later... - unbuffer meson test --timeout-multiplier=3 -C build --print-errorlogs + meson test --timeout-multiplier=3 -C build --print-errorlogs ;; CLEANUP) info "Cleanup phase" From 56a017cb0dda3e5134f5c7d42ef9516be81082a9 Mon Sep 17 00:00:00 2001 From: Frantisek Sumsal Date: Wed, 13 Jan 2021 21:48:36 +0100 Subject: [PATCH 2/4] ci: optimize the sanitized build under gcc as well --- .github/workflows/ubuntu-unit-tests.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ubuntu-unit-tests.sh b/.github/workflows/ubuntu-unit-tests.sh index eccd048e28..532ee7bf78 100755 --- a/.github/workflows/ubuntu-unit-tests.sh +++ b/.github/workflows/ubuntu-unit-tests.sh @@ -47,13 +47,15 @@ for phase in "${PHASES[@]}"; do meson test -C build --print-errorlogs ;; RUN_ASAN_UBSAN|RUN_GCC_ASAN_UBSAN|RUN_CLANG_ASAN_UBSAN) + MESON_ARGS=(--optimization=1) + if [[ "$phase" = "RUN_CLANG_ASAN_UBSAN" ]]; then export CC=clang export CXX=clang++ # Build fuzzer regression tests only with clang (for now), # see: https://github.com/systemd/systemd/pull/15886#issuecomment-632689604 # -Db_lundef=false: See https://github.com/mesonbuild/meson/issues/764 - MESON_ARGS=(-Db_lundef=false -Dfuzz-tests=true --optimization=1) + MESON_ARGS+=(-Db_lundef=false -Dfuzz-tests=true) fi meson --werror -Dtests=unsafe -Db_sanitize=address,undefined "${MESON_ARGS[@]}" build ninja -C build -v From f1a8fed286e3b9527b1837e9d5c6cb8d88bd2041 Mon Sep 17 00:00:00 2001 From: Frantisek Sumsal Date: Tue, 12 Jan 2021 22:14:59 +0100 Subject: [PATCH 3/4] ci: skip test-execute on GH Actions under ASan It seems to suffer from the same issue as on Travis CI, where the test randomly fails due to timeouts in its subtests. See: https://github.com/systemd/systemd/issues/10696#issuecomment-758501797 --- src/test/test-execute.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/test-execute.c b/src/test/test-execute.c index 1890cc4f75..83816f474c 100644 --- a/src/test/test-execute.c +++ b/src/test/test-execute.c @@ -897,8 +897,8 @@ int main(int argc, char *argv[]) { test_setup_logging(LOG_DEBUG); #if HAS_FEATURE_ADDRESS_SANITIZER - if (strstr_ptr(ci_environment(), "travis")) { - log_notice("Running on TravisCI under ASan, skipping, see https://github.com/systemd/systemd/issues/10696"); + if (strstr_ptr(ci_environment(), "travis") || strstr_ptr(ci_environment(), "github-actions")) { + log_notice("Running on Travis CI/GH Actions under ASan, skipping, see https://github.com/systemd/systemd/issues/10696"); return EXIT_TEST_SKIP; } #endif From e8fd7c9ed405dd17e252705324a31cc68ac10a9c Mon Sep 17 00:00:00 2001 From: Frantisek Sumsal Date: Wed, 13 Jan 2021 21:42:13 +0100 Subject: [PATCH 4/4] ci: introduce a watchdog for the sanitized unit test run --- .github/workflows/ubuntu-unit-tests.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/ubuntu-unit-tests.sh b/.github/workflows/ubuntu-unit-tests.sh index 532ee7bf78..d48c4b4bfe 100755 --- a/.github/workflows/ubuntu-unit-tests.sh +++ b/.github/workflows/ubuntu-unit-tests.sh @@ -63,6 +63,16 @@ for phase in "${PHASES[@]}"; do export ASAN_OPTIONS=strict_string_checks=1:detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1 # Never remove halt_on_error from UBSAN_OPTIONS. See https://github.com/systemd/systemd/commit/2614d83aa06592aedb. export UBSAN_OPTIONS=print_stacktrace=1:print_summary=1:halt_on_error=1 + + # FIXME + # For some strange reason the GH Actions VM stops responding after + # executing first ~150 tests, _unless_ there's something producing + # output (either running `meson test` in verbose mode, or something + # else in background). Despite my efforts so far I haven't been able + # to identify the culprit (since the issue is not reproducible + # during debugging, wonderful), so let's at least keep a workaround + # here to make the builds stable for the time being. + (set +x; while :; do echo -ne "\n[WATCHDOG] $(date)\n"; sleep 30; done) & meson test --timeout-multiplier=3 -C build --print-errorlogs ;; CLEANUP)