From ca2e19f2b73d8a489b31106c5d211bdfebf68672 Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Tue, 7 May 2024 11:50:11 +0200 Subject: [PATCH 1/4] test: Don't keep journals for skipped tests Let's make sure we don't save journals for tests that were skipped. --- test/integration-test-wrapper.py | 37 ++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/test/integration-test-wrapper.py b/test/integration-test-wrapper.py index 575403344e..25b903c4e6 100755 --- a/test/integration-test-wrapper.py +++ b/test/integration-test-wrapper.py @@ -144,24 +144,29 @@ def main(): ] result = subprocess.run(cmd) - # Return code 123 is the expected success code - if result.returncode != 123: - if result.returncode != 77 and journal_file: - cmd = [ - 'journalctl', - '--no-hostname', - '-o', 'short-monotonic', - '--file', journal_file, - '-u', test_unit, - '-p', 'info', - ] - print("Test failed, relevant logs can be viewed with: \n\n" - f"{shlex.join(str(a) for a in cmd)}\n", file=sys.stderr) - exit(result.returncode or 1) - # Do not keep journal files for tests that don't fail. + # Return code 123 is the expected success code + if result.returncode in (123, 77): + # Do not keep journal files for tests that don't fail. + if journal_file: + journal_file.unlink(missing_ok=True) + + exit(0 if result.returncode == 123 else 77) + if journal_file: - journal_file.unlink(missing_ok=True) + cmd = [ + 'journalctl', + '--no-hostname', + '-o', 'short-monotonic', + '--file', journal_file, + '-u', test_unit, + '-p', 'info', + ] + print("Test failed, relevant logs can be viewed with: \n\n" + f"{shlex.join(str(a) for a in cmd)}\n", file=sys.stderr) + + # 0 also means we failed so translate that to a non-zero exit code to mark the test as failed. + exit(result.returncode or 1) if __name__ == '__main__': From a77f65d0c9590761dd08ed78ea0ebfdd845ba84f Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Tue, 7 May 2024 12:24:51 +0200 Subject: [PATCH 2/4] ci: Print a helpful link to download and view a failed test's journal --- test/integration-test-wrapper.py | 37 ++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/test/integration-test-wrapper.py b/test/integration-test-wrapper.py index 25b903c4e6..fcdc9b6352 100755 --- a/test/integration-test-wrapper.py +++ b/test/integration-test-wrapper.py @@ -9,6 +9,7 @@ with the expectation that as part of formally defining the API it will be tidy. ''' import argparse +import json import os import shlex import subprocess @@ -154,16 +155,34 @@ def main(): exit(0 if result.returncode == 123 else 77) if journal_file: - cmd = [ - 'journalctl', - '--no-hostname', - '-o', 'short-monotonic', - '--file', journal_file, - '-u', test_unit, - '-p', 'info', - ] + ops = [] + + if os.getenv("GITHUB_ACTIONS"): + id = os.environ["GITHUB_RUN_ID"] + iteration = os.environ["GITHUB_RUN_ATTEMPT"] + j = json.loads( + subprocess.run( + [ + "mkosi", + "--directory", os.fspath(args.meson_source_dir), + "--json", + "summary", + ], + stdout=subprocess.PIPE, + text=True, + ).stdout + ) + images = {image["Image"]: image for image in j["Images"]} + distribution = images["system"]["Distribution"] + release = images["system"]["Release"] + artifact = f"ci-mkosi-{id}-{iteration}-{distribution}-{release}-failed-test-journals" + ops += [f"gh run download {id} --name {artifact} -D ci/{artifact}"] + journal_file = Path(f"ci/{artifact}/test/journal/{name}.journal") + + ops += [f"journalctl --file {journal_file} --no-hostname -o short-monotonic -u {test_unit} -p info"] + print("Test failed, relevant logs can be viewed with: \n\n" - f"{shlex.join(str(a) for a in cmd)}\n", file=sys.stderr) + f"{(' && '.join(ops))}\n", file=sys.stderr) # 0 also means we failed so translate that to a non-zero exit code to mark the test as failed. exit(result.returncode or 1) From f1f87f3b2e5a516314c6feb9e6c26a21f2e52bde Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Tue, 7 May 2024 15:20:44 +0200 Subject: [PATCH 3/4] test: Add infra to mark integration tests as slow Some integration tests take much more time than others, let's add a test param that can be used to configure this and integrate it with the slow-tests meson option. --- test/integration-test-wrapper.py | 13 +++++++++---- test/meson.build | 16 +++++++++++++--- 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/test/integration-test-wrapper.py b/test/integration-test-wrapper.py index fcdc9b6352..c41ca8e55a 100755 --- a/test/integration-test-wrapper.py +++ b/test/integration-test-wrapper.py @@ -39,10 +39,6 @@ ExecStart=false def main(): - if not bool(int(os.getenv("SYSTEMD_INTEGRATION_TESTS", "0"))): - print("SYSTEMD_INTEGRATION_TESTS=1 not found in environment, skipping", file=sys.stderr) - exit(77) - parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--meson-source-dir', required=True, type=Path) parser.add_argument('--meson-build-dir', required=True, type=Path) @@ -50,9 +46,18 @@ def main(): parser.add_argument('--test-number', required=True) parser.add_argument('--storage', required=True) parser.add_argument('--firmware', required=True) + parser.add_argument('--slow', action=argparse.BooleanOptionalAction) parser.add_argument('mkosi_args', nargs="*") args = parser.parse_args() + if not bool(int(os.getenv("SYSTEMD_INTEGRATION_TESTS", "0"))): + print(f"SYSTEMD_INTEGRATION_TESTS=1 not found in environment, skipping {args.test_name}", file=sys.stderr) + exit(77) + + if args.slow and not bool(int(os.getenv("SYSTEMD_SLOW_TESTS", "0"))): + print(f"SYSTEMD_SLOW_TESTS=1 not found in environment, skipping {args.test_name}", file=sys.stderr) + exit(77) + name = args.test_name + (f"-{i}" if (i := os.getenv("MESON_TEST_ITERATION")) else "") test_unit = f"testsuite-{args.test_number}.service" diff --git a/test/meson.build b/test/meson.build index 63b94b1c7a..30a0461f0d 100644 --- a/test/meson.build +++ b/test/meson.build @@ -416,6 +416,7 @@ foreach test_number, dirname : integration_tests 'storage' : 'volatile', 'priority' : 0, 'firmware' : 'linux', + 'slow' : false, } # TODO: This fs.exists call isn't included in rebuild logic @@ -432,13 +433,22 @@ foreach test_number, dirname : integration_tests '--test-number', test_number, '--storage', test_params['storage'], '--firmware', test_params['firmware'], - '--', - ] + test_params['mkosi_args'] + ] + + if test_params['slow'] + args += ['--slow'] + endif + + args += ['--'] + test_params['mkosi_args'] integration_test_env = {} if want_integration_tests - integration_test_env = {'SYSTEMD_INTEGRATION_TESTS': '1'} + integration_test_env += {'SYSTEMD_INTEGRATION_TESTS': '1'} + endif + + if want_slow_tests + integration_test_env += {'SYSTEMD_SLOW_TESTS': '1'} endif # We don't explicitly depend on the "mkosi" target because that means the image is rebuilt From 0d13ca5544c97b181d9ed2a062e3a279f6938b71 Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Tue, 7 May 2024 15:21:27 +0200 Subject: [PATCH 4/4] test: Mark TEST-21-DFUZZER as slow and skip it by default The test is not really useful without sanitizers, so let's mark it as slow and not run it by default in CI. --- test/TEST-21-DFUZZER/meson.build | 1 + 1 file changed, 1 insertion(+) diff --git a/test/TEST-21-DFUZZER/meson.build b/test/TEST-21-DFUZZER/meson.build index e107b85561..5a27fb2f3f 100644 --- a/test/TEST-21-DFUZZER/meson.build +++ b/test/TEST-21-DFUZZER/meson.build @@ -3,4 +3,5 @@ test_params += { 'timeout' : 3600, 'priority' : -50, + 'slow' : true, }