diff --git a/src/core/bpf-lsm.c b/src/core/bpf-lsm.c
index 79d17b0751..e0333963c5 100644
--- a/src/core/bpf-lsm.c
+++ b/src/core/bpf-lsm.c
@@ -176,7 +176,7 @@ int lsm_bpf_supported(void) {
 }
 
 int lsm_bpf_setup(Manager *m) {
-        struct restrict_fs_bpf *obj;
+        _cleanup_(restrict_fs_bpf_freep) struct restrict_fs_bpf *obj = NULL;
         _cleanup_(bpf_link_freep) struct bpf_link *link = NULL;
         int r;
 
@@ -186,17 +186,16 @@ int lsm_bpf_setup(Manager *m) {
         if (r < 0)
                 return r;
 
-        m->restrict_fs = obj;
-
-        link = sym_bpf_program__attach_lsm(m->restrict_fs->progs.restrict_filesystems);
+        link = sym_bpf_program__attach_lsm(obj->progs.restrict_filesystems);
         r = sym_libbpf_get_error(link);
         if (r != 0)
                 return log_error_errno(r, "Failed to link '%s' LSM BPF program: %m",
-                                       sym_bpf_program__name(m->restrict_fs->progs.restrict_filesystems));
+                                       sym_bpf_program__name(obj->progs.restrict_filesystems));
 
         log_info("LSM BPF program attached");
 
-        m->restrict_fs->links.restrict_filesystems = TAKE_PTR(link);
+        obj->links.restrict_filesystems = TAKE_PTR(link);
+        m->restrict_fs = TAKE_PTR(obj);
 
         return 0;
 }
@@ -210,6 +209,10 @@ int lsm_bpf_unit_restrict_filesystems(Unit *u, const Set *filesystems, bool allo
         assert(filesystems);
         assert(u);
 
+        if (!u->manager->restrict_fs)
+                return log_unit_error_errno(u, SYNTHETIC_ERRNO(EINVAL),
+                                            "Restrict filesystems BPF object is not set, BPF LSM setup has failed?");
+
         int inner_map_fd = sym_bpf_create_map(
                         BPF_MAP_TYPE_HASH,
                         sizeof(uint32_t),
diff --git a/src/core/execute.c b/src/core/execute.c
index e3b9134c3d..4c96c30cf4 100644
--- a/src/core/execute.c
+++ b/src/core/execute.c
@@ -1732,9 +1732,16 @@ static int apply_lock_personality(const Unit* u, const ExecContext *c) {
 
 #if HAVE_LIBBPF
 static bool skip_lsm_bpf_unsupported(const Unit* u, const char* msg) {
+        assert(u);
+        assert(u->manager);
+
         if (lsm_bpf_supported())
                 return false;
 
+        /* lsm_bpf_setup succeeded */
+        if (u->manager->restrict_fs)
+                return false;
+
         log_unit_debug(u, "LSM BPF not supported, skipping %s", msg);
         return true;
 }
diff --git a/src/core/manager.c b/src/core/manager.c
index 601e15f689..12c49e7fca 100644
--- a/src/core/manager.c
+++ b/src/core/manager.c
@@ -933,7 +933,7 @@ int manager_new(UnitFileScope scope, ManagerTestRunFlags test_run_flags, Manager
                 if (MANAGER_IS_SYSTEM(m) && lsm_bpf_supported()) {
                         r = lsm_bpf_setup(m);
                         if (r < 0)
-                                return r;
+                                log_warning_errno(r, "Failed to setup LSM BPF, ignoring: %m");
                 }
 #endif
         }