cryptenroll,cryptsetup,shutdown: only call mlockall if we have CAP_IPC_LOCK

Calling mlockall in an unprivileged process most notably had the effect
of making systemd-cryptenroll OOM while trying to open a normal-sized
argon2 keyslot due to it hitting RLIMIT_MEMLOCK.

src/basic/process-util.c

@@ -4,6 +4,7 @@
 #include <pthread.h>
 #include <spawn.h>
 #include <stdio.h>
+#include <sys/mman.h>
 #include <sys/mount.h>
 #include <sys/personality.h>
 #include <sys/prctl.h>
@@ -20,6 +21,7 @@
 #include "alloc-util.h"
 #include "architecture.h"
 #include "argv-util.h"
+#include "capability-util.h"
 #include "cgroup-util.h"
 #include "dirent-util.h"
 #include "dlfcn-util.h"
@@ -2227,6 +2229,26 @@ int proc_dir_read_pidref(DIR *d, PidRef *ret) {
         return 0;
 }
 
+int safe_mlockall(int flags) {
+        int r;
+
+        /* When dealing with sensitive data, let's lock ourselves into memory. We do this only when
+         * privileged however, as otherwise the amount of lockable memory that RLIMIT_MEMLOCK grants us is
+         * frequently too low to make this work. The resource limit has no effect on CAP_IPC_LOCK processes,
+         * hence that's the capability we check for. */
+        r = have_effective_cap(CAP_IPC_LOCK);
+        if (r < 0)
+                return log_debug_errno(r, "Failed to determine if we have CAP_IPC_LOCK: %m");
+        if (r == 0)
+                return log_debug_errno(SYNTHETIC_ERRNO(EPERM), "Lacking CAP_IPC_LOCK, skipping mlockall().");
+
+        if (mlockall(flags) < 0)
+                return log_debug_errno(errno, "Failed to call mlockall(): %m");
+
+        log_debug("Successfully called mlockall().");
+        return 0;
+}
+
 static const char *const sigchld_code_table[] = {
         [CLD_EXITED] = "exited",
         [CLD_KILLED] = "killed",

src/basic/process-util.h

@@ -257,5 +257,7 @@ int proc_dir_open(DIR **ret);
 int proc_dir_read(DIR *d, pid_t *ret);
 int proc_dir_read_pidref(DIR *d, PidRef *ret);
 
+int safe_mlockall(int flags);
+
 _noreturn_ void report_errno_and_exit(int errno_fd, int error);
 int read_errno(int errno_fd);

src/cryptenroll/cryptenroll.c

@@ -27,6 +27,7 @@
 #include "parse-util.h"
 #include "pkcs11-util.h"
 #include "pretty-print.h"
+#include "process-util.h"
 #include "string-table.h"
 #include "string-util.h"
 #include "tpm2-pcr.h"
@@ -850,7 +851,7 @@ static int run(int argc, char *argv[]) {
                 return r;
 
         /* A delicious drop of snake oil */
-        (void) mlockall(MCL_CURRENT|MCL_FUTURE|MCL_ONFAULT);
+        (void) safe_mlockall(MCL_CURRENT|MCL_FUTURE|MCL_ONFAULT);
 
         cryptsetup_enable_logging(NULL);
 

src/cryptsetup/cryptsetup.c

@@ -40,6 +40,7 @@
 #include "path-util.h"
 #include "pkcs11-util.h"
 #include "pretty-print.h"
+#include "process-util.h"
 #include "random-util.h"
 #include "string-table.h"
 #include "string-util.h"
@@ -2582,7 +2583,7 @@ static int verb_attach(int argc, char *argv[], void *userdata) {
                   volume, source, strempty(arg_type), strempty(arg_cipher));
 
         /* A delicious drop of snake oil */
-        (void) mlockall(MCL_CURRENT|MCL_FUTURE|MCL_ONFAULT);
+        (void) safe_mlockall(MCL_CURRENT|MCL_FUTURE|MCL_ONFAULT);
 
         if (key_file && arg_keyfile_erase)
                 destroy_key_file = key_file; /* let's get this baby erased when we leave */

src/shutdown/shutdown.c

@@ -401,9 +401,9 @@ int main(int argc, char *argv[]) {
 
         init_watchdog();
 
-        /* Lock us into memory */
-        (void) mlockall(MCL_FUTURE|MCL_ONFAULT);
-        (void) mlockall(MCL_CURRENT);
+        /* Lock us into memory. If the first mlockall call fails, don't attempt it again. */
+        if (safe_mlockall(MCL_FUTURE|MCL_ONFAULT) >= 0)
+                (void) mlockall(MCL_CURRENT);
 
         /* We need to make mounts private so that we can MS_MOVE in unmount_all(). Kernel does not allow
          * MS_MOVE when parent mountpoints have shared propagation. */