core: introduce MemoryTHP= unit file setting

Transparent Hugepages (THP) is a Linux kernel feature that manages
memory using larger pages (2MB on x86, compared to the default 4KB).
The main goal is to improve memory management efficiency and system
performance, especially for memory-intensive applications.
However, it can cause drawbacks in some scenarios, such as memory
regression and latency spikes. THP policy is governed for the entire
system via /sys/kernel/mm/transparent_hugepage/enabled.
However, it can be overridden for individual workloads via prctl(2)
call.
MemoryTHP= is used to disable THPs at exec-invoke to stop
providing THPs for workloads where the drawbacks outweigh the advantages.
When set to "disable", MemoryTHP= disables THPs completely for the
process, irrespecitive of global THP controls.
When set to "madvise", MemoryTHP= disables THPs for the process except
when specifically madvised by the process with MADV_HUGEPAGE or MADV_COLLAPSE.

src/core/exec-invoke.c

@@ -4866,6 +4866,32 @@ static int exec_fd_mark_hot(
         return 1;
 }
 
+static int set_memory_thp(MemoryTHP thp) {
+        switch (thp) {
+
+        case MEMORY_THP_INHERIT:
+                return 0;
+
+        case MEMORY_THP_DISABLE:
+                if (prctl(PR_SET_THP_DISABLE, 1, 0, 0, 0) < 0)
+                        return errno == EINVAL ? -EOPNOTSUPP : -errno;
+                return 0;
+
+        case MEMORY_THP_MADVISE:
+                if (prctl(PR_SET_THP_DISABLE, 1, PR_THP_DISABLE_EXCEPT_ADVISED, 0, 0) < 0)
+                        return errno == EINVAL ? -EOPNOTSUPP : -errno;
+                return 0;
+
+        case MEMORY_THP_SYSTEM:
+                if (prctl(PR_SET_THP_DISABLE, 0, 0, 0, 0) < 0)
+                        return errno == EINVAL ? -EOPNOTSUPP : -errno;
+                return 0;
+
+        default:
+                assert_not_reached();
+        }
+}
+
 static int send_handoff_timestamp(
                 const ExecContext *c,
                 ExecParameters *p,
@@ -5550,6 +5576,16 @@ int exec_invoke(
                         }
                 }
 
+        r = set_memory_thp(context->memory_thp);
+        if (r == -EOPNOTSUPP)
+                log_debug_errno(r, "Setting MemoryTHP=%s is not supported, ignoring: %m",
+                                memory_thp_to_string(context->memory_thp));
+        else if (r < 0) {
+                *exit_status = EXIT_MEMORY_THP;
+                return log_error_errno(r, "Failed to set MemoryTHP=%s: %m",
+                                       memory_thp_to_string(context->memory_thp));
+        }
+
 #if ENABLE_UTMP
         if (context->utmp_id) {
                 _cleanup_free_ char *username_alloc = NULL;