core: introduce MemoryTHP= unit file setting

Transparent Hugepages (THP) is a Linux kernel feature that manages
memory using larger pages (2MB on x86, compared to the default 4KB).
The main goal is to improve memory management efficiency and system
performance, especially for memory-intensive applications.
However, it can cause drawbacks in some scenarios, such as memory
regression and latency spikes. THP policy is governed for the entire
system via /sys/kernel/mm/transparent_hugepage/enabled.
However, it can be overridden for individual workloads via prctl(2)
call.
MemoryTHP= is used to disable THPs at exec-invoke to stop
providing THPs for workloads where the drawbacks outweigh the advantages.
When set to "disable", MemoryTHP= disables THPs completely for the
process, irrespecitive of global THP controls.
When set to "madvise", MemoryTHP= disables THPs for the process except
when specifically madvised by the process with MADV_HUGEPAGE or MADV_COLLAPSE.

src/core/dbus-execute.c

@@ -58,6 +58,7 @@ static BUS_DEFINE_PROPERTY_GET_ENUM(property_get_private_bpf, private_bpf, Priva
 static BUS_DEFINE_PROPERTY_GET_ENUM(property_get_protect_home, protect_home, ProtectHome);
 static BUS_DEFINE_PROPERTY_GET_ENUM(property_get_protect_system, protect_system, ProtectSystem);
 static BUS_DEFINE_PROPERTY_GET_ENUM(property_get_personality, personality, unsigned long);
+static BUS_DEFINE_PROPERTY_GET_ENUM(property_get_memory_thp, memory_thp, MemoryTHP);
 static BUS_DEFINE_PROPERTY_GET(property_get_ioprio, "i", ExecContext, exec_context_get_effective_ioprio);
 static BUS_DEFINE_PROPERTY_GET(property_get_mount_apivfs, "b", ExecContext, exec_context_get_effective_mount_apivfs);
 static BUS_DEFINE_PROPERTY_GET(property_get_bind_log_sockets, "b", ExecContext, exec_context_get_effective_bind_log_sockets);
@@ -1407,6 +1408,7 @@ const sd_bus_vtable bus_exec_vtable[] = {
         SD_BUS_PROPERTY("BPFDelegatePrograms", "s", property_get_bpf_delegate_programs, offsetof(ExecContext, bpf_delegate_programs), SD_BUS_VTABLE_PROPERTY_CONST),
         SD_BUS_PROPERTY("BPFDelegateAttachments", "s", property_get_bpf_delegate_attachments, offsetof(ExecContext, bpf_delegate_attachments), SD_BUS_VTABLE_PROPERTY_CONST),
         SD_BUS_PROPERTY("MemoryKSM", "b", bus_property_get_tristate, offsetof(ExecContext, memory_ksm), SD_BUS_VTABLE_PROPERTY_CONST),
+        SD_BUS_PROPERTY("MemoryTHP", "s", property_get_memory_thp, offsetof(ExecContext, memory_thp), SD_BUS_VTABLE_PROPERTY_CONST),
         SD_BUS_PROPERTY("UserNamespacePath", "s", NULL, offsetof(ExecContext, user_namespace_path), SD_BUS_VTABLE_PROPERTY_CONST),
         SD_BUS_PROPERTY("NetworkNamespacePath", "s", NULL, offsetof(ExecContext, network_namespace_path), SD_BUS_VTABLE_PROPERTY_CONST),
         SD_BUS_PROPERTY("IPCNamespacePath", "s", NULL, offsetof(ExecContext, ipc_namespace_path), SD_BUS_VTABLE_PROPERTY_CONST),
@@ -1845,6 +1847,7 @@ static BUS_DEFINE_SET_TRANSIENT_PARSE(keyring_mode, ExecKeyringMode, exec_keyrin
 static BUS_DEFINE_SET_TRANSIENT_PARSE(protect_proc, ProtectProc, protect_proc_from_string);
 static BUS_DEFINE_SET_TRANSIENT_PARSE(proc_subset, ProcSubset, proc_subset_from_string);
 static BUS_DEFINE_SET_TRANSIENT_PARSE(private_bpf, PrivateBPF, private_bpf_from_string);
+static BUS_DEFINE_SET_TRANSIENT_PARSE(memory_thp, MemoryTHP, memory_thp_from_string);
 static BUS_DEFINE_SET_TRANSIENT_PARSE_PTR(bpf_delegate_commands, uint64_t, bpf_delegate_commands_from_string);
 static BUS_DEFINE_SET_TRANSIENT_PARSE_PTR(bpf_delegate_maps, uint64_t, bpf_delegate_maps_from_string);
 static BUS_DEFINE_SET_TRANSIENT_PARSE_PTR(bpf_delegate_programs, uint64_t, bpf_delegate_programs_from_string);
@@ -2333,6 +2336,9 @@ int bus_exec_context_set_transient_property(
         if (streq(name, "MemoryKSM"))
                 return bus_set_transient_tristate(u, name, &c->memory_ksm, message, flags, reterr_error);
 
+        if (streq(name, "MemoryTHP"))
+                return bus_set_transient_memory_thp(u, name, &c->memory_thp, message, flags, reterr_error);
+
         if (streq(name, "UtmpIdentifier"))
                 return bus_set_transient_string(u, name, &c->utmp_id, message, flags, reterr_error);
 

src/core/exec-invoke.c

@@ -4866,6 +4866,32 @@ static int exec_fd_mark_hot(
         return 1;
 }
 
+static int set_memory_thp(MemoryTHP thp) {
+        switch (thp) {
+
+        case MEMORY_THP_INHERIT:
+                return 0;
+
+        case MEMORY_THP_DISABLE:
+                if (prctl(PR_SET_THP_DISABLE, 1, 0, 0, 0) < 0)
+                        return errno == EINVAL ? -EOPNOTSUPP : -errno;
+                return 0;
+
+        case MEMORY_THP_MADVISE:
+                if (prctl(PR_SET_THP_DISABLE, 1, PR_THP_DISABLE_EXCEPT_ADVISED, 0, 0) < 0)
+                        return errno == EINVAL ? -EOPNOTSUPP : -errno;
+                return 0;
+
+        case MEMORY_THP_SYSTEM:
+                if (prctl(PR_SET_THP_DISABLE, 0, 0, 0, 0) < 0)
+                        return errno == EINVAL ? -EOPNOTSUPP : -errno;
+                return 0;
+
+        default:
+                assert_not_reached();
+        }
+}
+
 static int send_handoff_timestamp(
                 const ExecContext *c,
                 ExecParameters *p,
@@ -5550,6 +5576,16 @@ int exec_invoke(
                         }
                 }
 
+        r = set_memory_thp(context->memory_thp);
+        if (r == -EOPNOTSUPP)
+                log_debug_errno(r, "Setting MemoryTHP=%s is not supported, ignoring: %m",
+                                memory_thp_to_string(context->memory_thp));
+        else if (r < 0) {
+                *exit_status = EXIT_MEMORY_THP;
+                return log_error_errno(r, "Failed to set MemoryTHP=%s: %m",
+                                       memory_thp_to_string(context->memory_thp));
+        }
+
 #if ENABLE_UTMP
         if (context->utmp_id) {
                 _cleanup_free_ char *username_alloc = NULL;

src/core/execute-serialize.c

@@ -1664,6 +1664,10 @@ static int exec_context_serialize(const ExecContext *c, FILE *f) {
         if (r < 0)
                 return r;
 
+        r = serialize_item(f, "exec-context-memory-thp", memory_thp_to_string(c->memory_thp));
+        if (r < 0)
+                return r;
+
         r = serialize_item(f, "exec-context-private-tmp", private_tmp_to_string(c->private_tmp));
         if (r < 0)
                 return r;
@@ -2649,6 +2653,10 @@ static int exec_context_deserialize(ExecContext *c, FILE *f) {
                         r = safe_atoi(val, &c->memory_ksm);
                         if (r < 0)
                                 return r;
+                } else if ((val = startswith(l, "exec-context-memory-thp="))) {
+                        c->memory_thp = memory_thp_from_string(val);
+                        if (c->memory_thp < 0)
+                                return c->memory_thp;
                 } else if ((val = startswith(l, "exec-context-private-tmp="))) {
                         c->private_tmp = private_tmp_from_string(val);
                         if (c->private_tmp < 0)

src/core/execute.c

@@ -1117,7 +1117,8 @@ void exec_context_dump(const ExecContext *c, FILE* f, const char *prefix) {
                 "%sProtectHostname: %s%s%s\n"
                 "%sProtectProc: %s\n"
                 "%sProcSubset: %s\n"
-                "%sPrivateBPF: %s\n",
+                "%sPrivateBPF: %s\n"
+                "%sMemoryTHP: %s\n",
                 prefix, c->umask,
                 prefix, empty_to_root(c->working_directory),
                 prefix, empty_to_root(c->root_directory),
@@ -1145,7 +1146,8 @@ void exec_context_dump(const ExecContext *c, FILE* f, const char *prefix) {
                 prefix, protect_hostname_to_string(c->protect_hostname), c->private_hostname ? ":" : "", strempty(c->private_hostname),
                 prefix, protect_proc_to_string(c->protect_proc),
                 prefix, proc_subset_to_string(c->proc_subset),
-                prefix, private_bpf_to_string(c->private_bpf));
+                prefix, private_bpf_to_string(c->private_bpf),
+                prefix, memory_thp_to_string(c->memory_thp));
 
         if (c->private_bpf == PRIVATE_BPF_YES) {
                 _cleanup_free_ char

src/core/execute.h

@@ -311,6 +311,7 @@ typedef struct ExecContext {
         int mount_apivfs;
         int bind_log_sockets;
         int memory_ksm;
+        MemoryTHP memory_thp;
         PrivateTmp private_tmp;
         PrivateTmp private_var_tmp; /* This is not an independent parameter, but calculated from other
                                      * parameters in unit_patch_contexts(). */

src/core/load-fragment-gperf.gperf.in

@@ -195,6 +195,7 @@
 {% endif %}
 {{type}}.ProtectHostname,                     config_parse_protect_hostname,                      0,                                  offsetof({{type}}, exec_context)
 {{type}}.MemoryKSM,                           config_parse_tristate,                              0,                                  offsetof({{type}}, exec_context.memory_ksm)
+{{type}}.MemoryTHP,                           config_parse_memory_thp,                            0,                                  offsetof({{type}}, exec_context.memory_thp)
 {%- endmacro -%}
 
 {%- macro KILL_CONTEXT_CONFIG_ITEMS(type) -%}

src/core/load-fragment.c

@@ -163,6 +163,7 @@ DEFINE_CONFIG_PARSE_PTR(config_parse_bpf_delegate_commands, bpf_delegate_command
 DEFINE_CONFIG_PARSE_PTR(config_parse_bpf_delegate_maps, bpf_delegate_maps_from_string, uint64_t);
 DEFINE_CONFIG_PARSE_PTR(config_parse_bpf_delegate_programs, bpf_delegate_programs_from_string, uint64_t);
 DEFINE_CONFIG_PARSE_PTR(config_parse_bpf_delegate_attachments, bpf_delegate_attachments_from_string, uint64_t);
+DEFINE_CONFIG_PARSE_ENUM(config_parse_memory_thp, memory_thp, MemoryTHP);
 
 bool contains_instance_specifier_superset(const char *s) {
         const char *p, *q;

src/core/load-fragment.h

@@ -168,6 +168,7 @@ CONFIG_PARSER_PROTOTYPE(config_parse_cgroup_nft_set);
 CONFIG_PARSER_PROTOTYPE(config_parse_mount_node);
 CONFIG_PARSER_PROTOTYPE(config_parse_concurrency_max);
 CONFIG_PARSER_PROTOTYPE(config_parse_bind_network_interface);
+CONFIG_PARSER_PROTOTYPE(config_parse_memory_thp);
 
 /* gperf prototypes */
 const struct ConfigPerfItem* load_fragment_gperf_lookup(const char *key, GPERF_LEN_TYPE length);

src/core/namespace.c

@@ -4047,6 +4047,15 @@ DEFINE_STRING_TABLE_LOOKUP(bpf_delegate_map_type, uint64_t);
 DEFINE_STRING_TABLE_LOOKUP(bpf_delegate_prog_type, uint64_t);
 DEFINE_STRING_TABLE_LOOKUP(bpf_delegate_attach_type, uint64_t);
 
+static const char* const memory_thp_table[_MEMORY_THP_MAX] = {
+        [MEMORY_THP_INHERIT] = "inherit",
+        [MEMORY_THP_DISABLE] = "disable",
+        [MEMORY_THP_MADVISE] = "madvise",
+        [MEMORY_THP_SYSTEM]  = "system",
+};
+
+DEFINE_STRING_TABLE_LOOKUP(memory_thp, MemoryTHP);
+
 char* bpf_delegate_to_string(uint64_t u, const char * (*parser)(uint64_t) _const_ ) {
         assert(parser);
 

src/core/namespace.h

@@ -91,6 +91,24 @@ typedef enum PrivatePIDs {
         _PRIVATE_PIDS_INVALID = -EINVAL,
 } PrivatePIDs;
 
+typedef enum MemoryTHP {
+        /*
+         * Inherit default from process that starts systemd, i.e. do not make
+         * any PR_SET_THP_DISABLE call.
+         */
+        MEMORY_THP_INHERIT,
+        MEMORY_THP_DISABLE, /* Disable THPs completely for the prcess */
+        MEMORY_THP_MADVISE, /* Disable THPs for the process except when madvised */
+        /*
+         * Use system default THP setting. this can be used when the process that
+         * starts systemd has already disabled THPs via PR_SET_THP_DISABLE, and we
+         * want to restore the system default THP setting at process invokation time.
+         */
+        MEMORY_THP_SYSTEM,
+        _MEMORY_THP_MAX,
+        _MEMORY_THP_INVALID = -EINVAL,
+} MemoryTHP;
+
 typedef struct BindMount {
         char *source;
         char *destination;
@@ -233,6 +251,8 @@ DECLARE_STRING_TABLE_LOOKUP(proc_subset, ProcSubset);
 
 DECLARE_STRING_TABLE_LOOKUP(private_bpf, PrivateBPF);
 
+DECLARE_STRING_TABLE_LOOKUP(memory_thp, MemoryTHP);
+
 DECLARE_STRING_TABLE_LOOKUP(bpf_delegate_cmd, uint64_t);
 
 DECLARE_STRING_TABLE_LOOKUP(bpf_delegate_map_type, uint64_t);

src/core/varlink-execute.c

@@ -908,6 +908,7 @@ int unit_exec_context_build_json(sd_json_variant **ret, const char *name, void *
                         SD_JSON_BUILD_PAIR_BOOLEAN("RemoveIPC", c->remove_ipc),
                         JSON_BUILD_PAIR_TRISTATE_NON_NULL("PrivateMounts", c->private_mounts),
                         JSON_BUILD_PAIR_STRING_NON_EMPTY("MountFlags", mount_propagation_flag_to_string(c->mount_propagation_flag)),
+                        SD_JSON_BUILD_PAIR_STRING("MemoryTHP", memory_thp_to_string(c->memory_thp)),
 
                         /* System Call Filtering */
                         JSON_BUILD_PAIR_CALLBACK_NON_NULL("SystemCallFilter", syscall_filter_build_json, c),