diff --git a/src/basic/sort-util.c b/src/basic/sort-util.c index a76a6c85a8..8aca5cac76 100644 --- a/src/basic/sort-util.c +++ b/src/basic/sort-util.c @@ -81,3 +81,12 @@ int cmp_uint16(const uint16_t *a, const uint16_t *b) { return CMP(*a, *b); } + +int cmp_unsigned(const unsigned *a, const unsigned *b) { + /* This is called from qsort()s inner loops. Correctly implemented qsort will never pass NULL so we + just suppress the check via POINTER_MAY_BE_NULL instead of assert() to avoid the runtime cost. */ + POINTER_MAY_BE_NULL(a); + POINTER_MAY_BE_NULL(b); + + return CMP(*a, *b); +} diff --git a/src/basic/sort-util.h b/src/basic/sort-util.h index 8e7a1991bb..37c196158e 100644 --- a/src/basic/sort-util.h +++ b/src/basic/sort-util.h @@ -44,3 +44,4 @@ void qsort_r_safe(void *base, size_t nmemb, size_t size, comparison_userdata_fn_ int cmp_int(const int *a, const int *b); int cmp_uint16(const uint16_t *a, const uint16_t *b); +int cmp_unsigned(const unsigned *a, const unsigned *b); diff --git a/src/shared/numa-util.c b/src/shared/numa-util.c index 228ea7ad2d..9097ccbc31 100644 --- a/src/shared/numa-util.c +++ b/src/shared/numa-util.c @@ -154,6 +154,63 @@ static int numa_max_node(void) { return max_node; } +int numa_get_node_from_cpu(unsigned cpu, unsigned *ret) { + _cleanup_closedir_ DIR *d = NULL; + int r; + + assert(ret); + + d = opendir("/sys/devices/system/node"); + if (!d) + return -errno; + + FOREACH_DIRENT(de, d, break) { + char p[STRLEN("/sys/devices/system/node/node/cpulist") + DECIMAL_STR_MAX(unsigned) + 1]; + _cleanup_(cpu_set_done) CPUSet cpus = {}; + _cleanup_free_ char *cpulist = NULL; + const char *n; + unsigned node; + + if (de->d_type != DT_DIR) + continue; + + n = startswith(de->d_name, "node"); + if (!n) + continue; + + r = safe_atou(n, &node); + if (r < 0) { + log_debug_errno(r, "Failed to parse node number %s to unsigned, ignoring: %m", n); + continue; + } + + xsprintf(p, "/sys/devices/system/node/node%u/cpulist", node); + + r = read_virtual_file(p, SIZE_MAX, &cpulist, /* ret_size= */ NULL); + if (r < 0) { + log_debug_errno(r, "Failed to read %s, ignoring: %m", p); + continue; + } + + r = parse_cpu_set(cpulist, &cpus); + if (r < 0) { + log_debug_errno(r, "Failed to parse cpu set %s, ignoring: %m", cpulist); + continue; + } + + if (CPU_ISSET_S(cpu, cpus.allocated, cpus.set)) { + *ret = node; + return 0; + } + } + + /* CPU not found in any NUMA node, assume node 0 */ + log_debug("CPU %u not found in any NUMA node, assuming node 0.", cpu); + *ret = 0; + + return 0; +} + int numa_mask_add_all(CPUSet *mask) { int m; diff --git a/src/shared/numa-util.h b/src/shared/numa-util.h index c684dea803..6fec7c587b 100644 --- a/src/shared/numa-util.h +++ b/src/shared/numa-util.h @@ -31,6 +31,8 @@ static inline void numa_policy_reset(NUMAPolicy *p) { int apply_numa_policy(const NUMAPolicy *policy); int numa_to_cpu_set(const NUMAPolicy *policy, CPUSet *ret); +int numa_get_node_from_cpu(unsigned cpu, unsigned *ret); + int numa_mask_add_all(CPUSet *mask); DECLARE_STRING_TABLE_LOOKUP(mpol, int); diff --git a/src/udev/net/link-config.c b/src/udev/net/link-config.c index 4a1512365d..6211081d79 100644 --- a/src/udev/net/link-config.c +++ b/src/udev/net/link-config.c @@ -1,6 +1,7 @@ /* SPDX-License-Identifier: LGPL-2.1-or-later */ #include +#include #include #include @@ -32,13 +33,16 @@ #include "netif-util.h" #include "netlink-util.h" #include "network-util.h" +#include "numa-util.h" #include "parse-util.h" #include "path-util.h" #include "proc-cmdline.h" #include "random-util.h" #include "socket-util.h" +#include "sort-util.h" #include "specifier.h" #include "stat-util.h" +#include "stdio-util.h" #include "string-table.h" #include "string-util.h" #include "strv.h" @@ -924,6 +928,456 @@ static int link_apply_sr_iov_config(Link *link) { return 0; } +/* CPU topology information for IRQ affinity spread algorithm. */ +typedef struct CPUTopology { + unsigned cpu; + unsigned numa_node; + unsigned package_id; + unsigned die_id; /* L3 cache domain / chiplet */ + unsigned core_id; + bool is_first_thread; /* First hyperthread of a physical core */ +} CPUTopology; + +/* Die (L3 cache domain) information for spread algorithm */ +typedef struct DieInfo { + unsigned die_id; + unsigned *cpus; /* CPUs in this die (first HT only, sorted by core) */ + size_t cpu_count; + size_t next_idx; /* For round-robin within die */ +} DieInfo; + +/* Returns the first thread of a CPU siblings list */ +static int cpu_topology_get_first_thread(sd_device *cpu_node, unsigned *ret) { + const char *content, *end; + int r; + + assert(cpu_node); + assert(ret); + + r = sd_device_get_sysattr_value(cpu_node, "topology/thread_siblings_list", &content); + if (r < 0) + return r; + + end = content + strcspn(content, ",-"); + + _cleanup_free_ char *first = strndup(content, end - content); + if (!first) + return -ENOMEM; + + return safe_atou(first, ret); +} + +static int cpu_topology_compare(const CPUTopology *a, const CPUTopology *b) { + int r; + + assert(a); + assert(b); + + /* Sort by die first (for L3 cache grouping), then core, then CPU number */ + r = CMP(a->die_id, b->die_id); + if (r != 0) + return r; + + r = CMP(a->core_id, b->core_id); + if (r != 0) + return r; + + return CMP(a->cpu, b->cpu); +} + +/* Comparison function for sorting CPUs by CPU number (for die ID assignment) */ +static int cpu_number_compare(const CPUTopology *a, const CPUTopology *b) { + assert(a); + assert(b); + + return CMP(a->cpu, b->cpu); +} + +/* Assign logical die IDs based on L3 cache sharing topology. + * + * For IRQ spreading, the goal is to distribute interrupts across CPUs that + * don't share cache, minimizing cache line contention when processing packets. + * The L3 cache boundary is the key locality domain: CPUs sharing an L3 can + * exchange data cheaply, while cross-L3 communication is expensive. + * + * We use L3 shared_cpu_list rather than the kernel's physical die_id because: + * - On AMD EPYC, multiple CCXs on the same physical die have separate L3 caches + * - On Intel with Sub-NUMA Clustering, one die may have multiple L3 domains + * - L3 sharing reflects actual data locality, not physical packaging */ +static int assign_sequential_die_ids(CPUTopology *cpus, size_t count) { + _cleanup_strv_free_ char **l3_groups = NULL; + int r; + + assert(cpus); + + /* First, sort CPUs by CPU number for consistent discovery order */ + typesafe_qsort(cpus, count, cpu_number_compare); + + /* Assign die IDs based on order of L3 shared_cpu_list discovery */ + FOREACH_ARRAY(cpu, cpus, count) { + _cleanup_(sd_device_unrefp) sd_device *cpu_node = NULL; + char cpu_path[STRLEN("/sys/devices/system/cpu/cpu") + DECIMAL_STR_MAX(unsigned) + 1]; + const char *l3_list; + unsigned die_id = 0; + bool found = false; + + xsprintf(cpu_path, "/sys/devices/system/cpu/cpu%u", cpu->cpu); + r = sd_device_new_from_syspath(&cpu_node, cpu_path); + if (r < 0) + return r; + + r = sd_device_get_sysattr_value(cpu_node, "cache/index3/shared_cpu_list", &l3_list); + if (r < 0) { + /* No L3 info, fall back to package ID */ + cpu->die_id = cpu->package_id; + continue; + } + + /* Check if we've seen this L3 group before */ + STRV_FOREACH(g, l3_groups) { + if (streq(*g, l3_list)) { + cpu->die_id = die_id; + found = true; + break; + } + die_id++; + } + + if (!found) { + /* New L3 group, assign next sequential die ID */ + cpu->die_id = die_id; + r = strv_extend(&l3_groups, l3_list); + if (r < 0) + return r; + } + } + + return 0; +} + +static int discover_cpu_topology(CPUTopology **ret, size_t *ret_count) { + _cleanup_(sd_device_unrefp) sd_device *parent_node = NULL; + _cleanup_free_ CPUTopology *cpus = NULL; + const char *name; + size_t count = 0; + int r; + + assert(ret); + assert(ret_count); + + r = sd_device_new_from_syspath(&parent_node, "/sys/devices/system/cpu"); + if (r < 0) + return r; + + FOREACH_DEVICE_CHILD_WITH_SUFFIX(parent_node, cpu_node, name) { + char topo_path[STRLEN("/sys/devices/system/cpu/cpu/topology") + DECIMAL_STR_MAX(unsigned) + 1]; + const char *n; + unsigned cpu, online, first_thread; + + n = startswith(name, "cpu"); + if (!n) + continue; + + r = safe_atou(n, &cpu); + if (r < 0) { + log_debug_errno(r, "Failed to convert %s to unsigned, skipping: %m", n); + continue; + } + + r = device_get_sysattr_unsigned(cpu_node, "online", &online); + if (r == -ENOENT) + online = 1; /* CPU 0 lacks 'online' file, assume online */ + else if (r < 0 || online == 0) + continue; + + /* Check if topology directory exists (filters out cpu0 on some systems) */ + xsprintf(topo_path, "/sys/devices/system/cpu/cpu%u/topology", cpu); + if (access(topo_path, F_OK) < 0) { + log_debug_errno(errno, "Failed to access %s, ignoring: %m", topo_path); + continue; + } + + if (!GREEDY_REALLOC(cpus, count + 1)) + return -ENOMEM; + + cpus[count].cpu = cpu; + + r = numa_get_node_from_cpu(cpu, &cpus[count].numa_node); + if (r < 0) { + log_debug_errno(r, "Failed to get NUMA node for CPU %u, assuming NUMA node 0: %m", cpu); + cpus[count].numa_node = 0; + } + + r = device_get_sysattr_unsigned(cpu_node, "topology/physical_package_id", &cpus[count].package_id); + if (r < 0) { + log_device_debug_errno(cpu_node, r, "Failed to get physical_package_id, assuming package ID 0: %m"); + cpus[count].package_id = 0; + } + + /* die_id will be assigned later by assign_sequential_die_ids() */ + cpus[count].die_id = 0; + + r = device_get_sysattr_unsigned(cpu_node, "topology/core_id", &cpus[count].core_id); + if (r < 0) { + log_device_debug_errno(cpu_node, r, "Failed to get core_id, assuming core ID %u: %m", cpu); + cpus[count].core_id = cpu; + } + + r = cpu_topology_get_first_thread(cpu_node, &first_thread); + if (r < 0) + cpus[count].is_first_thread = true; + else + cpus[count].is_first_thread = (first_thread == cpu); + + count++; + } + + if (count == 0) + return -ENOENT; + + /* Assign sequential die IDs based on L3 discovery order */ + r = assign_sequential_die_ids(cpus, count); + if (r < 0) + return r; + + /* Sort CPUs by topology for consistent ordering */ + typesafe_qsort(cpus, count, cpu_topology_compare); + + *ret = TAKE_PTR(cpus); + *ret_count = count; + + return 0; +} + +/* Reorder indices so consecutive elements are maximally spread apart. + * + * Uses recursive divide-and-conquer: split in half, permute each half, + * then interleave. This ensures elements originally far apart become adjacent. + * + * Example trace for [0,1,2,3,4,5,6,7]: + * split into [0,1,2,3] and [4,5,6,7] + * recurse left: [0,1,2,3] -> [0,2,1,3] + * recurse right: [4,5,6,7] -> [4,6,5,7] + * interleave -> [0,4,2,6,1,5,3,7] + * + * The first N elements of the output are roughly evenly distributed across the + * original range, for any N. This is useful when assigning IRQs to CPUs: if a + * NIC has fewer IRQs than CPUs, the assigned CPUs will still be spread across + * the CPUs rather than all at the beginning. */ +static int equidist_permute(size_t *indices, size_t n_indices) { + _cleanup_free_ size_t *left = NULL, *right = NULL; + size_t left_count, right_count; + size_t li = 0, ri = 0, ti = 0; + int r; + + assert(indices); + + if (n_indices <= 1) + return 0; + + left_count = DIV_ROUND_UP(n_indices, 2); + right_count = n_indices - left_count; + + /* Recursively permute each half */ + left = newdup(size_t, indices, left_count); + right = newdup(size_t, &indices[left_count], right_count); + if (!left || !right) + return log_oom(); + + r = equidist_permute(left, left_count); + if (r < 0) + return r; + + r = equidist_permute(right, right_count); + if (r < 0) + return r; + + /* Interleave: left[0], right[0], left[1], right[1], ... */ + for (size_t i = 0; i < n_indices; i++) { + if (i % 2 == 0 && li < left_count) + indices[ti++] = left[li++]; + else if (ri < right_count) + indices[ti++] = right[ri++]; + else if (li < left_count) + indices[ti++] = left[li++]; + } + + return 0; +} + +static void die_info_free(DieInfo *dies, size_t count) { + assert(dies || count == 0); + + FOREACH_ARRAY(die, dies, count) + free(die->cpus); + free(dies); +} + +/* Build die information from topology, grouping CPUs by L3/die and filtering to first HT only */ +static int build_die_info(const CPUTopology *topology, size_t topology_count, DieInfo **ret, size_t *ret_count) { + DieInfo *dies = NULL; + size_t die_count = 0; + int r; + + assert(topology); + assert(ret); + assert(ret_count); + + CLEANUP_ARRAY(dies, die_count, die_info_free); + + FOREACH_ARRAY(cpu_topology, topology, topology_count) { + DieInfo *die = NULL; + + /* Only consider first hyperthreads for initial spread */ + if (!cpu_topology->is_first_thread) + continue; + + /* Find or create die entry */ + for (size_t j = 0; j < die_count; j++) + if (dies[j].die_id == cpu_topology->die_id) { + die = &dies[j]; + break; + } + + if (!die) { + if (!GREEDY_REALLOC(dies, die_count + 1)) + return log_oom(); + die = &dies[die_count++]; + *die = (DieInfo) { .die_id = cpu_topology->die_id }; + } + + if (!GREEDY_REALLOC(die->cpus, die->cpu_count + 1)) + return log_oom(); + + die->cpus[die->cpu_count++] = cpu_topology->cpu; + } + + /* Sort dies by die_id for determinism, then apply equidist to CPUs within each die */ + FOREACH_ARRAY(die, dies, die_count) { + _cleanup_free_ unsigned *reordered = NULL; + _cleanup_free_ size_t *indices = new(size_t, die->cpu_count); + if (!indices) + return log_oom(); + + for (size_t j = 0; j < die->cpu_count; j++) + indices[j] = j; + + r = equidist_permute(indices, die->cpu_count); + if (r < 0) + return r; + + /* Reorder CPUs according to equidist permutation */ + reordered = new(unsigned, die->cpu_count); + if (!reordered) + return log_oom(); + + for (size_t j = 0; j < die->cpu_count; j++) + reordered[j] = die->cpus[indices[j]]; + + memcpy(die->cpus, reordered, die->cpu_count * sizeof(unsigned)); + } + + *ret = TAKE_PTR(dies); + *ret_count = die_count; + + return 0; +} + +/* Select CPUs for IRQ affinity spreading with optimal topology distribution. + * + * Algorithm: + * 1. Group CPUs by die (L3 cache domain), using only first hyperthreads + * 2. Apply equidistant permutation to both die order and CPUs within each die, + * so consecutive selections are maximally spread (e.g., [0,1,2,3] -> [0,2,1,3]) + * 3. Round-robin across dies, picking one CPU per die per round + * 4. If more IRQs than physical cores, wrap around and reuse the same CPUs + * + * Ensures each IRQ gets a dedicated physical core before any core handles + * multiple IRQs. Two IRQs on one physical core time-share but benefit from warm + * cache, whereas spreading across SMT siblings causes resource contention with + * no cache benefit. + * Maximizes physical distance between consecutively assigned IRQs, improving + * cache distribution even when only a few IRQs are assigned. */ +static int select_spread_cpus( + const CPUTopology *topology, + size_t topology_count, + size_t n_irqs, + unsigned **ret, + size_t *ret_count) { + + _cleanup_free_ unsigned *selected = NULL; + _cleanup_free_ size_t *die_order = NULL; + DieInfo *dies = NULL; + size_t die_count = 0, selected_count = 0; + int r; + + assert(topology); + assert(ret); + assert(ret_count); + + CLEANUP_ARRAY(dies, die_count, die_info_free); + + selected = new(unsigned, n_irqs); + if (!selected) + return -ENOMEM; + + /* Build die information with first HT CPUs only */ + r = build_die_info(topology, topology_count, &dies, &die_count); + if (r < 0) + return r; + + if (die_count == 0) + return -ENOENT; + + /* Create equidistant die ordering */ + die_order = new(size_t, die_count); + if (!die_order) + return -ENOMEM; + + for (size_t i = 0; i < die_count; i++) + die_order[i] = i; + + r = equidist_permute(die_order, die_count); + if (r < 0) + return r; + + /* Round-robin across dies, picking one CPU from each die at a time */ + size_t dies_exhausted = 0; + while (selected_count < n_irqs) { + bool made_progress = false; + + for (size_t i = 0; i < die_count && selected_count < n_irqs; i++) { + DieInfo *die = &dies[die_order[i]]; + + if (die->next_idx >= die->cpu_count) + continue; + + selected[selected_count++] = die->cpus[die->next_idx++]; + made_progress = true; + + if (die->next_idx >= die->cpu_count) + dies_exhausted++; + } + + if (made_progress) + continue; + + /* All first HTs exhausted, wrap around for remaining IRQs */ + if (dies_exhausted < die_count) + break; + + /* Reset all dies for round-robin wrap */ + FOREACH_ARRAY(die, dies, die_count) + die->next_idx = 0; + dies_exhausted = 0; + } + + *ret = TAKE_PTR(selected); + *ret_count = selected_count; + + return 0; +} + static int set_irq_affinity(Link *link, unsigned irq, unsigned cpu) { _cleanup_free_ char *affinity_path = NULL, *mask_str = NULL; unsigned n_groups = cpu / 32; @@ -954,6 +1408,64 @@ static int set_irq_affinity(Link *link, unsigned irq, unsigned cpu) { return 0; } +static int link_apply_irq_affinity_spread(Link *link) { + _cleanup_closedir_ DIR *dir = NULL; + _cleanup_free_ CPUTopology *topology = NULL; + _cleanup_free_ unsigned *irqs = NULL; + _cleanup_free_ unsigned *spread_cpus = NULL; + size_t topology_count = 0, irq_count = 0, spread_count = 0; + int r; + + assert(link); + + r = device_opendir(link->event->dev, "device/msi_irqs", &dir); + if (r < 0) { + if (r != -ENOENT) + return log_link_error_errno(link, r, "Failed to open device/msi_irqs: %m"); + log_link_debug_errno(link, r, "No MSI IRQs found, skipping IRQ affinity configuration: %m"); + return 0; + } + + FOREACH_DIRENT(de, dir, return log_link_error_errno(link, errno, "Failed to read directory device/msi_irqs: %m")) { + unsigned irq; + + r = safe_atou(de->d_name, &irq); + if (r < 0) + return log_link_error_errno(link, r, "Failed to convert parse IRQ number: %s", de->d_name); + + if (!GREEDY_REALLOC(irqs, irq_count + 1)) + return log_oom(); + + irqs[irq_count++] = irq; + } + + if (irq_count == 0) { + log_link_debug(link, "No IRQs found, skipping spread."); + return 0; + } + + typesafe_qsort(irqs, irq_count, cmp_unsigned); + + r = discover_cpu_topology(&topology, &topology_count); + if (r < 0) + return log_link_error_errno(link, r, "Failed to discover CPU topology: %m"); + + log_link_debug(link, "Discovered %zu CPUs, spreading %zu IRQs.", topology_count, irq_count); + + /* Select CPUs using maximum distance algorithm */ + r = select_spread_cpus(topology, topology_count, irq_count, &spread_cpus, &spread_count); + if (r < 0) + return log_link_error_errno(link, r, "Failed to select spread CPUs: %m"); + + for (size_t i = 0; i < spread_count; i++) + (void) set_irq_affinity(link, irqs[i], spread_cpus[i]); + + log_link_info(link, "Applied IRQ affinity policy 'spread' across %zu CPUs for %zu IRQs.", + MIN(topology_count, irq_count), irq_count); + + return 0; +} + static int link_apply_irq_affinity_single(Link *link) { _cleanup_closedir_ DIR *dir = NULL; int r; @@ -1009,6 +1521,8 @@ static int link_apply_irq_affinity(Link *link) { switch (link->config->irq_affinity_policy) { case IRQ_AFFINITY_POLICY_SINGLE: return link_apply_irq_affinity_single(link); + case IRQ_AFFINITY_POLICY_SPREAD: + return link_apply_irq_affinity_spread(link); default: assert_not_reached(); } @@ -1499,6 +2013,7 @@ DEFINE_CONFIG_PARSE_ENUMV(config_parse_alternative_names_policy, alternative_nam static const char* const irq_affinity_policy_table[_IRQ_AFFINITY_POLICY_MAX] = { [IRQ_AFFINITY_POLICY_SINGLE] = "single", + [IRQ_AFFINITY_POLICY_SPREAD] = "spread", }; DEFINE_STRING_TABLE_LOOKUP(irq_affinity_policy, IRQAffinityPolicy); diff --git a/src/udev/net/link-config.h b/src/udev/net/link-config.h index b6853cd922..ff581d8b02 100644 --- a/src/udev/net/link-config.h +++ b/src/udev/net/link-config.h @@ -24,6 +24,7 @@ typedef enum MACAddressPolicy { typedef enum IRQAffinityPolicy { IRQ_AFFINITY_POLICY_SINGLE, + IRQ_AFFINITY_POLICY_SPREAD, _IRQ_AFFINITY_POLICY_MAX, _IRQ_AFFINITY_POLICY_INVALID = -EINVAL, } IRQAffinityPolicy; diff --git a/test/units/TEST-17-UDEV.irq-affinity.sh b/test/units/TEST-17-UDEV.irq-affinity.sh index 5fa8bd331f..f9dde104fa 100755 --- a/test/units/TEST-17-UDEV.irq-affinity.sh +++ b/test/units/TEST-17-UDEV.irq-affinity.sh @@ -85,6 +85,47 @@ EOF fi done + # Test 1b: test spread policy on the same interface + cat >/run/systemd/network/00-test-irq-affinity.link <1 CPU and >1 IRQ)" + fi + # Cleanup rm -f /run/systemd/network/00-test-irq-affinity.link udevadm control --reload