diff --git a/man/systemd.link.xml b/man/systemd.link.xml index d26431b2b2..1d6fbf9fb8 100644 --- a/man/systemd.link.xml +++ b/man/systemd.link.xml @@ -1080,6 +1080,62 @@ + + IRQAffinityPolicy= + + Specifies the IRQ distribution strategy for network interface MSI/MSI-X interrupts. Takes one + of spread or single. When set to spread, + queue IRQs are distributed across CPUs using a topology-aware maximum distance algorithm that + prefers CPUs on different NUMA nodes, then different physical cores, then different hyperthreads. + When set to single, all IRQs are pinned to a single CPU (CPU 0 by default, or + the first CPU in the set specified by IRQAffinity=). When there are more IRQs + than available CPUs, queues wrap around using round-robin assignment. When unset, no affinity + management is performed. Setting this to an empty string explicitly disables affinity management. + + This option only applies to devices with MSI/MSI-X interrupts discoverable via + /sys/class/net/iface/device/msi_irqs/. Virtual + devices (veth, tap) and legacy INTx devices are skipped with a notice logged. + + Note that if irqbalance1 + or similar IRQ management daemons are running, they may override the configured + affinity. Consider disabling such daemons or configuring them to exclude managed interfaces. + + + + + + IRQAffinity= + + Filters the set of CPUs eligible for IRQ placement. Takes a list of CPU indices or ranges + separated by either whitespace or commas (e.g., 0-3, 0,2,4,6, + 0-3,8-11). This option works in conjunction with + IRQAffinityPolicy= and IRQAffinityNUMA= to constrain which + CPUs receive network IRQs. When specified with spread policy, only the listed + CPUs are considered for IRQ distribution. When specified with single policy, + IRQs are pinned to the first CPU in the allowed set instead of CPU 0. Has no effect if + IRQAffinityPolicy= is not set. + + + + + + IRQAffinityNUMA= + + Filters CPUs to those belonging to the specified NUMA node. Takes either + local or an explicit NUMA node number (0, 1, 2, ...). When set to + local, the NUMA node local to the NIC's PCIe slot is used (determined from + /sys/class/net/iface/device/numa_node). If the + device's NUMA node cannot be determined (e.g., non-NUMA system), a warning is logged and IRQ + affinity configuration is skipped. + + When both IRQAffinity= and IRQAffinityNUMA= are + specified, their intersection is used. If the intersection results in an empty set, an error is + logged and no affinity is applied. Has no effect if IRQAffinityPolicy= is not + set. + + + + ReceiveVLANCTAGHardwareAcceleration= diff --git a/src/basic/sort-util.c b/src/basic/sort-util.c index a76a6c85a8..8aca5cac76 100644 --- a/src/basic/sort-util.c +++ b/src/basic/sort-util.c @@ -81,3 +81,12 @@ int cmp_uint16(const uint16_t *a, const uint16_t *b) { return CMP(*a, *b); } + +int cmp_unsigned(const unsigned *a, const unsigned *b) { + /* This is called from qsort()s inner loops. Correctly implemented qsort will never pass NULL so we + just suppress the check via POINTER_MAY_BE_NULL instead of assert() to avoid the runtime cost. */ + POINTER_MAY_BE_NULL(a); + POINTER_MAY_BE_NULL(b); + + return CMP(*a, *b); +} diff --git a/src/basic/sort-util.h b/src/basic/sort-util.h index 8e7a1991bb..37c196158e 100644 --- a/src/basic/sort-util.h +++ b/src/basic/sort-util.h @@ -44,3 +44,4 @@ void qsort_r_safe(void *base, size_t nmemb, size_t size, comparison_userdata_fn_ int cmp_int(const int *a, const int *b); int cmp_uint16(const uint16_t *a, const uint16_t *b); +int cmp_unsigned(const unsigned *a, const unsigned *b); diff --git a/src/shared/numa-util.c b/src/shared/numa-util.c index 228ea7ad2d..34ddc0e547 100644 --- a/src/shared/numa-util.c +++ b/src/shared/numa-util.c @@ -89,6 +89,22 @@ int apply_numa_policy(const NUMAPolicy *policy) { return 0; } +int numa_node_get_cpus(size_t node, CPUSet *ret) { + char p[STRLEN("/sys/devices/system/node/node//cpulist") + DECIMAL_STR_MAX(size_t) + 1]; + _cleanup_free_ char *cpulist = NULL; + int r; + + assert(ret); + + xsprintf(p, "/sys/devices/system/node/node%zu/cpulist", node); + + r = read_virtual_file(p, SIZE_MAX, &cpulist, /* ret_size= */ NULL); + if (r < 0) + return r; + + return parse_cpu_set(cpulist, ret); +} + int numa_to_cpu_set(const NUMAPolicy *policy, CPUSet *ret) { _cleanup_(cpu_set_done) CPUSet s = {}; int r; @@ -97,20 +113,11 @@ int numa_to_cpu_set(const NUMAPolicy *policy, CPUSet *ret) { assert(ret); for (size_t i = 0; i < policy->nodes.allocated * 8; i++) { - _cleanup_free_ char *l = NULL; - char p[STRLEN("/sys/devices/system/node/node//cpulist") + DECIMAL_STR_MAX(size_t) + 1]; - if (!CPU_ISSET_S(i, policy->nodes.allocated, policy->nodes.set)) continue; - xsprintf(p, "/sys/devices/system/node/node%zu/cpulist", i); - - r = read_one_line_file(p, &l); - if (r < 0) - return r; - _cleanup_(cpu_set_done) CPUSet part = {}; - r = parse_cpu_set(l, &part); + r = numa_node_get_cpus(i, &part); if (r < 0) return r; @@ -154,6 +161,63 @@ static int numa_max_node(void) { return max_node; } +int numa_get_node_from_cpu(unsigned cpu, unsigned *ret) { + _cleanup_closedir_ DIR *d = NULL; + int r; + + assert(ret); + + d = opendir("/sys/devices/system/node"); + if (!d) + return -errno; + + FOREACH_DIRENT(de, d, break) { + char p[STRLEN("/sys/devices/system/node/node/cpulist") + DECIMAL_STR_MAX(unsigned) + 1]; + _cleanup_(cpu_set_done) CPUSet cpus = {}; + _cleanup_free_ char *cpulist = NULL; + const char *n; + unsigned node; + + if (de->d_type != DT_DIR) + continue; + + n = startswith(de->d_name, "node"); + if (!n) + continue; + + r = safe_atou(n, &node); + if (r < 0) { + log_debug_errno(r, "Failed to parse node number %s to unsigned, ignoring: %m", n); + continue; + } + + xsprintf(p, "/sys/devices/system/node/node%u/cpulist", node); + + r = read_virtual_file(p, SIZE_MAX, &cpulist, /* ret_size= */ NULL); + if (r < 0) { + log_debug_errno(r, "Failed to read %s, ignoring: %m", p); + continue; + } + + r = parse_cpu_set(cpulist, &cpus); + if (r < 0) { + log_debug_errno(r, "Failed to parse cpu set %s, ignoring: %m", cpulist); + continue; + } + + if (CPU_ISSET_S(cpu, cpus.allocated, cpus.set)) { + *ret = node; + return 0; + } + } + + /* CPU not found in any NUMA node, assume node 0 */ + log_debug("CPU %u not found in any NUMA node, assuming node 0.", cpu); + *ret = 0; + + return 0; +} + int numa_mask_add_all(CPUSet *mask) { int m; diff --git a/src/shared/numa-util.h b/src/shared/numa-util.h index c684dea803..01079351b0 100644 --- a/src/shared/numa-util.h +++ b/src/shared/numa-util.h @@ -29,8 +29,11 @@ static inline void numa_policy_reset(NUMAPolicy *p) { } int apply_numa_policy(const NUMAPolicy *policy); +int numa_node_get_cpus(size_t node, CPUSet *ret); int numa_to_cpu_set(const NUMAPolicy *policy, CPUSet *ret); +int numa_get_node_from_cpu(unsigned cpu, unsigned *ret); + int numa_mask_add_all(CPUSet *mask); DECLARE_STRING_TABLE_LOOKUP(mpol, int); diff --git a/src/udev/net/link-config-gperf.gperf b/src/udev/net/link-config-gperf.gperf index ef68dab339..a005cadd49 100644 --- a/src/udev/net/link-config-gperf.gperf +++ b/src/udev/net/link-config-gperf.gperf @@ -132,6 +132,10 @@ Link.TxMaxCoalescedHighFrames, config_parse_coalesce_u32, Link.CoalescePacketRateSampleIntervalSec, config_parse_coalesce_sec, 0, offsetof(LinkConfig, coalesce.rate_sample_interval) /* Rx RPS CPU mask */ Link.ReceivePacketSteeringCPUMask, config_parse_rps_cpu_mask, 0, offsetof(LinkConfig, rps_cpu_mask) +/* IRQ affinity settings */ +Link.IRQAffinityPolicy, config_parse_irq_affinity_policy, 0, offsetof(LinkConfig, irq_affinity_policy) +Link.IRQAffinity, config_parse_cpu_set, 0, offsetof(LinkConfig, irq_affinity_cpus) +Link.IRQAffinityNUMA, config_parse_irq_affinity_numa, 0, offsetof(LinkConfig, irq_affinity_numa) /* SR-IOV settings */ Link.SR-IOVVirtualFunctions, config_parse_sr_iov_num_vfs, 0, offsetof(LinkConfig, sr_iov_num_vfs) SR-IOV.VirtualFunction, config_parse_sr_iov_uint32, 0, offsetof(LinkConfig, sr_iov_by_section) diff --git a/src/udev/net/link-config.c b/src/udev/net/link-config.c index 3c4b886e67..306686845e 100644 --- a/src/udev/net/link-config.c +++ b/src/udev/net/link-config.c @@ -1,6 +1,7 @@ /* SPDX-License-Identifier: LGPL-2.1-or-later */ #include +#include #include #include @@ -15,6 +16,7 @@ #include "creds-util.h" #include "device-private.h" #include "device-util.h" +#include "dirent-util.h" #include "escape.h" #include "ether-addr-util.h" #include "ethtool-util.h" @@ -31,13 +33,16 @@ #include "netif-util.h" #include "netlink-util.h" #include "network-util.h" +#include "numa-util.h" #include "parse-util.h" #include "path-util.h" #include "proc-cmdline.h" #include "random-util.h" #include "socket-util.h" +#include "sort-util.h" #include "specifier.h" #include "stat-util.h" +#include "stdio-util.h" #include "string-table.h" #include "string-util.h" #include "strv.h" @@ -78,6 +83,7 @@ static LinkConfig* link_config_free(LinkConfig *config) { free(config->wol_password_file); erase_and_free(config->wol_password); cpu_set_done(&config->rps_cpu_mask); + cpu_set_done(&config->irq_affinity_cpus); ordered_hashmap_free(config->sr_iov_by_section); @@ -270,6 +276,8 @@ int link_load_one(LinkConfigContext *ctx, const char *filename) { .eee_enabled = -1, .eee_tx_lpi_enabled = -1, .eee_tx_lpi_timer_usec = USEC_INFINITY, + .irq_affinity_policy = _IRQ_AFFINITY_POLICY_INVALID, + .irq_affinity_numa = IRQ_AFFINITY_NUMA_UNSET, }; FOREACH_ELEMENT(feature, config->features) @@ -922,6 +930,741 @@ static int link_apply_sr_iov_config(Link *link) { return 0; } +/* Get the local NUMA node for a network device from sysfs. + * Returns -ENOENT if numa_node file doesn't exist or shows -1 (no NUMA). */ +static int link_get_device_numa_node(Link *link, unsigned *ret) { + int r, node; + + assert(link); + assert(link->event); + assert(link->event->dev); + assert(ret); + + r = device_get_sysattr_int(link->event->dev, "device/numa_node", &node); + if (r < 0) + return r; + + /* -1 means no NUMA node (non-NUMA system or device not associated with a node) */ + if (node < 0) + return -ENOENT; + + *ret = (unsigned) node; + return 0; +} + +/* CPU topology information for IRQ affinity spread algorithm. */ +typedef struct CPUTopology { + unsigned cpu; + unsigned numa_node; + unsigned package_id; + unsigned die_id; /* L3 cache domain / chiplet */ + unsigned core_id; + bool is_first_thread; /* First hyperthread of a physical core */ +} CPUTopology; + +/* Die (L3 cache domain) information for spread algorithm */ +typedef struct DieInfo { + unsigned die_id; + unsigned *cpus; /* CPUs in this die (first HT only, sorted by core) */ + size_t cpu_count; + size_t next_idx; /* For round-robin within die */ +} DieInfo; + +/* Returns the first thread of a CPU siblings list */ +static int cpu_topology_get_first_thread(sd_device *cpu_node, unsigned *ret) { + const char *content, *end; + int r; + + assert(cpu_node); + assert(ret); + + r = sd_device_get_sysattr_value(cpu_node, "topology/thread_siblings_list", &content); + if (r < 0) + return r; + + end = content + strcspn(content, ",-"); + + _cleanup_free_ char *first = strndup(content, end - content); + if (!first) + return -ENOMEM; + + return safe_atou(first, ret); +} + +static int cpu_topology_compare(const CPUTopology *a, const CPUTopology *b) { + int r; + + assert(a); + assert(b); + + /* Sort by die first (for L3 cache grouping), then core, then CPU number */ + r = CMP(a->die_id, b->die_id); + if (r != 0) + return r; + + r = CMP(a->core_id, b->core_id); + if (r != 0) + return r; + + return CMP(a->cpu, b->cpu); +} + +/* Comparison function for sorting CPUs by CPU number (for die ID assignment) */ +static int cpu_number_compare(const CPUTopology *a, const CPUTopology *b) { + assert(a); + assert(b); + + return CMP(a->cpu, b->cpu); +} + +/* Assign logical die IDs based on L3 cache sharing topology. + * + * For IRQ spreading, the goal is to distribute interrupts across CPUs that + * don't share cache, minimizing cache line contention when processing packets. + * The L3 cache boundary is the key locality domain: CPUs sharing an L3 can + * exchange data cheaply, while cross-L3 communication is expensive. + * + * We use L3 shared_cpu_list rather than the kernel's physical die_id because: + * - On AMD EPYC, multiple CCXs on the same physical die have separate L3 caches + * - On Intel with Sub-NUMA Clustering, one die may have multiple L3 domains + * - L3 sharing reflects actual data locality, not physical packaging */ +static int assign_sequential_die_ids(CPUTopology *cpus, size_t count) { + _cleanup_strv_free_ char **l3_groups = NULL; + int r; + + assert(cpus); + + /* First, sort CPUs by CPU number for consistent discovery order */ + typesafe_qsort(cpus, count, cpu_number_compare); + + /* Assign die IDs based on order of L3 shared_cpu_list discovery */ + FOREACH_ARRAY(cpu, cpus, count) { + _cleanup_(sd_device_unrefp) sd_device *cpu_node = NULL; + char cpu_path[STRLEN("/sys/devices/system/cpu/cpu") + DECIMAL_STR_MAX(unsigned) + 1]; + const char *l3_list; + unsigned die_id = 0; + bool found = false; + + xsprintf(cpu_path, "/sys/devices/system/cpu/cpu%u", cpu->cpu); + r = sd_device_new_from_syspath(&cpu_node, cpu_path); + if (r < 0) + return r; + + r = sd_device_get_sysattr_value(cpu_node, "cache/index3/shared_cpu_list", &l3_list); + if (r < 0) { + /* No L3 info, fall back to package ID */ + cpu->die_id = cpu->package_id; + continue; + } + + /* Check if we've seen this L3 group before */ + STRV_FOREACH(g, l3_groups) { + if (streq(*g, l3_list)) { + cpu->die_id = die_id; + found = true; + break; + } + die_id++; + } + + if (!found) { + /* New L3 group, assign next sequential die ID */ + cpu->die_id = die_id; + r = strv_extend(&l3_groups, l3_list); + if (r < 0) + return r; + } + } + + return 0; +} + +static int discover_cpu_topology(CPUTopology **ret, size_t *ret_count) { + _cleanup_(sd_device_unrefp) sd_device *parent_node = NULL; + _cleanup_free_ CPUTopology *cpus = NULL; + const char *name; + size_t count = 0; + int r; + + assert(ret); + assert(ret_count); + + r = sd_device_new_from_syspath(&parent_node, "/sys/devices/system/cpu"); + if (r < 0) + return r; + + FOREACH_DEVICE_CHILD_WITH_SUFFIX(parent_node, cpu_node, name) { + char topo_path[STRLEN("/sys/devices/system/cpu/cpu/topology") + DECIMAL_STR_MAX(unsigned) + 1]; + const char *n; + unsigned cpu, online, first_thread; + + n = startswith(name, "cpu"); + if (!n) + continue; + + r = safe_atou(n, &cpu); + if (r < 0) { + log_debug_errno(r, "Failed to convert %s to unsigned, skipping: %m", n); + continue; + } + + r = device_get_sysattr_unsigned(cpu_node, "online", &online); + if (r == -ENOENT) + online = 1; /* CPU 0 lacks 'online' file, assume online */ + else if (r < 0 || online == 0) + continue; + + /* Check if topology directory exists (filters out cpu0 on some systems) */ + xsprintf(topo_path, "/sys/devices/system/cpu/cpu%u/topology", cpu); + if (access(topo_path, F_OK) < 0) { + log_debug_errno(errno, "Failed to access %s, ignoring: %m", topo_path); + continue; + } + + if (!GREEDY_REALLOC(cpus, count + 1)) + return -ENOMEM; + + cpus[count].cpu = cpu; + + r = numa_get_node_from_cpu(cpu, &cpus[count].numa_node); + if (r < 0) { + log_debug_errno(r, "Failed to get NUMA node for CPU %u, assuming NUMA node 0: %m", cpu); + cpus[count].numa_node = 0; + } + + r = device_get_sysattr_unsigned(cpu_node, "topology/physical_package_id", &cpus[count].package_id); + if (r < 0) { + log_device_debug_errno(cpu_node, r, "Failed to get physical_package_id, assuming package ID 0: %m"); + cpus[count].package_id = 0; + } + + /* die_id will be assigned later by assign_sequential_die_ids() */ + cpus[count].die_id = 0; + + r = device_get_sysattr_unsigned(cpu_node, "topology/core_id", &cpus[count].core_id); + if (r < 0) { + log_device_debug_errno(cpu_node, r, "Failed to get core_id, assuming core ID %u: %m", cpu); + cpus[count].core_id = cpu; + } + + r = cpu_topology_get_first_thread(cpu_node, &first_thread); + if (r < 0) + cpus[count].is_first_thread = true; + else + cpus[count].is_first_thread = (first_thread == cpu); + + count++; + } + + if (count == 0) + return -ENOENT; + + /* Assign sequential die IDs based on L3 discovery order */ + r = assign_sequential_die_ids(cpus, count); + if (r < 0) + return r; + + /* Sort CPUs by topology for consistent ordering */ + typesafe_qsort(cpus, count, cpu_topology_compare); + + *ret = TAKE_PTR(cpus); + *ret_count = count; + + return 0; +} + +/* Reorder indices so consecutive elements are maximally spread apart. + * + * Uses recursive divide-and-conquer: split in half, permute each half, + * then interleave. This ensures elements originally far apart become adjacent. + * + * Example trace for [0,1,2,3,4,5,6,7]: + * split into [0,1,2,3] and [4,5,6,7] + * recurse left: [0,1,2,3] -> [0,2,1,3] + * recurse right: [4,5,6,7] -> [4,6,5,7] + * interleave -> [0,4,2,6,1,5,3,7] + * + * The first N elements of the output are roughly evenly distributed across the + * original range, for any N. This is useful when assigning IRQs to CPUs: if a + * NIC has fewer IRQs than CPUs, the assigned CPUs will still be spread across + * the CPUs rather than all at the beginning. */ +static int equidist_permute(size_t *indices, size_t n_indices) { + _cleanup_free_ size_t *left = NULL, *right = NULL; + size_t left_count, right_count; + size_t li = 0, ri = 0, ti = 0; + int r; + + assert(indices); + + if (n_indices <= 1) + return 0; + + left_count = DIV_ROUND_UP(n_indices, 2); + right_count = n_indices - left_count; + + /* Recursively permute each half */ + left = newdup(size_t, indices, left_count); + right = newdup(size_t, &indices[left_count], right_count); + if (!left || !right) + return log_oom(); + + r = equidist_permute(left, left_count); + if (r < 0) + return r; + + r = equidist_permute(right, right_count); + if (r < 0) + return r; + + /* Interleave: left[0], right[0], left[1], right[1], ... */ + for (size_t i = 0; i < n_indices; i++) { + if (i % 2 == 0 && li < left_count) + indices[ti++] = left[li++]; + else if (ri < right_count) + indices[ti++] = right[ri++]; + else if (li < left_count) + indices[ti++] = left[li++]; + } + + return 0; +} + +static void die_info_free(DieInfo *dies, size_t count) { + assert(dies || count == 0); + + FOREACH_ARRAY(die, dies, count) + free(die->cpus); + free(dies); +} + +/* Build die information from topology, grouping CPUs by L3/die and filtering to first HT only */ +static int build_die_info(const CPUTopology *topology, size_t topology_count, DieInfo **ret, size_t *ret_count) { + DieInfo *dies = NULL; + size_t die_count = 0; + int r; + + assert(topology); + assert(ret); + assert(ret_count); + + CLEANUP_ARRAY(dies, die_count, die_info_free); + + FOREACH_ARRAY(cpu_topology, topology, topology_count) { + DieInfo *die = NULL; + + /* Only consider first hyperthreads for initial spread */ + if (!cpu_topology->is_first_thread) + continue; + + /* Find or create die entry */ + for (size_t j = 0; j < die_count; j++) + if (dies[j].die_id == cpu_topology->die_id) { + die = &dies[j]; + break; + } + + if (!die) { + if (!GREEDY_REALLOC(dies, die_count + 1)) + return log_oom(); + die = &dies[die_count++]; + *die = (DieInfo) { .die_id = cpu_topology->die_id }; + } + + if (!GREEDY_REALLOC(die->cpus, die->cpu_count + 1)) + return log_oom(); + + die->cpus[die->cpu_count++] = cpu_topology->cpu; + } + + /* Sort dies by die_id for determinism, then apply equidist to CPUs within each die */ + FOREACH_ARRAY(die, dies, die_count) { + _cleanup_free_ unsigned *reordered = NULL; + _cleanup_free_ size_t *indices = new(size_t, die->cpu_count); + if (!indices) + return log_oom(); + + for (size_t j = 0; j < die->cpu_count; j++) + indices[j] = j; + + r = equidist_permute(indices, die->cpu_count); + if (r < 0) + return r; + + /* Reorder CPUs according to equidist permutation */ + reordered = new(unsigned, die->cpu_count); + if (!reordered) + return log_oom(); + + for (size_t j = 0; j < die->cpu_count; j++) + reordered[j] = die->cpus[indices[j]]; + + memcpy(die->cpus, reordered, die->cpu_count * sizeof(unsigned)); + } + + *ret = TAKE_PTR(dies); + *ret_count = die_count; + + return 0; +} + +/* Select CPUs for IRQ affinity spreading with optimal topology distribution. + * + * Algorithm: + * 1. Group CPUs by die (L3 cache domain), using only first hyperthreads + * 2. Apply equidistant permutation to both die order and CPUs within each die, + * so consecutive selections are maximally spread (e.g., [0,1,2,3] -> [0,2,1,3]) + * 3. Round-robin across dies, picking one CPU per die per round + * 4. If more IRQs than physical cores, wrap around and reuse the same CPUs + * + * Ensures each IRQ gets a dedicated physical core before any core handles + * multiple IRQs. Two IRQs on one physical core time-share but benefit from warm + * cache, whereas spreading across SMT siblings causes resource contention with + * no cache benefit. + * Maximizes physical distance between consecutively assigned IRQs, improving + * cache distribution even when only a few IRQs are assigned. */ +static int select_spread_cpus( + const CPUTopology *topology, + size_t topology_count, + size_t n_irqs, + unsigned **ret, + size_t *ret_count) { + + _cleanup_free_ unsigned *selected = NULL; + _cleanup_free_ size_t *die_order = NULL; + DieInfo *dies = NULL; + size_t die_count = 0, selected_count = 0; + int r; + + assert(topology); + assert(ret); + assert(ret_count); + + CLEANUP_ARRAY(dies, die_count, die_info_free); + + selected = new(unsigned, n_irqs); + if (!selected) + return -ENOMEM; + + /* Build die information with first HT CPUs only */ + r = build_die_info(topology, topology_count, &dies, &die_count); + if (r < 0) + return r; + + if (die_count == 0) + return -ENOENT; + + /* Create equidistant die ordering */ + die_order = new(size_t, die_count); + if (!die_order) + return -ENOMEM; + + for (size_t i = 0; i < die_count; i++) + die_order[i] = i; + + r = equidist_permute(die_order, die_count); + if (r < 0) + return r; + + /* Round-robin across dies, picking one CPU from each die at a time */ + size_t dies_exhausted = 0; + while (selected_count < n_irqs) { + bool made_progress = false; + + for (size_t i = 0; i < die_count && selected_count < n_irqs; i++) { + DieInfo *die = &dies[die_order[i]]; + + if (die->next_idx >= die->cpu_count) + continue; + + selected[selected_count++] = die->cpus[die->next_idx++]; + made_progress = true; + + if (die->next_idx >= die->cpu_count) + dies_exhausted++; + } + + if (made_progress) + continue; + + /* All first HTs exhausted, wrap around for remaining IRQs */ + if (dies_exhausted < die_count) + break; + + /* Reset all dies for round-robin wrap */ + FOREACH_ARRAY(die, dies, die_count) + die->next_idx = 0; + dies_exhausted = 0; + } + + *ret = TAKE_PTR(selected); + *ret_count = selected_count; + + return 0; +} + +static int set_irq_affinity(Link *link, unsigned irq, unsigned cpu) { + _cleanup_free_ char *affinity_path = NULL, *mask_str = NULL; + unsigned n_groups = cpu / 32; + int r; + + assert(link); + + if (asprintf(&affinity_path, "/proc/irq/%u/smp_affinity", irq) < 0) + return log_oom(); + + /* Convert CPU number to hex bitmask. + * For CPU N, set bit N (1 << N). CPUs are split by comma-separated + * 32-bits groups. To assign CPU 32, we should write 1,00000000 */ + + if (asprintf(&mask_str, "%x", 1U << (cpu % 32)) < 0) + return log_oom(); + + for (unsigned i = 0; i < n_groups; i++) + if (!strextend(&mask_str, ",00000000")) + return log_oom(); + + r = write_string_file(affinity_path, mask_str, WRITE_STRING_FILE_DISABLE_BUFFER); + if (r < 0) + return log_link_warning_errno(link, r, "Failed to set IRQ %u affinity to CPU %u: %m", irq, cpu); + + log_link_debug(link, "Set IRQ %u affinity to CPU %u.", irq, cpu); + + return 0; +} + +static int link_apply_irq_affinity_spread(Link *link, const CPUSet *allowed_cpus) { + _cleanup_closedir_ DIR *dir = NULL; + _cleanup_free_ CPUTopology *topology = NULL; + _cleanup_free_ unsigned *irqs = NULL; + _cleanup_free_ unsigned *spread_cpus = NULL; + size_t topology_count = 0, irq_count = 0, spread_count = 0; + int r; + + assert(link); + + r = device_opendir(link->event->dev, "device/msi_irqs", &dir); + if (r < 0) { + if (r != -ENOENT) + return log_link_error_errno(link, r, "Failed to open device/msi_irqs: %m"); + log_link_debug_errno(link, r, "No MSI IRQs found, skipping IRQ affinity configuration: %m"); + return 0; + } + + FOREACH_DIRENT(de, dir, return log_link_error_errno(link, errno, "Failed to read directory device/msi_irqs: %m")) { + unsigned irq; + + r = safe_atou(de->d_name, &irq); + if (r < 0) + return log_link_error_errno(link, r, "Failed to convert parse IRQ number: %s", de->d_name); + + if (!GREEDY_REALLOC(irqs, irq_count + 1)) + return log_oom(); + + irqs[irq_count++] = irq; + } + + if (irq_count == 0) { + log_link_debug(link, "No IRQs found, skipping spread."); + return 0; + } + + typesafe_qsort(irqs, irq_count, cmp_unsigned); + + r = discover_cpu_topology(&topology, &topology_count); + if (r < 0) + return log_link_error_errno(link, r, "Failed to discover CPU topology: %m"); + + /* Filter topology by allowed CPUs if specified */ + if (allowed_cpus && allowed_cpus->set) { + _cleanup_free_ CPUTopology *filtered_topology = new(CPUTopology, topology_count); + size_t filtered_count = 0; + + if (!filtered_topology) + return log_oom(); + + for (size_t i = 0; i < topology_count; i++) + if (CPU_ISSET_S(topology[i].cpu, allowed_cpus->allocated, allowed_cpus->set)) + filtered_topology[filtered_count++] = topology[i]; + + if (filtered_count == 0) { + log_link_warning(link, "IRQAffinity= filter excludes all CPUs, skipping spread."); + return 0; + } + + log_link_debug(link, "Filtered to %zu CPUs (from %zu) based on IRQAffinity=.", filtered_count, topology_count); + + free_and_replace(topology, filtered_topology); + topology_count = filtered_count; + } + + log_link_debug(link, "Spreading %zu IRQs across %zu CPUs.", irq_count, topology_count); + + /* Select CPUs using maximum distance algorithm */ + r = select_spread_cpus(topology, topology_count, irq_count, &spread_cpus, &spread_count); + if (r < 0) + return log_link_error_errno(link, r, "Failed to select spread CPUs: %m"); + + for (size_t i = 0; i < spread_count; i++) + (void) set_irq_affinity(link, irqs[i], spread_cpus[i]); + + log_link_info(link, "Applied IRQ affinity policy 'spread' across %zu CPUs for %zu IRQs.", + MIN(topology_count, irq_count), irq_count); + + return 0; +} + +static int link_apply_irq_affinity_single(Link *link, const CPUSet *allowed_cpus) { + _cleanup_closedir_ DIR *dir = NULL; + unsigned target_cpu = 0; + int r; + + assert(link); + assert(link->config); + assert(ASSERT_PTR(link->event)->dev); + + /* If IRQAffinity= is specified, use the first allowed CPU instead of CPU 0 */ + if (allowed_cpus && allowed_cpus->set) { + bool found = false; + + for (unsigned cpu = 0; cpu < allowed_cpus->allocated * 8; cpu++) + if (CPU_ISSET_S(cpu, allowed_cpus->allocated, allowed_cpus->set)) { + target_cpu = cpu; + found = true; + break; + } + + if (!found) { + log_link_warning(link, "IRQAffinity= filter excludes all CPUs, skipping single."); + return 0; + } + } + + r = device_opendir(link->event->dev, "device/msi_irqs", &dir); + if (r < 0) { + if (r != -ENOENT) + return log_link_error_errno(link, r, "Failed to open device/msi_irqs: %m"); + log_link_debug_errno(link, r, "No MSI IRQs found, skipping IRQ affinity configuration: %m"); + return 0; + } + + FOREACH_DIRENT(de, dir, return log_link_error_errno(link, errno, "Failed to read directory device/msi_irqs: %m")) { + unsigned irq; + + r = safe_atou(de->d_name, &irq); + if (r < 0) + return log_link_error_errno(link, r, "Failed to convert parse IRQ number: %s", de->d_name); + + (void) set_irq_affinity(link, irq, target_cpu); + } + + log_link_info(link, "Applied IRQ affinity policy 'single' (pinning to CPU %u).", target_cpu); + + return 0; +} + +static int link_apply_irq_affinity(Link *link) { + _cleanup_(cpu_set_done) CPUSet effective_cpus = {}; + const char *syspath; + unsigned numa_node = IRQ_AFFINITY_NUMA_UNSET; + int r; + + assert(link); + assert(link->config); + assert(ASSERT_PTR(link->event)->dev); + + if (link->event->event_mode != EVENT_UDEV_WORKER) { + log_link_debug(link, "Running in test mode, skipping application of IRQ affinity settings."); + return 0; + } + + if (link->config->irq_affinity_policy < 0) + return 0; + + r = sd_device_get_syspath(link->event->dev, &syspath); + if (r < 0) + return log_link_warning_errno(link, r, "Failed to get syspath: %m"); + + /* Compute effective CPU set from IRQAffinity= and IRQAffinityNUMA= */ + if (link->config->irq_affinity_numa != IRQ_AFFINITY_NUMA_UNSET) { + _cleanup_(cpu_set_done) CPUSet numa_cpus = {}; + + /* Resolve "local" to the actual NUMA node */ + if (link->config->irq_affinity_numa == IRQ_AFFINITY_NUMA_LOCAL) { + r = link_get_device_numa_node(link, &numa_node); + if (r < 0) { + log_link_warning_errno( + link, r, + "Failed to determine local NUMA node for device, skipping IRQ affinity configuration: %m"); + return 0; + } + log_link_debug(link, "Device is on NUMA node %u.", numa_node); + } else + numa_node = link->config->irq_affinity_numa; + + /* Get CPUs for the NUMA node */ + r = numa_node_get_cpus(numa_node, &numa_cpus); + if (r < 0) { + log_link_warning_errno( + link, r, + "Failed to get CPUs for NUMA node %u, skipping IRQ affinity configuration: %m", + numa_node); + return 0; + } + + /* If IRQAffinity= is also specified, compute intersection */ + if (link->config->irq_affinity_cpus.set) { + /* Compute intersection of IRQAffinity= and NUMA CPUs */ + size_t max_allocated = MAX(numa_cpus.allocated, link->config->irq_affinity_cpus.allocated); + + r = cpu_set_realloc(&effective_cpus, max_allocated * 8); + if (r < 0) + return log_oom(); + + for (size_t i = 0; i < max_allocated * 8; i++) { + bool in_numa = i < numa_cpus.allocated * 8 && + CPU_ISSET_S(i, numa_cpus.allocated, numa_cpus.set); + bool in_affinity = i < link->config->irq_affinity_cpus.allocated * 8 && + CPU_ISSET_S(i, link->config->irq_affinity_cpus.allocated, link->config->irq_affinity_cpus.set); + + if (in_numa && in_affinity) { + r = cpu_set_add(&effective_cpus, i); + if (r < 0) + return log_oom(); + } + } + + /* Check if intersection is empty */ + if (!effective_cpus.set || CPU_COUNT_S(effective_cpus.allocated, effective_cpus.set) == 0) { + log_link_warning( + link, + "IRQAffinity= and IRQAffinityNUMA= intersection is empty, skipping IRQ affinity configuration."); + return 0; + } + + log_link_debug(link, "Using intersection of IRQAffinity= and NUMA node %u CPUs.", numa_node); + } else { + /* Only NUMA filtering, use NUMA CPUs directly */ + effective_cpus = TAKE_STRUCT(numa_cpus); + log_link_debug(link, "Using CPUs from NUMA node %u.", numa_node); + } + } else if (link->config->irq_affinity_cpus.set) { + /* Only IRQAffinity= specified, copy it */ + r = cpu_set_add_set(&effective_cpus, &link->config->irq_affinity_cpus); + if (r < 0) + return log_oom(); + } + /* else: no filtering, effective_cpus remains empty (meaning use all CPUs) */ + + switch (link->config->irq_affinity_policy) { + case IRQ_AFFINITY_POLICY_SINGLE: + return link_apply_irq_affinity_single(link, effective_cpus.set ? &effective_cpus : NULL); + case IRQ_AFFINITY_POLICY_SPREAD: + return link_apply_irq_affinity_spread(link, effective_cpus.set ? &effective_cpus : NULL); + default: + assert_not_reached(); + } +} + static int link_apply_rps_cpu_mask(Link *link) { _cleanup_free_ char *mask_str = NULL; LinkConfig *config; @@ -1051,6 +1794,10 @@ int link_apply_config(LinkConfigContext *ctx, Link *link) { if (r < 0) return r; + r = link_apply_irq_affinity(link); + if (r < 0) + return r; + return link_apply_udev_properties(link); } @@ -1400,3 +2147,61 @@ DEFINE_CONFIG_PARSE_ENUMV(config_parse_name_policy, name_policy, NamePolicy, DEFINE_CONFIG_PARSE_ENUMV(config_parse_alternative_names_policy, alternative_names_policy, NamePolicy, _NAMEPOLICY_INVALID); + +int config_parse_irq_affinity_numa( + const char *unit, + const char *filename, + unsigned line, + const char *section, + unsigned section_line, + const char *lvalue, + int ltype, + const char *rvalue, + void *data, + void *userdata) { + + unsigned tmp, *numa = ASSERT_PTR(data); + int r; + + assert(filename); + assert(lvalue); + assert(rvalue); + + if (isempty(rvalue)) { + *numa = IRQ_AFFINITY_NUMA_UNSET; + return 0; + } + + if (streq(rvalue, "local")) { + *numa = IRQ_AFFINITY_NUMA_LOCAL; + return 0; + } + + /* Parse as NUMA node number */ + r = safe_atou(rvalue, &tmp); + if (r < 0) + return log_syntax_parse_error(unit, filename, line, r, lvalue, rvalue); + + /* UINT_MAX and UINT_MAX-1 are used to flag "unset" and "local NUMA node" respectively. */ + if (tmp >= IRQ_AFFINITY_NUMA_LOCAL) { + log_syntax(unit, LOG_WARNING, filename, line, 0, + "Invalid NUMA node number %u, ignoring assignment: %s", tmp, rvalue); + return 0; + } + + *numa = tmp; + + return 0; +} + +static const char* const irq_affinity_policy_table[_IRQ_AFFINITY_POLICY_MAX] = { + [IRQ_AFFINITY_POLICY_SINGLE] = "single", + [IRQ_AFFINITY_POLICY_SPREAD] = "spread", +}; + +DEFINE_STRING_TABLE_LOOKUP(irq_affinity_policy, IRQAffinityPolicy); +DEFINE_CONFIG_PARSE_ENUM_WITH_DEFAULT( + config_parse_irq_affinity_policy, + irq_affinity_policy, + IRQAffinityPolicy, + _IRQ_AFFINITY_POLICY_INVALID); diff --git a/src/udev/net/link-config.h b/src/udev/net/link-config.h index e00327c015..a2b0def4c3 100644 --- a/src/udev/net/link-config.h +++ b/src/udev/net/link-config.h @@ -1,6 +1,8 @@ /* SPDX-License-Identifier: LGPL-2.1-or-later */ #pragma once +#include + #include "sd-device.h" #include "cpu-set-util.h" @@ -22,6 +24,17 @@ typedef enum MACAddressPolicy { _MAC_ADDRESS_POLICY_INVALID = -EINVAL, } MACAddressPolicy; +typedef enum IRQAffinityPolicy { + IRQ_AFFINITY_POLICY_SINGLE, + IRQ_AFFINITY_POLICY_SPREAD, + _IRQ_AFFINITY_POLICY_MAX, + _IRQ_AFFINITY_POLICY_INVALID = -EINVAL, +} IRQAffinityPolicy; + +/* Special values for IRQAffinityNUMA= */ +#define IRQ_AFFINITY_NUMA_UNSET UINT_MAX +#define IRQ_AFFINITY_NUMA_LOCAL (IRQ_AFFINITY_NUMA_UNSET - 1) + typedef struct Link { UdevEvent *event; LinkConfig *config; @@ -113,6 +126,11 @@ struct LinkConfig { /* Rx RPS CPU mask */ CPUSet rps_cpu_mask; + /* IRQ affinity */ + IRQAffinityPolicy irq_affinity_policy; + CPUSet irq_affinity_cpus; + unsigned irq_affinity_numa; + /* SR-IOV */ uint32_t sr_iov_num_vfs; OrderedHashmap *sr_iov_by_section; @@ -136,6 +154,7 @@ int link_get_config(LinkConfigContext *ctx, Link *link); int link_apply_config(LinkConfigContext *ctx, Link *link); DECLARE_STRING_TABLE_LOOKUP(mac_address_policy, MACAddressPolicy); +DECLARE_STRING_TABLE_LOOKUP(irq_affinity_policy, IRQAffinityPolicy); /* gperf lookup function */ const struct ConfigPerfItem* link_config_gperf_lookup(const char *str, GPERF_LEN_TYPE length); @@ -150,3 +169,5 @@ CONFIG_PARSER_PROTOTYPE(config_parse_mac_address_policy); CONFIG_PARSER_PROTOTYPE(config_parse_name_policy); CONFIG_PARSER_PROTOTYPE(config_parse_alternative_names_policy); CONFIG_PARSER_PROTOTYPE(config_parse_rps_cpu_mask); +CONFIG_PARSER_PROTOTYPE(config_parse_irq_affinity_policy); +CONFIG_PARSER_PROTOTYPE(config_parse_irq_affinity_numa); diff --git a/src/udev/net/test-link-config-tables.c b/src/udev/net/test-link-config-tables.c index 499778505c..58b628bc9b 100644 --- a/src/udev/net/test-link-config-tables.c +++ b/src/udev/net/test-link-config-tables.c @@ -8,6 +8,7 @@ int main(int argc, char **argv) { test_setup_logging(LOG_DEBUG); test_table(MACAddressPolicy, mac_address_policy, MAC_ADDRESS_POLICY); + test_table(IRQAffinityPolicy, irq_affinity_policy, IRQ_AFFINITY_POLICY); return EXIT_SUCCESS; } diff --git a/test/units/TEST-17-UDEV.irq-affinity.sh b/test/units/TEST-17-UDEV.irq-affinity.sh new file mode 100755 index 0000000000..863fc91a39 --- /dev/null +++ b/test/units/TEST-17-UDEV.irq-affinity.sh @@ -0,0 +1,429 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: LGPL-2.1-or-later +set -eux +set -o pipefail + +# shellcheck source=test/units/util.sh +. "$(dirname "$0")"/util.sh + +# Test IRQAffinityPolicy configuration + +# Find a network interface with MSI IRQs (typically virtio-net in the VM) +# We need a real device to test actual IRQ affinity application +find_interface_with_msi_irqs() { + for iface in /sys/class/net/*; do + [[ -e "$iface" ]] || continue + iface_name=$(basename "$iface") + [[ "$iface_name" == "lo" ]] && continue + msi_irqs_path="$iface/device/msi_irqs" + if [[ -d "$msi_irqs_path" ]] && [[ -n "$(ls -A "$msi_irqs_path" 2>/dev/null)" ]]; then + echo "$iface_name" + return 0 + fi + done + return 1 +} + +get_interface_irqs() { + local iface="$1" + local msi_irqs_path="/sys/class/net/$iface/device/msi_irqs" + if [[ -d "$msi_irqs_path" ]]; then + ls "$msi_irqs_path" + fi +} + +check_irq_affinity() { + local irq="$1" + local expected_mask="$2" + local affinity + affinity=$(cat "/proc/irq/$irq/smp_affinity") + # Remove leading zeros and commas for comparison + affinity=$(echo "$affinity" | sed 's/^[0,]*//;s/,//g') + expected_mask=$(echo "$expected_mask" | sed 's/^[0,]*//;s/,//g') + [[ "$affinity" == "$expected_mask" ]] +} + +# Test 1: Verify IRQ affinity is actually applied on a real device +if iface=$(find_interface_with_msi_irqs); then + echo "Found interface with MSI IRQs: $iface" + + # Get the MAC address of the interface + mac=$(cat "/sys/class/net/$iface/address") + + # Get IRQs before applying policy + irqs=$(get_interface_irqs "$iface") + echo "Interface $iface has IRQs: $irqs" + + # Test 1a: test single policy + mkdir -p /run/systemd/network/ + cat >/run/systemd/network/00-test-irq-affinity.link </run/systemd/network/00-test-irq-affinity.link <1 CPU and >1 IRQ)" + fi + + # Test 1c: Test IRQAffinity= CPU filtering with single policy + # Pin to CPU 1 instead of default CPU 0 + cat >/run/systemd/network/00-test-irq-affinity.link <1 CPU)" + fi + + # Test 1d: Test IRQAffinity= with spread policy (restrict to subset of CPUs) + if [[ "$n_cpus" -ge 4 ]] && [[ "$irq_count" -gt 1 ]]; then + cat >/run/systemd/network/00-test-irq-affinity.link <=4 CPUs and >1 IRQ)" + fi + + # Test 1e: Test IRQAffinityNUMA= if NUMA is available + if [[ -d /sys/devices/system/node/node0 ]]; then + # Get CPUs on NUMA node 0 + numa0_cpus=$(cat /sys/devices/system/node/node0/cpulist) + echo "NUMA node 0 has CPUs: $numa0_cpus" + + cat >/run/systemd/network/00-test-irq-affinity.link </run/systemd/network/00-test-irq-affinity.link </dev/null; then + echo "Empty intersection correctly detected and logged" + else + echo "Note: Empty intersection test - check journal for error message" + fi + else + echo "Skipping empty intersection test (need 2 NUMA nodes)" + fi + + # Cleanup + rm -f /run/systemd/network/00-test-irq-affinity.link + udevadm control --reload +else + echo "No interface with MSI IRQs found, skipping actual IRQ affinity test" +fi + +# Test 2: Config parsing with dummy interfaces (no MSI IRQs) +mkdir -p /run/systemd/network/ +cat >/run/systemd/network/10-test-irq.link <&1) +assert_in "ID_NET_LINK_FILE=/run/systemd/network/10-test-irq.link" "$output" + +# Test 3: Invalid policy values are rejected/warned +cat >/run/systemd/network/10-test-irq-invalid.link </run/systemd/network/10-test-irq-empty.link </run/systemd/network/10-test-irq-affinity-cpus.link <&1) +assert_in "ID_NET_LINK_FILE=/run/systemd/network/10-test-irq-affinity-cpus.link" "$output" + +# Test 6: IRQAffinityNUMA= config parsing +cat >/run/systemd/network/10-test-irq-affinity-numa.link </run/systemd/network/10-test-irq-affinity-numa-explicit.link </run/systemd/network/10-test-irq-affinity-combined.link <