network: manage SR-IOV PF and VF ports

src/network/networkd-link.c

@@ -179,6 +179,7 @@ static Link *link_free(Link *link) {
 
         link_free_engines(link);
 
+        set_free(link->sr_iov_virt_port_ifindices);
         free(link->ifname);
         strv_free(link->alternative_names);
         free(link->kind);
@@ -890,6 +891,8 @@ static Link *link_drop(Link *link) {
         link_free_bound_to_list(link);
         link_free_bound_by_list(link);
 
+        link_clear_sr_iov_ifindices(link);
+
         link_drop_from_master(link);
 
         if (link->state_file)
@@ -1389,6 +1392,8 @@ static int link_initialized_handler(sd_netlink *rtnl, sd_netlink_message *m, Lin
 }
 
 static int link_initialized(Link *link, sd_device *device) {
+        int r;
+
         assert(link);
         assert(device);
 
@@ -1396,6 +1401,10 @@ static int link_initialized(Link *link, sd_device *device) {
          * or sysattrs) may be outdated. */
         device_unref_and_replace(link->dev, device);
 
+        r = link_set_sr_iov_ifindices(link);
+        if (r < 0)
+                log_link_warning_errno(link, r, "Failed to manage SR-IOV PF and VF ports, ignoring: %m");
+
         /* Do not ignore unamanaged state case here. If an interface is renamed after being once
          * configured, and the corresponding .network file has Name= in [Match] section, then the
          * interface may be already in unmanaged state. See #20657. */

src/network/networkd-link.h

@@ -52,6 +52,9 @@ typedef struct Link {
         int ifindex;
         int master_ifindex;
         int dsa_master_ifindex;
+        int sr_iov_phys_port_ifindex;
+        Set *sr_iov_virt_port_ifindices;
+
         char *ifname;
         char **alternative_names;
         char *kind;

src/network/networkd-sriov.c

@@ -1,6 +1,9 @@
 /* SPDX-License-Identifier: LGPL-2.1-or-later
  * Copyright © 2020 VMware, Inc. */
 
+#include "device-enumerator-private.h"
+#include "dirent-util.h"
+#include "fd-util.h"
 #include "networkd-link.h"
 #include "networkd-manager.h"
 #include "networkd-queue.h"
@@ -103,3 +106,216 @@ int link_request_sr_iov_vfs(Link *link) {
 
         return 0;
 }
+
+static int find_ifindex_from_pci_dev_port(sd_device *pci_dev, const char *dev_port) {
+        _cleanup_(sd_device_enumerator_unrefp) sd_device_enumerator *e = NULL;
+        sd_device *dev;
+        int ifindex, r;
+
+        assert(pci_dev);
+        assert(dev_port);
+
+        r = sd_device_enumerator_new(&e);
+        if (r < 0)
+                return r;
+
+        r = sd_device_enumerator_allow_uninitialized(e);
+        if (r < 0)
+                return r;
+
+        r = sd_device_enumerator_add_match_parent(e, pci_dev);
+        if (r < 0)
+                return r;
+
+        r = sd_device_enumerator_add_match_subsystem(e, "net", true);
+        if (r < 0)
+                return r;
+
+        r = sd_device_enumerator_add_match_sysattr(e, "dev_port", dev_port, true);
+        if (r < 0)
+                return r;
+
+        dev = sd_device_enumerator_get_device_first(e);
+        if (!dev)
+                return -ENODEV; /* no device found */
+
+        if (sd_device_enumerator_get_device_next(e))
+                return -ENXIO; /* multiple devices found */
+
+        r = sd_device_get_ifindex(dev, &ifindex);
+        if (r < 0)
+                return r;
+
+        assert(ifindex > 0);
+        return ifindex;
+}
+
+static int manager_update_sr_iov_ifindices(Manager *manager, int phys_port_ifindex, int virt_port_ifindex) {
+        Link *phys_link = NULL, *virt_link = NULL;
+        int r;
+
+        assert(manager);
+        assert(phys_port_ifindex > 0);
+        assert(virt_port_ifindex > 0);
+
+        /* This sets ifindices only whenn both interfaces are already managed by us. */
+
+        r = link_get_by_index(manager, phys_port_ifindex, &phys_link);
+        if (r < 0)
+                return r;
+
+        r = link_get_by_index(manager, virt_port_ifindex, &virt_link);
+        if (r < 0)
+                return r;
+
+        /* update VF ifindex in PF */
+        r = set_ensure_put(&phys_link->sr_iov_virt_port_ifindices, NULL, INT_TO_PTR(virt_port_ifindex));
+        if (r < 0)
+                return r;
+
+        log_link_debug(phys_link,
+                       "Found SR-IOV VF port %s(%i).",
+                       virt_link ? virt_link->ifname : "n/a", virt_port_ifindex);
+
+        /* update PF ifindex in VF */
+        if (virt_link->sr_iov_phys_port_ifindex > 0 && virt_link->sr_iov_phys_port_ifindex != phys_port_ifindex) {
+                Link *old_phys_link;
+
+                if (link_get_by_index(manager, virt_link->sr_iov_phys_port_ifindex, &old_phys_link) >= 0)
+                        set_remove(old_phys_link->sr_iov_virt_port_ifindices, INT_TO_PTR(virt_port_ifindex));
+        }
+
+        virt_link->sr_iov_phys_port_ifindex = phys_port_ifindex;
+
+        log_link_debug(virt_link,
+                       "Found SR-IOV PF port %s(%i).",
+                       phys_link ? phys_link->ifname : "n/a", phys_port_ifindex);
+
+        return 0;
+}
+
+static int link_set_sr_iov_phys_port(Link *link) {
+        _cleanup_(sd_device_unrefp) sd_device *pci_physfn_dev = NULL;
+        const char *dev_port;
+        sd_device *pci_dev;
+        int r;
+
+        assert(link);
+        assert(link->manager);
+
+        if (link->sr_iov_phys_port_ifindex > 0)
+                return 0;
+
+        if (!link->dev)
+                return -ENODEV;
+
+        r = sd_device_get_sysattr_value(link->dev, "dev_port", &dev_port);
+        if (r < 0)
+                return r;
+
+        r = sd_device_get_parent_with_subsystem_devtype(link->dev, "pci", NULL, &pci_dev);
+        if (r < 0)
+                return r;
+
+        r = sd_device_new_child(&pci_physfn_dev, pci_dev, "physfn");
+        if (r < 0)
+                return r;
+
+        r = find_ifindex_from_pci_dev_port(pci_physfn_dev, dev_port);
+        if (r < 0)
+                return r;
+
+        return manager_update_sr_iov_ifindices(link->manager, r, link->ifindex);
+}
+
+static int link_set_sr_iov_virt_ports(Link *link) {
+        const char *dev_port, *pci_syspath;
+        _cleanup_closedir_ DIR *dir = NULL;
+        sd_device *pci_dev;
+        int r;
+
+        assert(link);
+        assert(link->manager);
+
+        set_clear(link->sr_iov_virt_port_ifindices);
+
+        if (!link->dev)
+                return -ENODEV;
+
+        r = sd_device_get_sysattr_value(link->dev, "dev_port", &dev_port);
+        if (r < 0)
+                return r;
+
+        r = sd_device_get_parent_with_subsystem_devtype(link->dev, "pci", NULL, &pci_dev);
+        if (r < 0)
+                return r;
+
+        r = sd_device_get_syspath(pci_dev, &pci_syspath);
+        if (r < 0)
+                return r;
+
+        dir = opendir(pci_syspath);
+        if (!dir)
+                return -errno;
+
+        FOREACH_DIRENT_ALL(de, dir, break) {
+                _cleanup_(sd_device_unrefp) sd_device *pci_virtfn_dev = NULL;
+
+                if (de->d_type != DT_LNK)
+                        continue;
+
+                /* Accept name prefixed with "virtfn", but refuse "virtfn" itself. */
+                if (isempty(startswith(de->d_name, "virtfn")))
+                        continue;
+
+                if (sd_device_new_child(&pci_virtfn_dev, pci_dev, de->d_name) < 0)
+                        continue;
+
+                if (find_ifindex_from_pci_dev_port(pci_virtfn_dev, dev_port) < 0)
+                        continue;
+
+                if (manager_update_sr_iov_ifindices(link->manager, link->ifindex, r) < 0)
+                        continue;
+        }
+
+        return 0;
+}
+
+int link_set_sr_iov_ifindices(Link *link) {
+        int r;
+
+        assert(link);
+
+        r = link_set_sr_iov_phys_port(link);
+        if (r < 0 && !ERRNO_IS_DEVICE_ABSENT(r))
+                return r;
+
+        r = link_set_sr_iov_virt_ports(link);
+        if (r < 0 && !ERRNO_IS_DEVICE_ABSENT(r))
+                return r;
+
+        return 0;
+}
+
+void link_clear_sr_iov_ifindices(Link *link) {
+        void *v;
+
+        assert(link);
+        assert(link->manager);
+
+        if (link->sr_iov_phys_port_ifindex > 0) {
+                Link *phys_link;
+
+                if (link_get_by_index(link->manager, link->sr_iov_phys_port_ifindex, &phys_link) >= 0)
+                        set_remove(phys_link->sr_iov_virt_port_ifindices, INT_TO_PTR(link->ifindex));
+
+                link->sr_iov_phys_port_ifindex = 0;
+        }
+
+        while ((v = set_steal_first(link->sr_iov_virt_port_ifindices))) {
+                Link *virt_link;
+
+                if (link_get_by_index(link->manager, PTR_TO_INT(v), &virt_link) >= 0)
+                        virt_link->sr_iov_phys_port_ifindex = 0;
+        }
+}

src/network/networkd-sriov.h

@@ -7,3 +7,6 @@
 typedef struct Link Link;
 
 int link_request_sr_iov_vfs(Link *link);
+
+int link_set_sr_iov_ifindices(Link *link);
+void link_clear_sr_iov_ifindices(Link *link);