network: use request queue to configure addresses, routes, and nexthops

Why is this necessary? Several examples below.

- When a route sets prefsrc, then the address must be already assigned
  (see issue #19285), and also it must be ready if IPv6.
- When a route or nexthop sets gateway, then the address must be reachable.
- When a route sets nexthop ID, then the corresponding nexthop must be
  assigned.
- When a route sets multipath routes on another interface, then the
  interface must exist and be ready to configure.
- When configuring address, the same address must not be under removing
  (see issue #18108).
Etc,. etc,...

So, this makes all requests about addresses, routes, and nethops are once
stored in the queue, and will be processed when they are ready to configure.

Fixes #18108 and #19285.

src/network/networkd-address.c

@@ -9,9 +9,9 @@
 #include "netlink-util.h"
 #include "networkd-address-pool.h"
 #include "networkd-address.h"
-#include "networkd-ipv6-proxy-ndp.h"
 #include "networkd-manager.h"
 #include "networkd-network.h"
+#include "networkd-queue.h"
 #include "parse-util.h"
 #include "string-util.h"
 #include "strv.h"
@@ -614,8 +614,11 @@ int address_remove_handler_internal(sd_netlink *rtnl, sd_netlink_message *m, Lin
         assert(rtnl);
         assert(m);
         assert(link);
+        assert(link->address_remove_messages > 0);
         assert(error_msg);
 
+        link->address_remove_messages--;
+
         if (IN_SET(link->state, LINK_STATE_FAILED, LINK_STATE_LINGER))
                 return 0;
 
@@ -665,6 +668,7 @@ int address_remove(
                 return log_link_error_errno(link, r, "Could not send rtnetlink message: %m");
 
         link_ref(link);
+        link->address_remove_messages++;
 
         return 0;
 }
@@ -794,28 +798,6 @@ int link_drop_foreign_addresses(Link *link) {
         return r;
 }
 
-static int remove_static_address_handler(sd_netlink *rtnl, sd_netlink_message *m, Link *link) {
-        int r;
-
-        assert(link);
-        assert(link->address_remove_messages > 0);
-
-        link->address_remove_messages--;
-
-        r = address_remove_handler_internal(rtnl, m, link, "Could not drop address, ignoring");
-        if (r <= 0)
-                return r;
-
-        if (link->address_remove_messages == 0 && link->request_static_addresses) {
-                link_set_state(link, LINK_STATE_CONFIGURING);
-                r = link_set_addresses(link);
-                if (r < 0)
-                        link_enter_failed(link);
-        }
-
-        return 1;
-}
-
 int link_drop_addresses(Link *link) {
         Address *address, *pool_address;
         int k, r = 0;
@@ -827,14 +809,12 @@ int link_drop_addresses(Link *link) {
                 if (address->family == AF_INET6 && in6_addr_is_link_local(&address->in_addr.in6) == 1 && link_ipv6ll_enabled(link))
                         continue;
 
-                k = address_remove(address, link, remove_static_address_handler);
+                k = address_remove(address, link, NULL);
                 if (k < 0 && r >= 0) {
                         r = k;
                         continue;
                 }
 
-                link->address_remove_messages++;
-
                 SET_FOREACH(pool_address, link->pool_addresses)
                         if (address_equal(address, pool_address))
                                 address_free(set_remove(link->pool_addresses, pool_address));
@@ -924,7 +904,7 @@ int address_configure_handler_internal(sd_netlink *rtnl, sd_netlink_message *m,
 
 static int ipv4_dad_configure(Address *address);
 
-int address_configure(
+static int address_configure(
                 const Address *address,
                 Link *link,
                 link_netlink_message_handler_t callback,
@@ -1028,68 +1008,24 @@ int address_configure(
         return k;
 }
 
-static int static_address_ready_callback(Address *address) {
-        Address *a;
-        Link *link;
-        int r;
-
-        assert(address);
-        assert(address->link);
-
-        link = address->link;
-
-        if (!link->addresses_configured)
-                return 0;
-
-        SET_FOREACH(a, link->static_addresses)
-                if (!address_is_ready(a)) {
-                        _cleanup_free_ char *str = NULL;
-
-                        (void) in_addr_prefix_to_string(a->family, &a->in_addr, a->prefixlen, &str);
-                        log_link_debug(link, "an address %s is not ready", strnull(str));
-                        return 0;
-                }
-
-        /* This should not be called again */
-        SET_FOREACH(a, link->static_addresses)
-                a->callback = NULL;
-
-        link->addresses_ready = true;
-
-        r = link_set_ipv6_proxy_ndp_addresses(link);
-        if (r < 0)
-                return r;
-
-        return link_set_routes(link);
-}
-
-static int address_handler(sd_netlink *rtnl, sd_netlink_message *m, Link *link) {
+static int static_address_handler(sd_netlink *rtnl, sd_netlink_message *m, Link *link) {
         int r;
 
         assert(link);
-        assert(link->address_messages > 0);
+        assert(link->static_address_messages > 0);
 
-        link->address_messages--;
+        link->static_address_messages--;
 
         r = address_configure_handler_internal(rtnl, m, link, "Failed to set static address");
         if (r <= 0)
                 return r;
 
-        if (link->address_messages == 0) {
-                Address *a;
-
+        if (link->static_address_messages == 0) {
                 log_link_debug(link, "Addresses set");
-                link->addresses_configured = true;
+                link->static_addresses_configured = true;
+                link_check_ready(link);
 
-                /* When all static addresses are already ready, then static_address_ready_callback()
-                 * will not be called automatically. So, call it here. */
-                a = set_first(link->static_addresses);
-                if (!a) {
-                        log_link_debug(link, "No static address is stored. Already removed?");
-                        return 1;
-                }
-
-                r = static_address_ready_callback(a);
+                r = dhcp4_server_configure(link);
                 if (r < 0)
                         link_enter_failed(link);
         }
@@ -1097,55 +1033,65 @@ static int address_handler(sd_netlink *rtnl, sd_netlink_message *m, Link *link)
         return 1;
 }
 
-static int static_address_configure(const Address *address, Link *link) {
-        Address *ret;
+static int static_address_after_configure(Request *req, void *object) {
+        Address *address = object;
+        Link *link;
         int r;
 
+        assert(req);
+        assert(req->link);
+        assert(req->type == REQUEST_TYPE_ADDRESS);
         assert(address);
-        assert(link);
 
-        r = address_configure(address, link, address_handler, &ret);
-        if (r < 0)
-                return log_link_warning_errno(link, r, "Could not configure static address: %m");
+        link = req->link;
 
-        link->address_messages++;
-
-        r = set_ensure_put(&link->static_addresses, &address_hash_ops, ret);
+        r = set_ensure_put(&link->static_addresses, &address_hash_ops, address);
         if (r < 0)
                 return log_link_warning_errno(link, r, "Failed to store static address: %m");
 
-        ret->callback = static_address_ready_callback;
-
         return 0;
 }
 
-int link_set_addresses(Link *link) {
-        Address *ad;
+int link_request_address(
+                Link *link,
+                Address *address,
+                bool consume_object,
+                unsigned *message_counter,
+                link_netlink_message_handler_t netlink_handler,
+                Request **ret) {
+
+        assert(link);
+        assert(address);
+
+        log_address_debug(address, "Requesting", link);
+        return link_queue_request(link, REQUEST_TYPE_ADDRESS, address, consume_object,
+                                  message_counter, netlink_handler, ret);
+}
+
+int link_request_static_addresses(Link *link) {
+        Address *a;
         Prefix *p;
         int r;
 
         assert(link);
         assert(link->network);
 
-        if (link->address_remove_messages != 0) {
-                log_link_debug(link, "Removing old addresses, new addresses will be configured later.");
-                link->request_static_addresses = true;
-                return 0;
-        }
+        link->static_addresses_configured = false;
 
-        if (link->address_messages != 0) {
-                log_link_debug(link, "Static addresses are configuring.");
-                return 0;
-        }
+        ORDERED_HASHMAP_FOREACH(a, link->network->addresses_by_section) {
+                Request *req;
 
-        ORDERED_HASHMAP_FOREACH(ad, link->network->addresses_by_section) {
-                r = static_address_configure(ad, link);
+                r = link_request_address(link, a, false, &link->static_address_messages,
+                                         static_address_handler, &req);
                 if (r < 0)
                         return r;
+
+                req->after_configure = static_address_after_configure;
         }
 
         HASHMAP_FOREACH(p, link->network->prefixes_by_section) {
                 _cleanup_(address_freep) Address *address = NULL;
+                Request *req;
 
                 if (!p->assign)
                         continue;
@@ -1164,22 +1110,18 @@ int link_set_addresses(Link *link) {
 
                 address->family = AF_INET6;
                 address->route_metric = p->route_metric;
-                r = static_address_configure(address, link);
+
+                r = link_request_address(link, TAKE_PTR(address), true, &link->static_address_messages,
+                                         static_address_handler, &req);
                 if (r < 0)
                         return r;
+
+                req->after_configure = static_address_after_configure;
         }
 
-        if (link->address_messages == 0) {
-                link->addresses_configured = true;
-                link->addresses_ready = true;
-
-                r = link_set_ipv6_proxy_ndp_addresses(link);
-                if (r < 0)
-                        return r;
-
-                r = link_set_routes(link);
-                if (r < 0)
-                        return r;
+        if (link->static_address_messages == 0) {
+                link->static_addresses_configured = true;
+                link_check_ready(link);
         } else {
                 log_link_debug(link, "Setting addresses");
                 link_set_state(link, LINK_STATE_CONFIGURING);
@@ -1188,6 +1130,34 @@ int link_set_addresses(Link *link) {
         return 0;
 }
 
+int request_process_address(Request *req) {
+        Address *ret = NULL;  /* avoid false maybe-uninitialized warning */
+        int r;
+
+        assert(req);
+        assert(req->link);
+        assert(req->address);
+        assert(req->type == REQUEST_TYPE_ADDRESS);
+
+        if (!link_is_ready_to_configure(req->link, false))
+                return 0;
+
+        if (req->link->address_remove_messages > 0)
+                return 0;
+
+        r = address_configure(req->address, req->link, req->netlink_handler, &ret);
+        if (r < 0)
+                return r;
+
+        if (req->after_configure) {
+                r = req->after_configure(req, ret);
+                if (r < 0)
+                        return r;
+        }
+
+        return 1;
+}
+
 int manager_rtnl_process_address(sd_netlink *rtnl, sd_netlink_message *message, Manager *m) {
         _cleanup_(address_freep) Address *tmp = NULL;
         Link *link = NULL;

src/network/networkd-address.h

@@ -16,6 +16,7 @@
 
 typedef struct Manager Manager;
 typedef struct Network Network;
+typedef struct Request Request;
 typedef int (*address_ready_callback_t)(Address *address);
 
 typedef struct Address {
@@ -52,7 +53,6 @@ int address_new(Address **ret);
 Address *address_free(Address *address);
 int address_get(Link *link, const Address *in, Address **ret);
 int address_configure_handler_internal(sd_netlink *rtnl, sd_netlink_message *m, Link *link, const char *error_msg);
-int address_configure(const Address *address, Link *link, link_netlink_message_handler_t callback, Address **ret);
 int address_remove_handler_internal(sd_netlink *rtnl, sd_netlink_message *m, Link *link, const char *error_msg);
 int address_remove(const Address *address, Link *link, link_netlink_message_handler_t callback);
 bool address_equal(const Address *a1, const Address *a2);
@@ -62,7 +62,6 @@ int generate_ipv6_eui_64_address(const Link *link, struct in6_addr *ret);
 
 DEFINE_NETWORK_SECTION_FUNCTIONS(Address, address_free);
 
-int link_set_addresses(Link *link);
 int link_drop_addresses(Link *link);
 int link_drop_foreign_addresses(Link *link);
 bool link_address_is_dynamic(const Link *link, const Address *address);
@@ -73,6 +72,16 @@ void ipv4_dad_unref(Link *link);
 int ipv4_dad_stop(Link *link);
 int ipv4_dad_update_mac(Link *link);
 
+int link_request_address(
+                Link *link,
+                Address *address,
+                bool consume_object,
+                unsigned *message_counter,
+                link_netlink_message_handler_t netlink_handler,
+                Request **ret);
+int link_request_static_addresses(Link *link);
+int request_process_address(Request *req);
+
 int manager_rtnl_process_address(sd_netlink *nl, sd_netlink_message *message, Manager *m);
 
 void network_drop_invalid_addresses(Network *network);

src/network/networkd-dhcp4.c

@@ -17,13 +17,16 @@
 #include "networkd-link.h"
 #include "networkd-manager.h"
 #include "networkd-network.h"
+#include "networkd-nexthop.h"
+#include "networkd-queue.h"
+#include "networkd-route.h"
 #include "networkd-state-file.h"
 #include "string-table.h"
 #include "strv.h"
 #include "sysctl-util.h"
 #include "web-util.h"
 
-static int dhcp4_update_address(Link *link, bool announce);
+static int dhcp4_request_address_and_routes(Link *link, bool announce);
 static int dhcp4_remove_all(Link *link);
 
 void network_adjust_dhcp4(Network *network) {
@@ -83,8 +86,20 @@ static void dhcp4_check_ready(Link *link) {
         if (link->network->dhcp_send_decline && !link->dhcp4_address_bind)
                 return;
 
-        if (link->dhcp4_messages > 0)
+        if (link->dhcp4_messages > 0) {
+                log_link_debug(link, "%s(): DHCPv4 address and routes are not set.", __func__);
                 return;
+        }
+
+        if (!link->dhcp_address) {
+                log_link_debug(link, "%s(): DHCPv4 address is not set.", __func__);
+                return;
+        }
+
+        if (!address_is_ready(link->dhcp_address)) {
+                log_link_debug(link, "%s(): DHCPv4 address is not ready.", __func__);
+                return;
+        }
 
         link->dhcp4_configured = true;
 
@@ -102,6 +117,27 @@ static void dhcp4_check_ready(Link *link) {
         link_check_ready(link);
 }
 
+static int dhcp4_after_route_configure(Request *req, void *object) {
+        Route *route = object;
+        Link *link;
+        int r;
+
+        assert(req);
+        assert(req->link);
+        assert(req->type == REQUEST_TYPE_ROUTE);
+        assert(route);
+
+        link = req->link;
+
+        r = set_ensure_put(&link->dhcp_routes, &route_hash_ops, route);
+        if (r < 0)
+                return log_link_error_errno(link, r, "Failed to store DHCPv4 route: %m");
+
+        set_remove(link->dhcp_routes_old, route);
+
+        return 0;
+}
+
 static int dhcp4_route_handler(sd_netlink *rtnl, sd_netlink_message *m, Link *link) {
         int r;
 
@@ -135,6 +171,15 @@ static int dhcp4_route_handler(sd_netlink *rtnl, sd_netlink_message *m, Link *li
                 r = dhcp4_remove_all(link);
                 if (r < 0)
                         link_enter_failed(link);
+
+                r = link_request_static_nexthops(link, true);
+                if (r < 0)
+                        link_enter_failed(link);
+
+                r = link_request_static_routes(link, true);
+                if (r < 0)
+                        link_enter_failed(link);
+
                 return 1;
         }
 
@@ -143,24 +188,26 @@ static int dhcp4_route_handler(sd_netlink *rtnl, sd_netlink_message *m, Link *li
         return 1;
 }
 
-static int dhcp_route_configure(Route *route, Link *link) {
-        Route *ret;
+static int dhcp4_request_route(Route *in, Link *link) {
+        _cleanup_(route_freep) Route *route = in;
+        Request *req;
         int r;
 
         assert(route);
         assert(link);
 
-        r = route_configure(route, link, dhcp4_route_handler, &ret);
+        r = link_has_route(link, route);
         if (r < 0)
-                return log_link_error_errno(link, r, "Failed to set DHCPv4 route: %m");
+                return r;
+        if (r == 0)
+                link->dhcp4_configured = false;
 
-        link->dhcp4_messages++;
-
-        r = set_ensure_put(&link->dhcp_routes, &route_hash_ops, ret);
+        r = link_request_route(link, TAKE_PTR(route), true, &link->dhcp4_messages,
+                               dhcp4_route_handler, &req);
         if (r < 0)
-                return log_link_error_errno(link, r, "Failed to store DHCPv4 route: %m");
+                return r;
 
-        (void) set_remove(link->dhcp_routes_old, ret);
+        req->after_configure = dhcp4_after_route_configure;
 
         return 0;
 }
@@ -171,7 +218,7 @@ static bool link_prefixroute(Link *link) {
                 link->manager->dhcp4_prefix_root_cannot_set_table;
 }
 
-static int link_set_dhcp_prefix_route(Link *link) {
+static int dhcp4_request_prefix_route(Link *link) {
         _cleanup_(route_freep) Route *route = NULL;
         struct in_addr address, netmask;
         int r;
@@ -204,10 +251,10 @@ static int link_set_dhcp_prefix_route(Link *link) {
         route->table = link_get_dhcp_route_table(link);
         route->mtu = link->network->dhcp_route_mtu;
 
-        return dhcp_route_configure(route, link);
+        return dhcp4_request_route(TAKE_PTR(route), link);
 }
 
-static int link_set_dhcp_route_to_gateway(Link *link, const struct in_addr *gw) {
+static int dhcp4_request_route_to_gateway(Link *link, const struct in_addr *gw) {
         _cleanup_(route_freep) Route *route = NULL;
         struct in_addr address;
         int r;
@@ -234,14 +281,15 @@ static int link_set_dhcp_route_to_gateway(Link *link, const struct in_addr *gw)
         route->table = link_get_dhcp_route_table(link);
         route->mtu = link->network->dhcp_route_mtu;
 
-        return dhcp_route_configure(route, link);
+        return dhcp4_request_route(TAKE_PTR(route), link);
 }
 
-static int dhcp_route_configure_auto(
-                Route *route,
+static int dhcp4_request_route_auto(
+                Route *in,
                 Link *link,
                 const struct in_addr *gw) {
 
+        _cleanup_(route_freep) Route *route = in;
         struct in_addr address, netmask, prefix;
         uint8_t prefixlen;
         int r;
@@ -251,8 +299,6 @@ static int dhcp_route_configure_auto(
         assert(link->dhcp_lease);
         assert(gw);
 
-        /* The route object may be reused in an iteration. All elements must be set or cleared. */
-
         r = sd_dhcp_lease_get_address(link->dhcp_lease, &address);
         if (r < 0)
                 return r;
@@ -309,7 +355,7 @@ static int dhcp_route_configure_auto(
                         return 0;
                 }
 
-                r = link_set_dhcp_route_to_gateway(link, gw);
+                r = dhcp4_request_route_to_gateway(link, gw);
                 if (r < 0)
                         return r;
 
@@ -319,13 +365,12 @@ static int dhcp_route_configure_auto(
                 route->prefsrc.in = address;
         }
 
-        return dhcp_route_configure(route, link);
+        return dhcp4_request_route(TAKE_PTR(route), link);
 }
 
-static int link_set_dhcp_static_routes(Link *link, struct in_addr *ret_default_gw) {
+static int dhcp4_request_static_routes(Link *link, struct in_addr *ret_default_gw) {
         _cleanup_free_ sd_dhcp_route **static_routes = NULL;
         bool classless_route = false, static_route = false;
-        _cleanup_(route_freep) Route *route = NULL;
         struct in_addr default_gw = {};
         int n, r;
 
@@ -360,24 +405,25 @@ static int link_set_dhcp_static_routes(Link *link, struct in_addr *ret_default_g
         if (classless_route && static_route)
                 log_link_debug(link, "Classless static routes received from DHCP server: ignoring static-route option");
 
-        r = route_new(&route);
-        if (r < 0)
-                return r;
-
-        route->family = AF_INET;
-        route->gw_family = AF_INET;
-        route->protocol = RTPROT_DHCP;
-        route->priority = link->network->dhcp_route_metric;
-        route->table = link_get_dhcp_route_table(link);
-        route->mtu = link->network->dhcp_route_mtu;
-
         for (int i = 0; i < n; i++) {
+                _cleanup_(route_freep) Route *route = NULL;
                 struct in_addr gw;
 
                 if (sd_dhcp_route_get_option(static_routes[i]) !=
                     (classless_route ? SD_DHCP_OPTION_CLASSLESS_STATIC_ROUTE : SD_DHCP_OPTION_STATIC_ROUTE))
                         continue;
 
+                r = route_new(&route);
+                if (r < 0)
+                        return r;
+
+                route->family = AF_INET;
+                route->gw_family = AF_INET;
+                route->protocol = RTPROT_DHCP;
+                route->priority = link->network->dhcp_route_metric;
+                route->table = link_get_dhcp_route_table(link);
+                route->mtu = link->network->dhcp_route_mtu;
+
                 r = sd_dhcp_route_get_gateway(static_routes[i], &gw);
                 if (r < 0)
                         return r;
@@ -398,7 +444,7 @@ static int link_set_dhcp_static_routes(Link *link, struct in_addr *ret_default_g
                     in4_addr_is_null(&default_gw))
                         default_gw = gw;
 
-                r = dhcp_route_configure_auto(route, link, &gw);
+                r = dhcp4_request_route_auto(TAKE_PTR(route), link, &gw);
                 if (r < 0)
                         return r;
         }
@@ -407,7 +453,7 @@ static int link_set_dhcp_static_routes(Link *link, struct in_addr *ret_default_g
         return classless_route;
 }
 
-static int link_set_dhcp_gateway(Link *link, struct in_addr *ret_gw) {
+static int dhcp4_request_gateway(Link *link, struct in_addr *ret_gw) {
         _cleanup_(route_freep) Route *route = NULL;
         const struct in_addr *router;
         struct in_addr address;
@@ -439,7 +485,7 @@ static int link_set_dhcp_gateway(Link *link, struct in_addr *ret_gw) {
 
         /* The dhcp netmask may mask out the gateway. First, add an explicit route for the gateway host
          * so that we can route no matter the netmask or existing kernel route tables. */
-        r = link_set_dhcp_route_to_gateway(link, &router[0]);
+        r = dhcp4_request_route_to_gateway(link, &router[0]);
         if (r < 0)
                 return r;
 
@@ -457,7 +503,7 @@ static int link_set_dhcp_gateway(Link *link, struct in_addr *ret_gw) {
         route->table = link_get_dhcp_route_table(link);
         route->mtu = link->network->dhcp_route_mtu;
 
-        r = dhcp_route_configure(route, link);
+        r = dhcp4_request_route(TAKE_PTR(route), link);
         if (r < 0)
                 return r;
 
@@ -468,17 +514,21 @@ static int link_set_dhcp_gateway(Link *link, struct in_addr *ret_gw) {
                 if (rt->gw_family != AF_INET)
                         continue;
 
-                rt->gw.in = router[0];
-                if (!rt->protocol_set)
-                        rt->protocol = RTPROT_DHCP;
-                if (!rt->priority_set)
-                        rt->priority = link->network->dhcp_route_metric;
-                if (!rt->table_set)
-                        rt->table = link_get_dhcp_route_table(link);
-                if (rt->mtu == 0)
-                        rt->mtu = link->network->dhcp_route_mtu;
+                r = route_dup(rt, &route);
+                if (r < 0)
+                        return r;
 
-                r = dhcp_route_configure(rt, link);
+                route->gw.in = router[0];
+                if (!route->protocol_set)
+                        route->protocol = RTPROT_DHCP;
+                if (!route->priority_set)
+                        route->priority = link->network->dhcp_route_metric;
+                if (!route->table_set)
+                        route->table = link_get_dhcp_route_table(link);
+                if (route->mtu == 0)
+                        route->mtu = link->network->dhcp_route_mtu;
+
+                r = dhcp4_request_route(TAKE_PTR(route), link);
                 if (r < 0)
                         return r;
         }
@@ -487,13 +537,12 @@ static int link_set_dhcp_gateway(Link *link, struct in_addr *ret_gw) {
         return 0;
 }
 
-static int link_set_routes_to_servers(
+static int dhcp4_request_routes_to_servers(
                 Link *link,
                 const struct in_addr *servers,
                 size_t n_servers,
                 const struct in_addr *gw) {
 
-        _cleanup_(route_freep) Route *route = NULL;
         int r;
 
         assert(link);
@@ -502,24 +551,25 @@ static int link_set_routes_to_servers(
         assert(servers || n_servers == 0);
         assert(gw);
 
-        r = route_new(&route);
-        if (r < 0)
-                return r;
-
-        route->family = AF_INET;
-        route->dst_prefixlen = 32;
-        route->protocol = RTPROT_DHCP;
-        route->priority = link->network->dhcp_route_metric;
-        route->table = link_get_dhcp_route_table(link);
-        route->mtu = link->network->dhcp_route_mtu;
-
         for (size_t i = 0; i < n_servers; i++) {
+                _cleanup_(route_freep) Route *route = NULL;
+
                 if (in4_addr_is_null(&servers[i]))
                         continue;
 
-                route->dst.in = servers[i];
+                r = route_new(&route);
+                if (r < 0)
+                        return r;
 
-                r = dhcp_route_configure_auto(route, link, gw);
+                route->family = AF_INET;
+                route->dst.in = servers[i];
+                route->dst_prefixlen = 32;
+                route->protocol = RTPROT_DHCP;
+                route->priority = link->network->dhcp_route_metric;
+                route->table = link_get_dhcp_route_table(link);
+                route->mtu = link->network->dhcp_route_mtu;
+
+                r = dhcp4_request_route_auto(TAKE_PTR(route), link, gw);
                 if (r < 0)
                         return r;
         }
@@ -527,7 +577,7 @@ static int link_set_routes_to_servers(
         return 0;
 }
 
-static int link_set_routes_to_dns(Link *link, const struct in_addr *gw) {
+static int dhcp4_request_routes_to_dns(Link *link, const struct in_addr *gw) {
         const struct in_addr *dns;
         int r;
 
@@ -546,10 +596,10 @@ static int link_set_routes_to_dns(Link *link, const struct in_addr *gw) {
         if (r < 0)
                 return r;
 
-        return link_set_routes_to_servers(link, dns, r, gw);
+        return dhcp4_request_routes_to_servers(link, dns, r, gw);
 }
 
-static int link_set_routes_to_ntp(Link *link, const struct in_addr *gw) {
+static int dhcp4_request_routes_to_ntp(Link *link, const struct in_addr *gw) {
         const struct in_addr *ntp;
         int r;
 
@@ -568,25 +618,17 @@ static int link_set_routes_to_ntp(Link *link, const struct in_addr *gw) {
         if (r < 0)
                 return r;
 
-        return link_set_routes_to_servers(link, ntp, r, gw);
+        return dhcp4_request_routes_to_servers(link, ntp, r, gw);
 }
 
-static int link_set_dhcp_routes(Link *link) {
+static int dhcp4_request_routes(Link *link) {
         struct in_addr gw = {};
         Route *rt;
         int r;
 
         assert(link);
 
-        if (!link->dhcp_lease) /* link went down while we configured the IP addresses? */
-                return 0;
-
-        if (!link->network) /* link went down while we configured the IP addresses? */
-                return 0;
-
-        if (!link_has_carrier(link) && !link->network->configure_without_carrier)
-                /* During configuring addresses, the link lost its carrier. As networkd is dropping
-                 * the addresses now, let's not configure the routes either. */
+        if (!link->dhcp_lease)
                 return 0;
 
         while ((rt = set_steal_first(link->dhcp_routes))) {
@@ -595,28 +637,28 @@ static int link_set_dhcp_routes(Link *link) {
                         return log_link_error_errno(link, r, "Failed to store old DHCPv4 route: %m");
         }
 
-        r = link_set_dhcp_prefix_route(link);
+        r = dhcp4_request_prefix_route(link);
         if (r < 0)
-                return log_link_error_errno(link, r, "DHCP error: Could not set prefix route: %m");
+                return log_link_error_errno(link, r, "DHCP error: Could not request prefix route: %m");
 
-        r = link_set_dhcp_static_routes(link, &gw);
+        r = dhcp4_request_static_routes(link, &gw);
         if (r < 0)
-                return log_link_error_errno(link, r, "DHCP error: Could not set static routes: %m");
+                return log_link_error_errno(link, r, "DHCP error: Could not request static routes: %m");
         if (r == 0) {
                 /* According to RFC 3442: If the DHCP server returns both a Classless Static Routes option and
                  * a Router option, the DHCP client MUST ignore the Router option. */
-                r = link_set_dhcp_gateway(link, &gw);
+                r = dhcp4_request_gateway(link, &gw);
                 if (r < 0)
-                        return log_link_error_errno(link, r, "DHCP error: Could not set gateway: %m");
+                        return log_link_error_errno(link, r, "DHCP error: Could not request gateway: %m");
         }
 
-        r = link_set_routes_to_dns(link, &gw);
+        r = dhcp4_request_routes_to_dns(link, &gw);
         if (r < 0)
-                return log_link_error_errno(link, r, "DHCP error: Could not set routes to DNS servers: %m");
+                return log_link_error_errno(link, r, "DHCP error: Could not request routes to DNS servers: %m");
 
-        r = link_set_routes_to_ntp(link, &gw);
+        r = dhcp4_request_routes_to_ntp(link, &gw);
         if (r < 0)
-                return log_link_error_errno(link, r, "DHCP error: Could not set routes to NTP servers: %m");
+                return log_link_error_errno(link, r, "DHCP error: Could not request routes to NTP servers: %m");
 
         return 0;
 }
@@ -683,7 +725,7 @@ static int dhcp4_route_remove_handler(sd_netlink *rtnl, sd_netlink_message *m, L
                 return r;
 
         if (link->dhcp4_remove_messages == 0) {
-                r = dhcp4_update_address(link, false);
+                r = dhcp4_request_address_and_routes(link, false);
                 if (r < 0)
                         link_enter_failed(link);
         }
@@ -704,7 +746,7 @@ static int dhcp4_address_remove_handler(sd_netlink *rtnl, sd_netlink_message *m,
                 return r;
 
         if (link->dhcp4_remove_messages == 0) {
-                r = dhcp4_update_address(link, false);
+                r = dhcp4_request_address_and_routes(link, false);
                 if (r < 0)
                         link_enter_failed(link);
         }
@@ -769,7 +811,14 @@ static int dhcp_lease_lost(Link *link) {
 
         (void) sd_ipv4acd_stop(link->dhcp_acd);
 
-        return r;
+        if (r < 0)
+                return r;
+
+        r = link_request_static_nexthops(link, true);
+        if (r < 0)
+                return r;
+
+        return link_request_static_routes(link, true);
 }
 
 static void dhcp_address_on_acd(sd_ipv4acd *acd, int event, void *userdata) {
@@ -916,30 +965,44 @@ static int dhcp4_start_acd(Link *link) {
 }
 
 static int dhcp4_address_ready_callback(Address *address) {
-        Link *link;
-        int r;
-
         assert(address);
 
-        link = address->link;
-
         /* Do not call this again. */
         address->callback = NULL;
 
-        r = link_set_dhcp_routes(link);
-        if (r < 0)
-                return r;
+        dhcp4_check_ready(address->link);
+        return 0;
+}
 
-        /* Reconfigure static routes as kernel may remove some routes when lease expires. */
-        r = link_set_routes(link);
-        if (r < 0)
-                return r;
+static int dhcp4_after_address_configure(Request *req, void *object) {
+        Address *address = object;
+        Link *link;
+        int r;
+
+        assert(req);
+        assert(req->link);
+        assert(req->type == REQUEST_TYPE_ADDRESS);
+        assert(address);
+
+        link = req->link;
+
+        if (!address_equal(link->dhcp_address, address)) {
+                if (link->dhcp_address_old &&
+                    !address_equal(link->dhcp_address_old, link->dhcp_address)) {
+                        /* Still too old address exists? Let's remove it immediately. */
+                        r = address_remove(link->dhcp_address_old, link, NULL);
+                        if (r < 0)
+                                return r;
+                }
+                link->dhcp_address_old = link->dhcp_address;
+        }
+
+        link->dhcp_address = address;
 
         r = dhcp4_start_acd(link);
         if (r < 0)
-                return log_link_error_errno(link, r, "Failed to start IPv4ACD for DHCP4 address: %m");
+                return log_link_error_errno(link, r, "Failed to start IPv4ACD for DHCPv4 address: %m");
 
-        dhcp4_check_ready(link);
         return 0;
 }
 
@@ -947,6 +1010,9 @@ static int dhcp4_address_handler(sd_netlink *rtnl, sd_netlink_message *m, Link *
         int r;
 
         assert(link);
+        assert(link->dhcp4_messages > 0);
+
+        link->dhcp4_messages--;
 
         r = address_configure_handler_internal(rtnl, m, link, "Could not set DHCPv4 address");
         if (r <= 0)
@@ -962,12 +1028,12 @@ static int dhcp4_address_handler(sd_netlink *rtnl, sd_netlink_message *m, Link *
         return 1;
 }
 
-static int dhcp4_update_address(Link *link, bool announce) {
+static int dhcp4_request_address(Link *link, bool announce) {
         _cleanup_(address_freep) Address *addr = NULL;
         uint32_t lifetime = CACHE_INFO_INFINITY_LIFE_TIME;
         struct in_addr address, netmask;
         unsigned prefixlen;
-        Address *ret;
+        Request *req;
         int r;
 
         assert(link);
@@ -976,15 +1042,6 @@ static int dhcp4_update_address(Link *link, bool announce) {
         if (!link->dhcp_lease)
                 return 0;
 
-        link_set_state(link, LINK_STATE_CONFIGURING);
-        link->dhcp4_configured = false;
-
-        /* address_handler calls link_set_routes() and link_set_nexthop(). Before they are called, the
-         * related flags must be cleared. Otherwise, the link becomes configured state before routes
-         * are configured. */
-        link->static_routes_configured = false;
-        link->static_nexthops_configured = false;
-
         r = sd_dhcp_lease_get_address(link->dhcp_lease, &address);
         if (r < 0)
                 return log_link_warning_errno(link, r, "DHCP error: no address: %m");
@@ -1042,15 +1099,34 @@ static int dhcp4_update_address(Link *link, bool announce) {
         SET_FLAG(addr->flags, IFA_F_NOPREFIXROUTE, !link_prefixroute(link));
         addr->route_metric = link->network->dhcp_route_metric;
 
-        /* allow reusing an existing address and simply update its lifetime
-         * in case it already exists */
-        r = address_configure(addr, link, dhcp4_address_handler, &ret);
-        if (r < 0)
-                return log_link_error_errno(link, r, "Failed to set DHCPv4 address: %m");
+        if (address_get(link, addr, NULL) < 0)
+                link->dhcp4_configured = false;
 
-        if (!address_equal(link->dhcp_address, ret))
-                link->dhcp_address_old = link->dhcp_address;
-        link->dhcp_address = ret;
+        r = link_request_address(link, TAKE_PTR(addr), true, &link->dhcp4_messages,
+                                 dhcp4_address_handler, &req);
+        if (r < 0)
+                return log_link_error_errno(link, r, "Failed to request DHCPv4 address: %m");
+
+        req->after_configure = dhcp4_after_address_configure;
+
+        return 0;
+}
+
+static int dhcp4_request_address_and_routes(Link *link, bool announce) {
+        int r;
+
+        assert(link);
+
+        r = dhcp4_request_address(link, announce);
+        if (r < 0)
+                return r;
+
+        r = dhcp4_request_routes(link);
+        if (r < 0)
+                return r;
+
+        link_set_state(link, LINK_STATE_CONFIGURING);
+        link_check_ready(link);
 
         return 0;
 }
@@ -1070,7 +1146,7 @@ static int dhcp_lease_renew(sd_dhcp_client *client, Link *link) {
         link->dhcp_lease = sd_dhcp_lease_ref(lease);
         link_dirty(link);
 
-        return dhcp4_update_address(link, false);
+        return dhcp4_request_address_and_routes(link, false);
 }
 
 static int dhcp_lease_acquired(sd_dhcp_client *client, Link *link) {
@@ -1135,16 +1211,7 @@ static int dhcp_lease_acquired(sd_dhcp_client *client, Link *link) {
                 }
         }
 
-        if (link->dhcp4_remove_messages == 0) {
-                r = dhcp4_update_address(link, true);
-                if (r < 0)
-                        return r;
-        } else
-                log_link_debug(link,
-                               "The link has previously assigned DHCPv4 address or routes. "
-                               "The newly assigned address and routes will set up after old ones are removed.");
-
-        return 0;
+        return dhcp4_request_address_and_routes(link, true);
 }
 
 static int dhcp_lease_ip_change(sd_dhcp_client *client, Link *link) {

src/network/networkd-dhcp6.c

@@ -18,6 +18,7 @@
 #include "networkd-dhcp6.h"
 #include "networkd-link.h"
 #include "networkd-manager.h"
+#include "networkd-queue.h"
 #include "networkd-radv.h"
 #include "siphash24.h"
 #include "string-table.h"
@@ -258,11 +259,32 @@ static int dhcp6_pd_route_handler(sd_netlink *rtnl, sd_netlink_message *m, Link
         return 1;
 }
 
-static int dhcp6_set_pd_route(Link *link, const struct in6_addr *prefix, const struct in6_addr *pd_prefix) {
+static int dhcp6_pd_after_route_configure(Request *req, void *object) {
+        Route *route = object;
+        Link *link;
+        int r;
+
+        assert(req);
+        assert(req->link);
+        assert(req->type == REQUEST_TYPE_ROUTE);
+        assert(route);
+
+        link = req->link;
+
+        r = set_ensure_put(&link->dhcp6_pd_routes, &route_hash_ops, route);
+        if (r < 0)
+                return log_link_error_errno(link, r, "Failed to store DHCPv6 prefix route: %m");
+
+        set_remove(link->dhcp6_pd_routes_old, route);
+
+        return 0;
+}
+
+static int dhcp6_pd_request_route(Link *link, const struct in6_addr *prefix, const struct in6_addr *pd_prefix) {
         _cleanup_(dhcp6_pd_freep) DHCP6DelegatedPrefix *pd = NULL;
         _cleanup_(route_freep) Route *route = NULL;
         Link *assigned_link;
-        Route *ret;
+        Request *req;
         int r;
 
         assert(link);
@@ -280,19 +302,18 @@ static int dhcp6_set_pd_route(Link *link, const struct in6_addr *prefix, const s
         route->protocol = RTPROT_DHCP;
         route->priority = link->network->dhcp6_pd_route_metric;
 
-        r = route_configure(route, link, dhcp6_pd_route_handler, &ret);
+        r = link_has_route(link, route);
         if (r < 0)
-                return log_link_error_errno(link, r, "Failed to set DHCPv6 prefix route: %m");
-        if (r > 0)
+                return r;
+        if (r == 0)
                 link->dhcp6_pd_route_configured = false;
 
-        link->dhcp6_pd_route_messages++;
-
-        r = set_ensure_put(&link->dhcp6_pd_routes, &route_hash_ops, ret);
+        r = link_request_route(link, TAKE_PTR(route), true, &link->dhcp6_pd_route_messages,
+                               dhcp6_pd_route_handler, &req);
         if (r < 0)
-                return log_link_error_errno(link, r, "Failed to store DHCPv6 prefix route: %m");
+                return log_link_error_errno(link, r, "Failed to request DHCPv6 prefix route: %m");
 
-        (void) set_remove(link->dhcp6_pd_routes_old, ret);
+        req->after_configure = dhcp6_pd_after_route_configure;
 
         assigned_link = dhcp6_pd_get_link_by_prefix(link, prefix);
         if (assigned_link) {
@@ -381,14 +402,35 @@ static void log_dhcp6_pd_address(Link *link, const Address *address) {
                       preferred_str ? "for " : "forever", strempty(preferred_str));
 }
 
-static int dhcp6_set_pd_address(
+static int dhcp6_pd_after_address_configure(Request *req, void *object) {
+        Address *address = object;
+        Link *link;
+        int r;
+
+        assert(req);
+        assert(req->link);
+        assert(req->type == REQUEST_TYPE_ADDRESS);
+        assert(address);
+
+        link = req->link;
+
+        r = set_ensure_put(&link->dhcp6_pd_addresses, &address_hash_ops, address);
+        if (r < 0)
+                return log_link_error_errno(link, r, "Failed to store DHCPv6 delegated prefix address: %m");
+
+        set_remove(link->dhcp6_pd_addresses_old, address);
+
+        return 0;
+}
+
+static int dhcp6_pd_request_address(
                 Link *link,
                 const struct in6_addr *prefix,
                 uint32_t lifetime_preferred,
                 uint32_t lifetime_valid) {
 
         _cleanup_(address_freep) Address *address = NULL;
-        Address *ret;
+        Request *req;
         int r;
 
         assert(link);
@@ -420,19 +462,16 @@ static int dhcp6_set_pd_address(
         address->route_metric = link->network->dhcp6_pd_route_metric;
 
         log_dhcp6_pd_address(link, address);
-        r = address_configure(address, link, dhcp6_pd_address_handler, &ret);
-        if (r < 0)
-                return log_link_error_errno(link, r, "Failed to set DHCPv6 delegated prefix address: %m");
-        if (r > 0)
+
+        if (address_get(link, address, NULL) < 0)
                 link->dhcp6_pd_address_configured = false;
 
-        link->dhcp6_pd_address_messages++;
-
-        r = set_ensure_put(&link->dhcp6_pd_addresses, &address_hash_ops, ret);
+        r = link_request_address(link, TAKE_PTR(address), true, &link->dhcp6_pd_address_messages,
+                                 dhcp6_pd_address_handler, &req);
         if (r < 0)
-                return log_link_error_errno(link, r, "Failed to store DHCPv6 delegated prefix address: %m");
+                return log_link_error_errno(link, r, "Failed to request DHCPv6 delegated prefix address: %m");
 
-        (void) set_remove(link->dhcp6_pd_addresses_old, ret);
+        req->after_configure = dhcp6_pd_after_address_configure;
 
         return 0;
 }
@@ -456,11 +495,11 @@ static int dhcp6_pd_assign_prefix(
                         return r;
         }
 
-        r = dhcp6_set_pd_route(link, prefix, pd_prefix);
+        r = dhcp6_pd_request_route(link, prefix, pd_prefix);
         if (r < 0)
                 return r;
 
-        r = dhcp6_set_pd_address(link, prefix, lifetime_preferred, lifetime_valid);
+        r = dhcp6_pd_request_address(link, prefix, lifetime_preferred, lifetime_valid);
         if (r < 0)
                 return r;
 
@@ -649,11 +688,10 @@ static int dhcp6_pd_finalize(Link *link) {
         if (r < 0)
                 return r;
 
-        if (link->dhcp6_pd_address_configured && link->dhcp6_pd_route_configured)
-                link_check_ready(link);
-        else
+        if (!link->dhcp6_pd_address_configured || !link->dhcp6_pd_route_configured)
                 link_set_state(link, LINK_STATE_CONFIGURING);
 
+        link_check_ready(link);
         return 0;
 }
 
@@ -799,10 +837,31 @@ static int dhcp6_route_handler(sd_netlink *rtnl, sd_netlink_message *m, Link *li
         return 1;
 }
 
-static int dhcp6_set_unreachable_route(Link *link, const struct in6_addr *addr, uint8_t prefixlen) {
+static int dhcp6_after_route_configure(Request *req, void *object) {
+        Route *route = object;
+        Link *link;
+        int r;
+
+        assert(req);
+        assert(req->link);
+        assert(req->type == REQUEST_TYPE_ROUTE);
+        assert(route);
+
+        link = req->link;
+
+        r = set_ensure_put(&link->dhcp6_routes, &route_hash_ops, route);
+        if (r < 0)
+                return log_link_error_errno(link, r, "Failed to store unreachable route for DHCPv6 delegated subnet: %m");
+
+        set_remove(link->dhcp6_routes_old, route);
+
+        return 0;
+}
+
+static int dhcp6_request_unreachable_route(Link *link, const struct in6_addr *addr, uint8_t prefixlen) {
         _cleanup_(route_freep) Route *route = NULL;
         _cleanup_free_ char *buf = NULL;
-        Route *ret;
+        Request *req;
         int r;
 
         assert(link);
@@ -827,21 +886,19 @@ static int dhcp6_set_unreachable_route(Link *link, const struct in6_addr *addr,
         route->type = RTN_UNREACHABLE;
         route->protocol = RTPROT_DHCP;
 
-        r = route_configure(route, link, dhcp6_route_handler, &ret);
+        r = link_has_route(link, route);
         if (r < 0)
-                return log_link_error_errno(link, r, "Failed to set unreachable route for DHCPv6 delegated subnet %s: %m",
-                                            strna(buf));
-        if (r > 0)
+                return r;
+        if (r == 0)
                 link->dhcp6_route_configured = false;
 
-        link->dhcp6_route_messages++;
-
-        r = set_ensure_put(&link->dhcp6_routes, &route_hash_ops, ret);
+        r = link_request_route(link, TAKE_PTR(route), true, &link->dhcp6_route_messages,
+                               dhcp6_route_handler, &req);
         if (r < 0)
-                return log_link_error_errno(link, r, "Failed to store unreachable route for DHCPv6 delegated subnet %s: %m",
+                return log_link_error_errno(link, r, "Failed to request unreachable route for DHCPv6 delegated subnet %s: %m",
                                             strna(buf));
 
-        (void) set_remove(link->dhcp6_routes_old, ret);
+        req->after_configure = dhcp6_after_route_configure;
 
         return 0;
 }
@@ -917,7 +974,7 @@ static int dhcp6_pd_prefix_acquired(Link *dhcp6_link) {
                 if (r == 0)
                         continue;
 
-                r = dhcp6_set_unreachable_route(dhcp6_link, &pd_prefix, pd_prefix_len);
+                r = dhcp6_request_unreachable_route(dhcp6_link, &pd_prefix, pd_prefix_len);
                 if (r < 0)
                         return r;
 
@@ -1064,7 +1121,28 @@ finalize:
                 *ret = TAKE_PTR(buffer);
 }
 
-static int dhcp6_update_address(
+static int dhcp6_after_address_configure(Request *req, void *object) {
+        Address *address = object;
+        Link *link;
+        int r;
+
+        assert(req);
+        assert(req->link);
+        assert(req->type == REQUEST_TYPE_ADDRESS);
+        assert(address);
+
+        link = req->link;
+
+        r = set_ensure_put(&link->dhcp6_addresses, &address_hash_ops, address);
+        if (r < 0)
+                return log_link_error_errno(link, r, "Failed to store DHCPv6 address: %m");
+
+        set_remove(link->dhcp6_addresses_old, address);
+
+        return 0;
+}
+
+static int dhcp6_request_address(
                 Link *link,
                 const struct in6_addr *ip6_addr,
                 uint32_t lifetime_preferred,
@@ -1072,7 +1150,7 @@ static int dhcp6_update_address(
 
         _cleanup_(address_freep) Address *addr = NULL;
         _cleanup_free_ char *buffer = NULL;
-        Address *ret;
+        Request *req;
         int r;
 
         r = address_new(&addr);
@@ -1088,19 +1166,15 @@ static int dhcp6_update_address(
 
         log_dhcp6_address(link, addr, &buffer);
 
-        r = address_configure(addr, link, dhcp6_address_handler, &ret);
-        if (r < 0)
-                return log_link_error_errno(link, r, "Failed to set DHCPv6 address %s: %m", strna(buffer));
-        if (r > 0)
+        if (address_get(link, addr, NULL) < 0)
                 link->dhcp6_address_configured = false;
 
-        link->dhcp6_address_messages++;
-
-        r = set_ensure_put(&link->dhcp6_addresses, &address_hash_ops, ret);
+        r = link_request_address(link, TAKE_PTR(addr), true, &link->dhcp6_address_messages,
+                                 dhcp6_address_handler, &req);
         if (r < 0)
-                return log_link_error_errno(link, r, "Failed to store DHCPv6 address %s: %m", strna(buffer));
+                return log_link_error_errno(link, r, "Failed to request DHCPv6 address %s: %m", strna(buffer));
 
-        (void) set_remove(link->dhcp6_addresses_old, ret);
+        req->after_configure = dhcp6_after_address_configure;
 
         return 0;
 }
@@ -1123,7 +1197,7 @@ static int dhcp6_address_acquired(Link *link) {
                 if (r < 0)
                         break;
 
-                r = dhcp6_update_address(link, &ip6_addr, lifetime_preferred, lifetime_valid);
+                r = dhcp6_request_address(link, &ip6_addr, lifetime_preferred, lifetime_valid);
                 if (r < 0)
                         return r;
         }
@@ -1203,11 +1277,10 @@ static int dhcp6_lease_ip_acquired(sd_dhcp6_client *client, Link *link) {
         if (r < 0)
                 return r;
 
-        if (link->dhcp6_address_configured && link->dhcp6_route_configured)
-                link_check_ready(link);
-        else
+        if (!link->dhcp6_address_configured || !link->dhcp6_route_configured)
                 link_set_state(link, LINK_STATE_CONFIGURING);
 
+        link_check_ready(link);
         return 0;
 }
 
@@ -1277,7 +1350,7 @@ static void dhcp6_handler(sd_dhcp6_client *client, int event, void *userdata) {
         }
 }
 
-int dhcp6_request_address(Link *link, int ir) {
+int dhcp6_request_information(Link *link, int ir) {
         int r, inf_req, pd;
         bool running;
 
@@ -1355,7 +1428,7 @@ int dhcp6_start(Link *link) {
 
         log_link_debug(link, "Acquiring DHCPv6 lease");
 
-        return dhcp6_request_address(link, link->network->dhcp6_without_ra == DHCP6_CLIENT_START_MODE_INFORMATION_REQUEST);
+        return dhcp6_request_information(link, link->network->dhcp6_without_ra == DHCP6_CLIENT_START_MODE_INFORMATION_REQUEST);
 }
 
 int dhcp6_request_prefix_delegation(Link *link) {

src/network/networkd-dhcp6.h

@@ -32,7 +32,7 @@ int dhcp6_pd_remove(Link *link);
 int dhcp6_configure(Link *link);
 int dhcp6_update_mac(Link *link);
 int dhcp6_start(Link *link);
-int dhcp6_request_address(Link *link, int ir);
+int dhcp6_request_information(Link *link, int ir);
 int dhcp6_request_prefix_delegation(Link *link);
 
 int link_serialize_dhcp6_client(Link *link, FILE *f);

src/network/networkd-ipv4ll.c

@@ -8,39 +8,55 @@
 #include "networkd-ipv4ll.h"
 #include "networkd-link.h"
 #include "networkd-manager.h"
+#include "networkd-queue.h"
 #include "parse-util.h"
 
+static int address_new_from_ipv4ll(Link *link, Address **ret) {
+        _cleanup_(address_freep) Address *address = NULL;
+        struct in_addr addr;
+        int r;
+
+        assert(link);
+        assert(link->ipv4ll);
+        assert(ret);
+
+        r = sd_ipv4ll_get_address(link->ipv4ll, &addr);
+        if (r < 0)
+                return r;
+
+        r = address_new(&address);
+        if (r < 0)
+                return -ENOMEM;
+
+        address->family = AF_INET;
+        address->in_addr.in = addr;
+        address->prefixlen = 16;
+        address->broadcast.s_addr = address->in_addr.in.s_addr | htobe32(UINT32_C(0xffffffff) >> address->prefixlen);
+        address->scope = RT_SCOPE_LINK;
+        address->route_metric = IPV4LL_ROUTE_METRIC;
+
+        *ret = TAKE_PTR(address);
+        return 0;
+}
+
 static int ipv4ll_address_lost(Link *link) {
         _cleanup_(address_freep) Address *address = NULL;
-        struct in_addr addr;
         int r;
 
         assert(link);
 
         link->ipv4ll_address_configured = false;
 
-        r = sd_ipv4ll_get_address(link->ipv4ll, &addr);
-        if (r < 0)
+        r = address_new_from_ipv4ll(link, &address);
+        if (r == -ENOENT)
                 return 0;
-
-        log_link_debug(link, "IPv4 link-local release "IPV4_ADDRESS_FMT_STR, IPV4_ADDRESS_FMT_VAL(addr));
-
-        r = address_new(&address);
-        if (r < 0)
-                return log_link_error_errno(link, r, "Could not allocate address: %m");
-
-        address->family = AF_INET;
-        address->in_addr.in = addr;
-        address->prefixlen = 16;
-        address->scope = RT_SCOPE_LINK;
-
-        r = address_remove(address, link, NULL);
         if (r < 0)
                 return r;
 
-        link_check_ready(link);
+        log_link_debug(link, "IPv4 link-local release "IPV4_ADDRESS_FMT_STR,
+                       IPV4_ADDRESS_FMT_VAL(address->in_addr.in));
 
-        return 0;
+        return address_remove(address, link, NULL);
 }
 
 static int ipv4ll_address_handler(sd_netlink *rtnl, sd_netlink_message *m, Link *link) {
@@ -60,8 +76,7 @@ static int ipv4ll_address_handler(sd_netlink *rtnl, sd_netlink_message *m, Link
 }
 
 static int ipv4ll_address_claimed(sd_ipv4ll *ll, Link *link) {
-        _cleanup_(address_freep) Address *ll_addr = NULL;
-        struct in_addr address;
+        _cleanup_(address_freep) Address *address = NULL;
         int r;
 
         assert(ll);
@@ -69,31 +84,16 @@ static int ipv4ll_address_claimed(sd_ipv4ll *ll, Link *link) {
 
         link->ipv4ll_address_configured = false;
 
-        r = sd_ipv4ll_get_address(ll, &address);
+        r = address_new_from_ipv4ll(link, &address);
         if (r == -ENOENT)
                 return 0;
-        else if (r < 0)
+        if (r < 0)
                 return r;
 
         log_link_debug(link, "IPv4 link-local claim "IPV4_ADDRESS_FMT_STR,
-                       IPV4_ADDRESS_FMT_VAL(address));
+                       IPV4_ADDRESS_FMT_VAL(address->in_addr.in));
 
-        r = address_new(&ll_addr);
-        if (r < 0)
-                return r;
-
-        ll_addr->family = AF_INET;
-        ll_addr->in_addr.in = address;
-        ll_addr->prefixlen = 16;
-        ll_addr->broadcast.s_addr = ll_addr->in_addr.in.s_addr | htobe32(0xfffffffflu >> ll_addr->prefixlen);
-        ll_addr->scope = RT_SCOPE_LINK;
-        ll_addr->route_metric = IPV4LL_ROUTE_METRIC;
-
-        r = address_configure(ll_addr, link, ipv4ll_address_handler, NULL);
-        if (r < 0)
-                return r;
-
-        return 0;
+        return link_request_address(link, TAKE_PTR(address), true, NULL, ipv4ll_address_handler, NULL);
 }
 
 static void ipv4ll_handler(sd_ipv4ll *ll, int event, void *userdata) {

src/network/networkd-link.c

@@ -33,6 +33,7 @@
 #include "networkd-dhcp6.h"
 #include "networkd-fdb.h"
 #include "networkd-ipv4ll.h"
+#include "networkd-ipv6-proxy-ndp.h"
 #include "networkd-link-bus.h"
 #include "networkd-link.h"
 #include "networkd-lldp-tx.h"
@@ -793,7 +794,7 @@ void link_check_ready(Link *link) {
                 return;
         }
 
-        if (!link->addresses_configured)
+        if (!link->static_addresses_configured)
                 return (void) log_link_debug(link, "%s(): static addresses are not configured.", __func__);
 
         SET_FOREACH(a, link->addresses)
@@ -807,12 +808,12 @@ void link_check_ready(Link *link) {
         if (!link->static_neighbors_configured)
                 return (void) log_link_debug(link, "%s(): static neighbors are not configured.", __func__);
 
-        if (!link->static_routes_configured)
-                return (void) log_link_debug(link, "%s(): static routes are not configured.", __func__);
-
         if (!link->static_nexthops_configured)
                 return (void) log_link_debug(link, "%s(): static nexthops are not configured.", __func__);
 
+        if (!link->static_routes_configured)
+                return (void) log_link_debug(link, "%s(): static routes are not configured.", __func__);
+
         if (!link->static_routing_policy_rules_configured)
                 return (void) log_link_debug(link, "%s(): static routing policy rules are not configured.", __func__);
 
@@ -877,13 +878,6 @@ static int link_set_static_configs(Link *link) {
         assert(link->network);
         assert(link->state != _LINK_STATE_INVALID);
 
-        /* Reset all *_configured flags we are configuring. */
-        link->request_static_addresses = false;
-        link->addresses_configured = false;
-        link->addresses_ready = false;
-        link->static_routes_configured = false;
-        link->static_nexthops_configured = false;
-
         r = link_set_bridge_fdb(link);
         if (r < 0)
                 return r;
@@ -892,11 +886,15 @@ static int link_set_static_configs(Link *link) {
         if (r < 0)
                 return r;
 
+        r = link_set_ipv6_proxy_ndp_addresses(link);
+        if (r < 0)
+                return r;
+
         r = link_set_address_labels(link);
         if (r < 0)
                 return r;
 
-        r = link_set_addresses(link);
+        r = link_request_static_addresses(link);
         if (r < 0)
                 return r;
 
@@ -904,12 +902,15 @@ static int link_set_static_configs(Link *link) {
         if (r < 0)
                 return r;
 
-        r = link_request_static_routing_policy_rules(link);
+        r = link_request_static_nexthops(link, false);
         if (r < 0)
                 return r;
 
-        /* now that we can figure out a default address for the dhcp server, start it */
-        r = dhcp4_server_configure(link);
+        r = link_request_static_routes(link, false);
+        if (r < 0)
+                return r;
+
+        r = link_request_static_routing_policy_rules(link);
         if (r < 0)
                 return r;
 

src/network/networkd-link.h

@@ -78,14 +78,16 @@ typedef struct Link {
         LinkAddressState ipv4_address_state;
         LinkAddressState ipv6_address_state;
 
-        unsigned address_messages;
-        unsigned address_remove_messages;
         unsigned address_label_messages;
+        unsigned static_address_messages;
         unsigned static_neighbor_messages;
-        unsigned route_messages;
-        unsigned nexthop_messages;
+        unsigned static_nexthop_messages;
+        unsigned static_route_messages;
         unsigned static_routing_policy_rule_messages;
+        unsigned address_remove_messages;
         unsigned neighbor_remove_messages;
+        unsigned nexthop_remove_messages;
+        unsigned route_remove_messages;
         unsigned tc_messages;
         unsigned sr_iov_messages;
         unsigned enslaving;
@@ -119,12 +121,10 @@ typedef struct Link {
         sd_ipv4ll *ipv4ll;
         bool ipv4ll_address_configured:1;
 
-        bool request_static_addresses:1;
-        bool addresses_configured:1;
-        bool addresses_ready:1;
+        bool static_addresses_configured:1;
         bool static_neighbors_configured:1;
-        bool static_routes_configured:1;
         bool static_nexthops_configured:1;
+        bool static_routes_configured:1;
         bool static_routing_policy_rules_configured:1;
         bool tc_configured:1;
         bool sr_iov_configured:1;

src/network/networkd-manager.h

@@ -70,10 +70,12 @@ struct Manager {
         Hashmap *nexthops_by_id;
 
         /* Manager stores nexthops without RTA_OIF attribute. */
+        unsigned nexthop_remove_messages;
         Set *nexthops;
         Set *nexthops_foreign;
 
         /* Manager stores routes without RTA_OIF attribute. */
+        unsigned route_remove_messages;
         Set *routes;
         Set *routes_foreign;
 

src/network/networkd-ndisc.c

@@ -15,6 +15,7 @@
 #include "networkd-dhcp6.h"
 #include "networkd-manager.h"
 #include "networkd-ndisc.h"
+#include "networkd-queue.h"
 #include "networkd-state-file.h"
 #include "string-table.h"
 #include "string-util.h"
@@ -325,24 +326,23 @@ static int ndisc_route_handler(sd_netlink *rtnl, sd_netlink_message *m, Link *li
         return 1;
 }
 
-static int ndisc_route_configure(Route *route, Link *link, sd_ndisc_router *rt) {
+static int ndisc_after_route_configure(Request *req, void *object) {
         _cleanup_free_ NDiscRoute *nr = NULL;
         NDiscRoute *nr_exist;
         struct in6_addr router;
-        Route *ret;
+        Route *route = object;
+        sd_ndisc_router *rt;
+        Link *link;
         int r;
 
+        assert(req);
+        assert(req->link);
+        assert(req->type == REQUEST_TYPE_ROUTE);
+        assert(req->userdata);
         assert(route);
-        assert(link);
-        assert(rt);
 
-        r = route_configure(route, link, ndisc_route_handler, &ret);
-        if (r < 0)
-                return log_link_error_errno(link, r, "Failed to set NDisc route: %m");
-        if (r > 0)
-                link->ndisc_routes_configured = false;
-
-        link->ndisc_routes_messages++;
+        link = req->link;
+        rt = req->userdata;
 
         r = sd_ndisc_router_get_address(rt, &router);
         if (r < 0)
@@ -354,7 +354,7 @@ static int ndisc_route_configure(Route *route, Link *link, sd_ndisc_router *rt)
 
         *nr = (NDiscRoute) {
                 .router = router,
-                .route = ret,
+                .route = route,
         };
 
         nr_exist = set_get(link->ndisc_routes, nr);
@@ -373,6 +373,39 @@ static int ndisc_route_configure(Route *route, Link *link, sd_ndisc_router *rt)
         return 0;
 }
 
+static void ndisc_request_on_free(Request *req) {
+        assert(req);
+
+        sd_ndisc_router_unref(req->userdata);
+}
+
+static int ndisc_request_route(Route *in, Link *link, sd_ndisc_router *rt) {
+        _cleanup_(route_freep) Route *route = in;
+        Request *req;
+        int r;
+
+        assert(route);
+        assert(link);
+        assert(rt);
+
+        r = link_has_route(link, route);
+        if (r < 0)
+                return r;
+        if (r == 0)
+                link->ndisc_routes_configured = false;
+
+        r = link_request_route(link, TAKE_PTR(route), true, &link->ndisc_routes_messages,
+                               ndisc_route_handler, &req);
+        if (r < 0)
+                return r;
+
+        req->userdata = sd_ndisc_router_ref(rt);
+        req->after_configure = ndisc_after_route_configure;
+        req->on_free = ndisc_request_on_free;
+
+        return 0;
+}
+
 static void ndisc_address_hash_func(const NDiscAddress *x, struct siphash *state) {
         address_hash_func(x->address, state);
 }
@@ -414,24 +447,23 @@ static int ndisc_address_handler(sd_netlink *rtnl, sd_netlink_message *m, Link *
         return 1;
 }
 
-static int ndisc_address_configure(Address *address, Link *link, sd_ndisc_router *rt) {
+static int ndisc_after_address_configure(Request *req, void *object) {
         _cleanup_free_ NDiscAddress *na = NULL;
         NDiscAddress *na_exist;
         struct in6_addr router;
-        Address *ret;
+        sd_ndisc_router *rt;
+        Address *address = object;
+        Link *link;
         int r;
 
+        assert(req);
+        assert(req->link);
+        assert(req->type == REQUEST_TYPE_ADDRESS);
+        assert(req->userdata);
         assert(address);
-        assert(link);
-        assert(rt);
 
-        r = address_configure(address, link, ndisc_address_handler, &ret);
-        if (r < 0)
-                return log_link_error_errno(link, r, "Failed to set NDisc SLAAC address: %m");
-        if (r > 0)
-                link->ndisc_addresses_configured = false;
-
-        link->ndisc_addresses_messages++;
+        link = req->link;
+        rt = req->userdata;
 
         r = sd_ndisc_router_get_address(rt, &router);
         if (r < 0)
@@ -443,7 +475,7 @@ static int ndisc_address_configure(Address *address, Link *link, sd_ndisc_router
 
         *na = (NDiscAddress) {
                 .router = router,
-                .address = ret,
+                .address = address,
         };
 
         na_exist = set_get(link->ndisc_addresses, na);
@@ -462,6 +494,30 @@ static int ndisc_address_configure(Address *address, Link *link, sd_ndisc_router
         return 0;
 }
 
+static int ndisc_request_address(Address *in, Link *link, sd_ndisc_router *rt) {
+        _cleanup_(address_freep) Address *address = in;
+        Request *req;
+        int r;
+
+        assert(address);
+        assert(link);
+        assert(rt);
+
+        if (address_get(link, address, NULL) < 0)
+                link->ndisc_addresses_configured = false;
+
+        r = link_request_address(link, TAKE_PTR(address), true, &link->ndisc_addresses_messages,
+                                 ndisc_address_handler, &req);
+        if (r < 0)
+                return r;
+
+        req->userdata = sd_ndisc_router_ref(rt);
+        req->after_configure = ndisc_after_address_configure;
+        req->on_free = ndisc_request_on_free;
+
+        return 0;
+}
+
 static int ndisc_router_process_default(Link *link, sd_ndisc_router *rt) {
         _cleanup_(route_freep) Route *route = NULL;
         struct in6_addr gateway;
@@ -526,9 +582,9 @@ static int ndisc_router_process_default(Link *link, sd_ndisc_router *rt) {
         route->lifetime = usec_add(time_now, lifetime * USEC_PER_SEC);
         route->mtu = mtu;
 
-        r = ndisc_route_configure(route, link, rt);
+        r = ndisc_request_route(TAKE_PTR(route), link, rt);
         if (r < 0)
-                return log_link_error_errno(link, r, "Could not set default route: %m");
+                return log_link_error_errno(link, r, "Could not request default route: %m");
 
         Route *route_gw;
         HASHMAP_FOREACH(route_gw, link->network->routes_by_section) {
@@ -538,22 +594,26 @@ static int ndisc_router_process_default(Link *link, sd_ndisc_router *rt) {
                 if (route_gw->gw_family != AF_INET6)
                         continue;
 
-                route_gw->gw.in6 = gateway;
-                if (!route_gw->table_set)
-                        route_gw->table = table;
-                if (!route_gw->priority_set)
-                        route_gw->priority = link->network->ipv6_accept_ra_route_metric;
-                if (!route_gw->protocol_set)
-                        route_gw->protocol = RTPROT_RA;
-                if (!route_gw->pref_set)
-                        route_gw->pref = preference;
-                route_gw->lifetime = usec_add(time_now, lifetime * USEC_PER_SEC);
-                if (route_gw->mtu == 0)
-                        route_gw->mtu = mtu;
-
-                r = ndisc_route_configure(route_gw, link, rt);
+                r = route_dup(route_gw, &route);
                 if (r < 0)
-                        return log_link_error_errno(link, r, "Could not set gateway: %m");
+                        return r;
+
+                route->gw.in6 = gateway;
+                if (!route->table_set)
+                        route->table = table;
+                if (!route->priority_set)
+                        route->priority = link->network->ipv6_accept_ra_route_metric;
+                if (!route->protocol_set)
+                        route->protocol = RTPROT_RA;
+                if (!route->pref_set)
+                        route->pref = preference;
+                route->lifetime = usec_add(time_now, lifetime * USEC_PER_SEC);
+                if (route->mtu == 0)
+                        route->mtu = mtu;
+
+                r = ndisc_request_route(TAKE_PTR(route), link, rt);
+                if (r < 0)
+                        return log_link_error_errno(link, r, "Could not request gateway: %m");
         }
 
         return 0;
@@ -697,7 +757,6 @@ static int ndisc_router_generate_addresses(Link *link, struct in6_addr *address,
 static int ndisc_router_process_autonomous_prefix(Link *link, sd_ndisc_router *rt) {
         uint32_t lifetime_valid, lifetime_preferred, lifetime_remaining;
         _cleanup_set_free_free_ Set *addresses = NULL;
-        _cleanup_(address_freep) Address *address = NULL;
         struct in6_addr addr, *a;
         unsigned prefixlen;
         usec_t time_now;
@@ -734,18 +793,18 @@ static int ndisc_router_process_autonomous_prefix(Link *link, sd_ndisc_router *r
         if (r < 0)
                 return r;
 
-        r = address_new(&address);
-        if (r < 0)
-                return log_oom();
-
-        address->family = AF_INET6;
-        address->prefixlen = prefixlen;
-        address->flags = IFA_F_NOPREFIXROUTE|IFA_F_MANAGETEMPADDR;
-        address->cinfo.ifa_prefered = lifetime_preferred;
-
         SET_FOREACH(a, addresses) {
+                _cleanup_(address_freep) Address *address = NULL;
                 Address *existing_address;
 
+                r = address_new(&address);
+                if (r < 0)
+                        return log_oom();
+
+                address->family = AF_INET6;
+                address->prefixlen = prefixlen;
+                address->flags = IFA_F_NOPREFIXROUTE|IFA_F_MANAGETEMPADDR;
+                address->cinfo.ifa_prefered = lifetime_preferred;
                 address->in_addr.in6 = *a;
 
                 /* see RFC4862 section 5.5.3.e */
@@ -766,9 +825,9 @@ static int ndisc_router_process_autonomous_prefix(Link *link, sd_ndisc_router *r
                 if (address->cinfo.ifa_valid == 0)
                         continue;
 
-                r = ndisc_address_configure(address, link, rt);
+                r = ndisc_request_address(TAKE_PTR(address), link, rt);
                 if (r < 0)
-                        return log_link_error_errno(link, r, "Could not set SLAAC address: %m");
+                        return log_link_error_errno(link, r, "Could not request SLAAC address: %m");
         }
 
         return 0;
@@ -812,9 +871,9 @@ static int ndisc_router_process_onlink_prefix(Link *link, sd_ndisc_router *rt) {
         if (r < 0)
                 return log_link_error_errno(link, r, "Failed to get prefix address: %m");
 
-        r = ndisc_route_configure(route, link, rt);
+        r = ndisc_request_route(TAKE_PTR(route), link, rt);
         if (r < 0)
-                return log_link_error_errno(link, r, "Could not set prefix route: %m");;
+                return log_link_error_errno(link, r, "Could not request prefix route: %m");;
 
         return 0;
 }
@@ -896,9 +955,9 @@ static int ndisc_router_process_route(Link *link, sd_ndisc_router *rt) {
         route->dst_prefixlen = prefixlen;
         route->lifetime = usec_add(time_now, lifetime * USEC_PER_SEC);
 
-        r = ndisc_route_configure(route, link, rt);
+        r = ndisc_request_route(TAKE_PTR(route), link, rt);
         if (r < 0)
-                return log_link_error_errno(link, r, "Could not set additional route: %m");
+                return log_link_error_errno(link, r, "Could not request additional route: %m");
 
         return 0;
 }
@@ -1223,11 +1282,11 @@ static int ndisc_router_handler(Link *link, sd_ndisc_router *rt) {
 
                 if (flags & (ND_RA_FLAG_MANAGED | ND_RA_FLAG_OTHER))
                         /* (re)start DHCPv6 client in stateful or stateless mode according to RA flags */
-                        r = dhcp6_request_address(link, !(flags & ND_RA_FLAG_MANAGED));
+                        r = dhcp6_request_information(link, !(flags & ND_RA_FLAG_MANAGED));
                 else
                         /* When IPv6AcceptRA.DHCPv6Client=always, start dhcp6 client in managed mode
                          * even if router does not have M or O flag. */
-                        r = dhcp6_request_address(link, false);
+                        r = dhcp6_request_information(link, false);
                 if (r < 0 && r != -EBUSY)
                         return log_link_error_errno(link, r, "Could not acquire DHCPv6 lease on NDisc request: %m");
                 else
@@ -1255,11 +1314,10 @@ static int ndisc_router_handler(Link *link, sd_ndisc_router *rt) {
         if (r < 0)
                 return r;
 
-        if (link->ndisc_addresses_configured && link->ndisc_routes_configured)
-                link_check_ready(link);
-        else
+        if (!link->ndisc_addresses_configured || !link->ndisc_routes_configured)
                 link_set_state(link, LINK_STATE_CONFIGURING);
 
+        link_check_ready(link);
         return 0;
 }
 

src/network/networkd-nexthop.c

@@ -10,6 +10,8 @@
 #include "networkd-manager.h"
 #include "networkd-network.h"
 #include "networkd-nexthop.h"
+#include "networkd-queue.h"
+#include "networkd-route.h"
 #include "parse-util.h"
 #include "set.h"
 #include "string-util.h"
@@ -356,13 +358,16 @@ static void log_nexthop_debug(const NextHop *nexthop, uint32_t id, const char *s
                                str, nexthop->id, id, strna(gw), yes_no(nexthop->blackhole));
 }
 
-static int nexthop_remove_handler(sd_netlink *rtnl, sd_netlink_message *m, Link *link) {
+static int link_nexthop_remove_handler(sd_netlink *rtnl, sd_netlink_message *m, Link *link) {
         int r;
 
         assert(m);
+        assert(link);
+        assert(link->nexthop_remove_messages > 0);
 
-        /* Note that link may be NULL. */
-        if (link && IN_SET(link->state, LINK_STATE_FAILED, LINK_STATE_LINGER))
+        link->nexthop_remove_messages--;
+
+        if (IN_SET(link->state, LINK_STATE_FAILED, LINK_STATE_LINGER))
                 return 1;
 
         r = sd_netlink_message_get_errno(m);
@@ -372,6 +377,22 @@ static int nexthop_remove_handler(sd_netlink *rtnl, sd_netlink_message *m, Link
         return 1;
 }
 
+static int manager_nexthop_remove_handler(sd_netlink *rtnl, sd_netlink_message *m, Manager *manager) {
+        int r;
+
+        assert(m);
+        assert(manager);
+        assert(manager->nexthop_remove_messages > 0);
+
+        manager->nexthop_remove_messages--;
+
+        r = sd_netlink_message_get_errno(m);
+        if (r < 0 && r != -ENOENT)
+                log_message_warning_errno(m, r, "Could not drop nexthop, ignoring");
+
+        return 1;
+}
+
 static int nexthop_remove(const NextHop *nexthop, Manager *manager, Link *link) {
         _cleanup_(sd_netlink_message_unrefp) sd_netlink_message *req = NULL;
         int r;
@@ -396,13 +417,22 @@ static int nexthop_remove(const NextHop *nexthop, Manager *manager, Link *link)
         if (r < 0)
                 return log_link_error_errno(link, r, "Could not append NHA_ID attribute: %m");
 
-        r = netlink_call_async(manager->rtnl, NULL, req, nexthop_remove_handler,
-                               link_netlink_destroy_callback, link);
+        if (link)
+                r = netlink_call_async(manager->rtnl, NULL, req, link_nexthop_remove_handler,
+                                       link_netlink_destroy_callback, link);
+        else
+                r = netlink_call_async(manager->rtnl, NULL, req, manager_nexthop_remove_handler,
+                                       NULL, manager);
         if (r < 0)
                 return log_link_error_errno(link, r, "Could not send rtnetlink message: %m");
 
         link_ref(link); /* link may be NULL, link_ref() is OK with that */
 
+        if (link)
+                link->nexthop_remove_messages++;
+        else
+                manager->nexthop_remove_messages++;
+
         return 0;
 }
 
@@ -476,9 +506,9 @@ static int static_nexthop_handler(sd_netlink *rtnl, sd_netlink_message *m, Link
         int r;
 
         assert(link);
-        assert(link->nexthop_messages > 0);
+        assert(link->static_nexthop_messages > 0);
 
-        link->nexthop_messages--;
+        link->static_nexthop_messages--;
 
         if (IN_SET(link->state, LINK_STATE_FAILED, LINK_STATE_LINGER))
                 return 1;
@@ -490,57 +520,57 @@ static int static_nexthop_handler(sd_netlink *rtnl, sd_netlink_message *m, Link
                 return 1;
         }
 
-        if (link->nexthop_messages == 0) {
+        if (link->static_nexthop_messages == 0) {
                 log_link_debug(link, "Nexthops set");
                 link->static_nexthops_configured = true;
-                /* Now all nexthops are configured. Let's configure remaining routes. */
-                r = link_set_routes_with_gateway(link);
-                if (r < 0)
-                        link_enter_failed(link);
+                link_check_ready(link);
         }
 
         return 1;
 }
 
-int link_set_nexthops(Link *link) {
-        enum {
-                PHASE_ID,         /* First phase: Nexthops with ID */
-                PHASE_WITHOUT_ID, /* Second phase: Nexthops without ID */
-                _PHASE_MAX,
-        } phase;
+static int link_request_nexthop(
+                Link *link,
+                NextHop *nexthop,
+                bool consume_object,
+                unsigned *message_counter,
+                link_netlink_message_handler_t netlink_handler,
+                Request **ret) {
+
+        assert(link);
+        assert(nexthop);
+
+        log_nexthop_debug(nexthop, nexthop->id, "Requesting", link);
+        return link_queue_request(link, REQUEST_TYPE_NEXTHOP, nexthop, consume_object,
+                                  message_counter, netlink_handler, ret);
+}
+
+int link_request_static_nexthops(Link *link, bool only_ipv4) {
         NextHop *nh;
         int r;
 
         assert(link);
         assert(link->network);
 
-        if (link->nexthop_messages != 0) {
-                log_link_debug(link, "Nexthops are configuring.");
-                return 0;
-        }
-
         link->static_nexthops_configured = false;
 
-        for (phase = PHASE_ID; phase < _PHASE_MAX; phase++)
-                HASHMAP_FOREACH(nh, link->network->nexthops_by_section) {
-                        if ((nh->id > 0) != (phase == PHASE_ID))
-                                continue;
+        HASHMAP_FOREACH(nh, link->network->nexthops_by_section) {
+                if (only_ipv4 && nh->family != AF_INET)
+                        continue;
 
-                        r = nexthop_configure(nh, link, static_nexthop_handler, NULL);
-                        if (r < 0)
-                                return log_link_warning_errno(link, r, "Could not set nexthop: %m");
-
-                        link->nexthop_messages++;
-                }
-
-        if (link->nexthop_messages == 0) {
-                link->static_nexthops_configured = true;
-                /* Finally, configure routes with gateways. */
-                return link_set_routes_with_gateway(link);
+                r = link_request_nexthop(link, nh, false, &link->static_nexthop_messages,
+                                         static_nexthop_handler, NULL);
+                if (r < 0)
+                        return log_link_warning_errno(link, r, "Could not request nexthop: %m");
         }
 
-        log_link_debug(link, "Setting nexthops");
-        link_set_state(link, LINK_STATE_CONFIGURING);
+        if (link->static_nexthop_messages == 0) {
+                link->static_nexthops_configured = true;
+                link_check_ready(link);
+        } else {
+                log_link_debug(link, "Requesting nexthops");
+                link_set_state(link, LINK_STATE_CONFIGURING);
+        }
 
         return 0;
 }
@@ -663,6 +693,73 @@ int link_drop_nexthops(Link *link) {
         return r;
 }
 
+static bool nexthop_is_ready_to_configure(Link *link, const NextHop *nexthop) {
+        assert(link);
+        assert(nexthop);
+
+        if (nexthop->blackhole) {
+                if (link->manager->nexthop_remove_messages > 0)
+                        return false;
+        } else {
+                Link *l;
+
+                HASHMAP_FOREACH(l, link->manager->links) {
+                        if (l->address_remove_messages > 0)
+                                return false;
+                        if (l->nexthop_remove_messages > 0)
+                                return false;
+                        if (l->route_remove_messages > 0)
+                                return false;
+                }
+        }
+
+        if (nexthop->id == 0) {
+                Request *req;
+
+                ORDERED_SET_FOREACH(req, link->manager->request_queue) {
+                        if (req->type != REQUEST_TYPE_NEXTHOP)
+                                continue;
+                        if (req->nexthop->id != 0)
+                                return false; /* first configure nexthop with id. */
+                }
+        }
+
+        if (nexthop->onlink <= 0 &&
+            in_addr_is_set(nexthop->family, &nexthop->gw) &&
+            !manager_address_is_reachable(link->manager, nexthop->family, &nexthop->gw))
+                return false;
+
+        return true;
+}
+
+int request_process_nexthop(Request *req) {
+        NextHop *ret;
+        int r;
+
+        assert(req);
+        assert(req->link);
+        assert(req->nexthop);
+        assert(req->type == REQUEST_TYPE_NEXTHOP);
+
+        if (!link_is_ready_to_configure(req->link, false))
+                return 0;
+
+        if (!nexthop_is_ready_to_configure(req->link, req->nexthop))
+                return 0;
+
+        r = nexthop_configure(req->nexthop, req->link, req->netlink_handler, &ret);
+        if (r < 0)
+                return r;
+
+        if (req->after_configure) {
+                r = req->after_configure(req, ret);
+                if (r < 0)
+                        return r;
+        }
+
+        return 1;
+}
+
 int manager_rtnl_process_nexthop(sd_netlink *rtnl, sd_netlink_message *message, Manager *m) {
         _cleanup_(nexthop_freep) NextHop *tmp = NULL;
         NextHop *nexthop = NULL;

src/network/networkd-nexthop.h

@@ -15,6 +15,7 @@
 typedef struct Link Link;
 typedef struct Manager Manager;
 typedef struct Network Network;
+typedef struct Request Request;
 
 typedef struct NextHop {
         Network *network;
@@ -36,10 +37,12 @@ NextHop *nexthop_free(NextHop *nexthop);
 
 void network_drop_invalid_nexthops(Network *network);
 
-int link_set_nexthops(Link *link);
 int link_drop_nexthops(Link *link);
 int link_drop_foreign_nexthops(Link *link);
 
+int link_request_static_nexthops(Link *link, bool only_ipv4);
+int request_process_nexthop(Request *req);
+
 int manager_get_nexthop_by_id(Manager *manager, uint32_t id, NextHop **ret);
 int manager_rtnl_process_nexthop(sd_netlink *rtnl, sd_netlink_message *message, Manager *m);
 

src/network/networkd-queue.c

@@ -10,9 +10,18 @@
 
 static void request_free_object(RequestType type, void *object) {
         switch(type) {
+        case REQUEST_TYPE_ADDRESS:
+                address_free(object);
+                break;
         case REQUEST_TYPE_NEIGHBOR:
                 neighbor_free(object);
                 break;
+        case REQUEST_TYPE_NEXTHOP:
+                nexthop_free(object);
+                break;
+        case REQUEST_TYPE_ROUTE:
+                route_free(object);
+                break;
         case REQUEST_TYPE_ROUTING_POLICY_RULE:
                 routing_policy_rule_free(object);
                 break;
@@ -107,9 +116,18 @@ int manager_process_requests(sd_event_source *s, void *userdata) {
 
                 ORDERED_SET_FOREACH(req, manager->request_queue) {
                         switch(req->type) {
+                        case REQUEST_TYPE_ADDRESS:
+                                r = request_process_address(req);
+                                break;
                         case REQUEST_TYPE_NEIGHBOR:
                                 r = request_process_neighbor(req);
                                 break;
+                        case REQUEST_TYPE_NEXTHOP:
+                                r = request_process_nexthop(req);
+                                break;
+                        case REQUEST_TYPE_ROUTE:
+                                r = request_process_route(req);
+                                break;
                         case REQUEST_TYPE_ROUTING_POLICY_RULE:
                                 r = request_process_routing_policy_rule(req);
                                 break;

src/network/networkd-queue.h

@@ -5,7 +5,10 @@
 
 #include "networkd-link.h"
 
+typedef struct Address Address;
 typedef struct Neighbor Neighbor;
+typedef struct NextHop NextHop;
+typedef struct Route Route;
 typedef struct RoutingPolicyRule RoutingPolicyRule;
 
 typedef struct Request Request;
@@ -14,7 +17,10 @@ typedef int (*request_after_configure_handler_t)(Request*, void*);
 typedef void (*request_on_free_handler_t)(Request*);
 
 typedef enum RequestType {
+        REQUEST_TYPE_ADDRESS,
         REQUEST_TYPE_NEIGHBOR,
+        REQUEST_TYPE_NEXTHOP,
+        REQUEST_TYPE_ROUTE,
         REQUEST_TYPE_ROUTING_POLICY_RULE,
         _REQUEST_TYPE_MAX,
         _REQUEST_TYPE_INVALID = -EINVAL,
@@ -25,7 +31,10 @@ typedef struct Request {
         RequestType type;
         bool consume_object;
         union {
+                Address *address;
                 Neighbor *neighbor;
+                NextHop *nexthop;
+                Route *route;
                 RoutingPolicyRule *rule;
                 void *object;
         };

src/network/networkd-route.c

@@ -9,8 +9,8 @@
 #include "networkd-manager.h"
 #include "networkd-network.h"
 #include "networkd-nexthop.h"
+#include "networkd-queue.h"
 #include "networkd-route.h"
-#include "networkd-routing-policy-rule.h"
 #include "parse-util.h"
 #include "socket-netlink.h"
 #include "string-table.h"
@@ -887,8 +887,11 @@ int link_route_remove_handler_internal(sd_netlink *rtnl, sd_netlink_message *m,
 
         assert(m);
         assert(link);
+        assert(link->route_remove_messages > 0);
         assert(error_msg);
 
+        link->route_remove_messages--;
+
         if (IN_SET(link->state, LINK_STATE_FAILED, LINK_STATE_LINGER))
                 return 0;
 
@@ -907,6 +910,10 @@ static int manager_route_remove_handler(sd_netlink *rtnl, sd_netlink_message *m,
         int r;
 
         assert(m);
+        assert(manager);
+        assert(manager->route_remove_messages > 0);
+
+        manager->route_remove_messages--;
 
         r = sd_netlink_message_get_errno(m);
         if (r < 0 && r != -ESRCH)
@@ -966,12 +973,15 @@ int route_remove(
                         return log_link_error_errno(link, r, "Could not send rtnetlink message: %m");
 
                 link_ref(link);
+                link->route_remove_messages++;
         } else {
                 r = netlink_call_async(manager->rtnl, NULL, req,
                                        manager_route_remove_handler,
                                        NULL, manager);
                 if (r < 0)
                         return log_error_errno(r, "Could not send rtnetlink message: %m");
+
+                manager->route_remove_messages++;
         }
 
         return 0;
@@ -1277,7 +1287,7 @@ int route_configure_handler_internal(sd_netlink *rtnl, sd_netlink_message *m, Li
         return 1;
 }
 
-int route_configure(
+static int route_configure(
                 const Route *route,
                 Link *link,
                 link_netlink_message_handler_t callback,
@@ -1410,20 +1420,20 @@ int route_configure(
         return k;
 }
 
-static int route_handler_with_gateway(sd_netlink *rtnl, sd_netlink_message *m, Link *link) {
+static int static_route_handler(sd_netlink *rtnl, sd_netlink_message *m, Link *link) {
         int r;
 
         assert(link);
-        assert(link->route_messages > 0);
+        assert(link->static_route_messages > 0);
 
-        link->route_messages--;
+        link->static_route_messages--;
 
-        r = route_configure_handler_internal(rtnl, m, link, "Could not set route with gateway");
+        r = route_configure_handler_internal(rtnl, m, link, "Could not set route");
         if (r <= 0)
                 return r;
 
-        if (link->route_messages == 0) {
-                log_link_debug(link, "Routes with gateway set");
+        if (link->static_route_messages == 0) {
+                log_link_debug(link, "Routes set");
                 link->static_routes_configured = true;
                 link_check_ready(link);
         }
@@ -1431,151 +1441,151 @@ static int route_handler_with_gateway(sd_netlink *rtnl, sd_netlink_message *m, L
         return 1;
 }
 
-static int route_handler_without_gateway(sd_netlink *rtnl, sd_netlink_message *m, Link *link) {
-        int r;
+int link_request_route(
+                Link *link,
+                Route *route,
+                bool consume_object,
+                unsigned *message_counter,
+                link_netlink_message_handler_t netlink_handler,
+                Request **ret) {
 
         assert(link);
-        assert(link->route_messages > 0);
-
-        link->route_messages--;
-
-        r = route_configure_handler_internal(rtnl, m, link, "Could not set route without gateway");
-        if (r <= 0)
-                return r;
-
-        if (link->route_messages == 0) {
-                log_link_debug(link, "Routes set without gateway");
-                /* Now, we can talk to gateways, let's configure nexthops. */
-                r = link_set_nexthops(link);
-                if (r < 0)
-                        link_enter_failed(link);
-        }
-
-        return 1;
-}
-
-static bool route_has_gateway(const Route *route) {
+        assert(link->manager);
         assert(route);
 
-        if (in_addr_is_set(route->gw_family, &route->gw))
-                return true;
-
-        if (!ordered_set_isempty(route->multipath_routes))
-                return true;
-
-        if (route->nexthop_id > 0)
-                return true;
-
-        return false;
+        log_route_debug(route, "Requesting", link, link->manager);
+        return link_queue_request(link, REQUEST_TYPE_ROUTE, route, consume_object,
+                                  message_counter, netlink_handler, ret);
 }
 
-static int link_set_routes_internal(Link *link, bool with_gateway) {
+int link_request_static_routes(Link *link, bool only_ipv4) {
         Route *route;
         int r;
 
         assert(link);
         assert(link->network);
 
-        HASHMAP_FOREACH(route, link->network->routes_by_section) {
-                bool multipath_ok = true;
-                MultipathRoute *m;
+        link->static_routes_configured = false;
 
+        HASHMAP_FOREACH(route, link->network->routes_by_section) {
                 if (route->gateway_from_dhcp_or_ra)
                         continue;
 
-                if (route_has_gateway(route) != with_gateway)
+                if (only_ipv4 && route->family != AF_INET)
                         continue;
 
-                ORDERED_SET_FOREACH(m, route->multipath_routes) {
-                        if (isempty(m->ifname))
-                                continue;
-
-                        r = resolve_interface(&link->manager->rtnl, m->ifname);
-                        if (r < 0) {
-                                log_link_debug_errno(link, r,
-                                                     "Failed to resolve interface name '%s' for multipath route, "
-                                                     "ignoring the route: %m", m->ifname);
-                                multipath_ok = false;
-                                break;
-                        }
-
-                        m->ifindex = r;
-                }
-                if (!multipath_ok)
-                        continue;
-
-                r = route_configure(route, link, with_gateway ? route_handler_with_gateway : route_handler_without_gateway, NULL);
+                r = link_request_route(link, route, false, &link->static_route_messages,
+                                       static_route_handler, NULL);
                 if (r < 0)
-                        return log_link_warning_errno(link, r, "Could not set routes: %m");
-
-                link->route_messages++;
+                        return r;
         }
 
-        return 0;
-}
-
-int link_set_routes_with_gateway(Link *link) {
-        int r;
-
-        assert(link);
-        assert(link->network);
-
-        if (!link_has_carrier(link) && !link->network->configure_without_carrier)
-                /* During configuring addresses, the link lost its carrier. As networkd is dropping
-                 * the addresses now, let's not configure the routes either. */
-                return 0;
-
-        /* Finally, add routes that needs a gateway. */
-        r = link_set_routes_internal(link, true);
-        if (r < 0)
-                return r;
-
-        if (link->route_messages == 0) {
+        if (link->static_route_messages == 0) {
                 link->static_routes_configured = true;
                 link_check_ready(link);
         } else {
-                log_link_debug(link, "Setting routes with gateway");
+                log_link_debug(link, "Requesting routes");
                 link_set_state(link, LINK_STATE_CONFIGURING);
         }
 
         return 0;
 }
 
-int link_set_routes(Link *link) {
+static int route_is_ready_to_configure(const Route *route, Link *link) {
+        MultipathRoute *m;
+        NextHop *nh = NULL;
         int r;
 
+        assert(route);
         assert(link);
-        assert(link->network);
-        assert(link->state != _LINK_STATE_INVALID);
 
-        link->static_routes_configured = false;
+        if (route->nexthop_id > 0 &&
+            manager_get_nexthop_by_id(link->manager, route->nexthop_id, &nh) < 0)
+                return false;
 
-        if (!link->addresses_ready)
-                return 0;
+        if (route_type_is_reject(route) || (nh && nh->blackhole)) {
+                if (nh && link->manager->nexthop_remove_messages > 0)
+                        return false;
+                if (link->manager->route_remove_messages > 0)
+                        return false;
+        } else {
+                Link *l;
 
-        if (!link_has_carrier(link) && !link->network->configure_without_carrier)
-                /* During configuring addresses, the link lost its carrier. As networkd is dropping
-                 * the addresses now, let's not configure the routes either. */
-                return 0;
-
-        if (link->route_messages != 0) {
-                log_link_debug(link, "Static routes are configuring.");
-                return 0;
+                HASHMAP_FOREACH(l, link->manager->links) {
+                        if (l->address_remove_messages > 0)
+                                return false;
+                        if (l->nexthop_remove_messages > 0)
+                                return false;
+                        if (l->route_remove_messages > 0)
+                                return false;
+                }
         }
 
-        /* First, add the routes that enable us to talk to gateways. */
-        r = link_set_routes_internal(link, false);
+        if (in_addr_is_set(route->family, &route->prefsrc) > 0) {
+                r = manager_has_address(link->manager, route->family, &route->prefsrc, route->family == AF_INET6);
+                if (r <= 0)
+                        return r;
+        }
+
+        if (route->gateway_onlink <= 0 &&
+            in_addr_is_set(route->gw_family, &route->gw) > 0 &&
+            !manager_address_is_reachable(link->manager, route->gw_family, &route->gw))
+                return false;
+
+        ORDERED_SET_FOREACH(m, route->multipath_routes) {
+                union in_addr_union a = m->gateway.address;
+
+                if (route->gateway_onlink <= 0 &&
+                    !manager_address_is_reachable(link->manager, m->gateway.family, &a))
+                        return false;
+
+                if (m->ifname) {
+                        Link *l;
+
+                        r = resolve_interface(&link->manager->rtnl, m->ifname);
+                        if (r < 0)
+                                return false;
+                        m->ifindex = r;
+
+                        if (link_get(link->manager, m->ifindex, &l) < 0)
+                                return false;
+
+                        if (!link_is_ready_to_configure(l, true))
+                                return false;
+                }
+        }
+
+        return true;
+}
+
+int request_process_route(Request *req) {
+        Route *ret = NULL;
+        int r;
+
+        assert(req);
+        assert(req->link);
+        assert(req->route);
+        assert(req->type == REQUEST_TYPE_ROUTE);
+
+        if (!link_is_ready_to_configure(req->link, false))
+                return 0;
+
+        r = route_is_ready_to_configure(req->route, req->link);
+        if (r <= 0)
+                return r;
+
+        r = route_configure(req->route, req->link, req->netlink_handler,
+                            ordered_set_isempty(req->route->multipath_routes) ? &ret : NULL);
         if (r < 0)
                 return r;
 
-        if (link->route_messages == 0)
-                /* If no route is configured, then configure nexthops. */
-                return link_set_nexthops(link);
+        if (req->after_configure) {
+                r = req->after_configure(req, ret);
+                if (r < 0)
+                        return r;
+        }
 
-        log_link_debug(link, "Setting routes without gateway");
-        link_set_state(link, LINK_STATE_CONFIGURING);
-
-        return 0;
+        return 1;
 }
 
 static int process_route_one(Manager *manager, Link *link, uint16_t type, const Route *tmp, const MultipathRoute *m) {

src/network/networkd-route.h

@@ -3,7 +3,6 @@
 
 #include <inttypes.h>
 #include <stdbool.h>
-#include <stdio.h>
 
 #include "sd-netlink.h"
 
@@ -14,6 +13,7 @@
 
 typedef struct Manager Manager;
 typedef struct Network Network;
+typedef struct Request Request;
 
 typedef struct Route {
         Network *network;
@@ -74,21 +74,28 @@ DEFINE_NETWORK_SECTION_FUNCTIONS(Route, route_free);
 int route_dup(const Route *src, Route **ret);
 
 int route_configure_handler_internal(sd_netlink *rtnl, sd_netlink_message *m, Link *link, const char *error_msg);
-int route_configure(const Route *route, Link *link, link_netlink_message_handler_t callback, Route **ret);
 int link_route_remove_handler_internal(sd_netlink *rtnl, sd_netlink_message *m, Link *link, const char *error_msg);
 int route_remove(const Route *route, Manager *manager, Link *link, link_netlink_message_handler_t callback);
 
 int link_has_route(Link *link, const Route *route);
 bool manager_address_is_reachable(Manager *manager, int family, const union in_addr_union *address);
 
-int link_set_routes(Link *link);
-int link_set_routes_with_gateway(Link *link);
 int link_drop_routes(Link *link);
 int link_drop_foreign_routes(Link *link);
 
 uint32_t link_get_dhcp_route_table(const Link *link);
 uint32_t link_get_ipv6_accept_ra_route_table(const Link *link);
 
+int link_request_route(
+                Link *link,
+                Route *route,
+                bool consume_object,
+                unsigned *message_counter,
+                link_netlink_message_handler_t netlink_handler,
+                Request **ret);
+int link_request_static_routes(Link *link, bool only_ipv4);
+int request_process_route(Request *req);
+
 int manager_rtnl_process_route(sd_netlink *rtnl, sd_netlink_message *message, Manager *m);
 
 int network_add_ipv4ll_route(Network *network);