diff --git a/src/network/networkd-address.c b/src/network/networkd-address.c
index 582069cf39..888ecdf260 100644
--- a/src/network/networkd-address.c
+++ b/src/network/networkd-address.c
@@ -1335,6 +1335,9 @@ int address_remove_and_cancel(Address *address, Link *link) {
         if (waiting || (address->link && address_exists(address)))
                 return address_remove(address, link);
 
+        /* Otherwise, the address should be already gone or not configured yet. */
+        ipv4acd_detach(link, address);
+
         return 0;
 }
 
diff --git a/src/network/networkd-link.c b/src/network/networkd-link.c
index fc61c3818e..64d1f1f6fe 100644
--- a/src/network/networkd-link.c
+++ b/src/network/networkd-link.c
@@ -1095,8 +1095,12 @@ int link_drop_requests(Link *link, NetworkConfigSource source) {
                         if (source >= 0 && address->source != source)
                                 continue;
 
-                        if (remove && address_get(link, address, /* ret= */ NULL) < 0)
-                                RET_GATHER(ret, address_remove(address, link));
+                        if (address_get(link, address, /* ret= */ NULL) < 0) {
+                                if (remove)
+                                        RET_GATHER(ret, address_remove(address, link));
+                                else
+                                        ipv4acd_detach(link, address);
+                        }
                         break;
                 }
                 case REQUEST_TYPE_NEIGHBOR: {