diff --git a/src/core/manager.c b/src/core/manager.c
index 20ec7ba874..0359999784 100644
--- a/src/core/manager.c
+++ b/src/core/manager.c
@@ -1961,7 +1961,7 @@ bool manager_unit_cache_should_retry_load(Unit *u) {
 
         /* The cache has been updated since the last time we tried to load the unit. There might be new
          * fragment paths to read. */
-        if (u->manager->unit_cache_mtime > u->fragment_loadtime)
+        if (u->manager->unit_cache_mtime != u->fragment_not_found_time)
                 return true;
 
         /* The cache needs to be updated because there are modifications on disk. */
diff --git a/src/core/unit.c b/src/core/unit.c
index 0124451c78..e70831869e 100644
--- a/src/core/unit.c
+++ b/src/core/unit.c
@@ -1679,10 +1679,10 @@ fail:
                                                      UNIT_ERROR;
         u->load_error = r;
 
-        /* Record the last time we tried to load the unit, so that if the cache gets updated between now
-         * and the next time an attempt is made to load this unit, we know we need to check again. */
+        /* Record the timestamp on the cache, so that if the cache gets updated between now and the next time
+         * an attempt is made to load this unit, we know we need to check again. */
         if (u->load_state == UNIT_NOT_FOUND)
-                u->fragment_loadtime = now(CLOCK_REALTIME);
+                u->fragment_not_found_time = u->manager->unit_cache_mtime;
 
         unit_add_to_dbus_queue(u);
         unit_add_to_gc_queue(u);
diff --git a/src/core/unit.h b/src/core/unit.h
index e217cec219..a54d649cd3 100644
--- a/src/core/unit.h
+++ b/src/core/unit.h
@@ -136,7 +136,7 @@ typedef struct Unit {
         char *source_path; /* if converted, the source file */
         char **dropin_paths;
 
-        usec_t fragment_loadtime;
+        usec_t fragment_not_found_time;
         usec_t fragment_mtime;
         usec_t source_mtime;
         usec_t dropin_mtime;