SymbX/systemd
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

core: add RestartRandomizedDelaySec= service option

Browse Source 
We already support exponential backoff for automatic restarts via
RestartSec=/RestartSteps=/RestartMaxDelaySec=, but there is no way to
randomize the restart delay. When many instances of a service fail at
the same time (e.g. because a shared resource briefly went away) they
are all restarted in lockstep, creating a thundering herd problem.

So this commit adds a simple `RestartRandomizedDelaySec=` service
option which is similar to the timer `RandomizedDelaySec=` and
adds a randomized restart delay.
This commit is contained in:
Michael Vogt
2026-06-20 12:31:28 +02:00
committed by Lennart Poettering
parent 23b0bd230b
commit 046de3caea
10 changed files with 134 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/core/dbus-service.c
+4
View File
@@ -358,6 +358,7 @@ const sd_bus_vtable bus_service_vtable[] = {
SD_BUS_PROPERTY("RestartUSec", "t", bus_property_get_usec, offsetof(Service, restart_usec), SD_BUS_VTABLE_PROPERTY_CONST),
SD_BUS_PROPERTY("RestartSteps", "u", bus_property_get_unsigned, offsetof(Service, restart_steps), SD_BUS_VTABLE_PROPERTY_CONST),
SD_BUS_PROPERTY("RestartMaxDelayUSec", "t", bus_property_get_usec, offsetof(Service, restart_max_delay_usec), SD_BUS_VTABLE_PROPERTY_CONST),
SD_BUS_PROPERTY("RestartRandomizedDelayUSec", "t", bus_property_get_usec, offsetof(Service, restart_randomized_delay_usec), SD_BUS_VTABLE_PROPERTY_CONST),
SD_BUS_PROPERTY("RestartUSecNext", "t", property_get_restart_usec_next, 0, 0),
SD_BUS_PROPERTY("TimeoutStartUSec", "t", bus_property_get_usec, offsetof(Service, timeout_start_usec), SD_BUS_VTABLE_PROPERTY_CONST),
SD_BUS_PROPERTY("TimeoutStopUSec", "t", bus_property_get_usec, offsetof(Service, timeout_stop_usec), SD_BUS_VTABLE_PROPERTY_CONST),
@@ -611,6 +612,9 @@ static int bus_service_set_transient_property(
if (streq(name, "RestartMaxDelayUSec"))
return bus_set_transient_usec(u, name, &s->restart_max_delay_usec, message, flags, reterr_error);
if (streq(name, "RestartRandomizedDelayUSec"))
return bus_set_transient_usec(u, name, &s->restart_randomized_delay_usec, message, flags, reterr_error);
if (streq(name, "TimeoutStartUSec")) {
r = bus_set_transient_usec(u, name, &s->timeout_start_usec, message, flags, reterr_error);
if (r >= 0 && !UNIT_WRITE_FLAGS_NOOP(flags))
src/core/load-fragment-gperf.gperf.in
+1
View File
@@ -453,6 +453,7 @@ Service.ExecStopPost, config_parse_exec,
Service.RestartSec, config_parse_sec, 0, offsetof(Service, restart_usec)
Service.RestartSteps, config_parse_unsigned, 0, offsetof(Service, restart_steps)
Service.RestartMaxDelaySec, config_parse_sec, 0, offsetof(Service, restart_max_delay_usec)
Service.RestartRandomizedDelaySec, config_parse_sec, 0, offsetof(Service, restart_randomized_delay_usec)
Service.TimeoutSec, config_parse_service_timeout, 0, 0
Service.TimeoutStartSec, config_parse_service_timeout, 0, 0
Service.TimeoutStopSec, config_parse_sec_fix_0, 0, offsetof(Service, timeout_stop_usec)
src/core/service.c
+28 -3
View File
@@ -400,6 +400,13 @@ usec_t service_restart_usec_next(const Service *s) {
(long double) (n_restarts_next - 1) / s->restart_steps));
}
static usec_t service_restart_usec_next_jittered(const Service *s) {
assert(s);
/* Single helper for the restart timer and the deadline reconstructed at coldplug so they can't drift */
return usec_add(service_restart_usec_next(s), s->restart_randomized_delay_chosen_usec);
}
static void service_extend_event_source_timeout(Service *s, sd_event_source *source, usec_t extended) {
usec_t current;
int r;
@@ -1080,6 +1087,11 @@ static int service_verify(Service *s) {
s->restart_usec = s->restart_max_delay_usec;
}
if (s->restart_randomized_delay_usec == USEC_INFINITY) {
log_unit_warning(UNIT(s), "RestartRandomizedDelaySec= cannot be infinity, ignoring.");
s->restart_randomized_delay_usec = 0;
}
if (s->refresh_on_reload_set && s->refresh_on_reload_flags != _SERVICE_REFRESH_ON_RELOAD_ALL) {
if (FLAGS_SET(s->refresh_on_reload_flags, SERVICE_RELOAD_EXTENSIONS))
service_can_reload_extensions(s, /* warn = */ true);
@@ -1388,6 +1400,7 @@ static void service_dump(Unit *u, FILE *f, const char *prefix) {
"%sRestartSec: %s\n"
"%sRestartSteps: %u\n"
"%sRestartMaxDelaySec: %s\n"
"%sRestartRandomizedDelaySec: %s\n"
"%sTimeoutStartSec: %s\n"
"%sTimeoutStopSec: %s\n"
"%sTimeoutStartFailureMode: %s\n"
@@ -1395,6 +1408,7 @@ static void service_dump(Unit *u, FILE *f, const char *prefix) {
prefix, FORMAT_TIMESPAN(s->restart_usec, USEC_PER_SEC),
prefix, s->restart_steps,
prefix, FORMAT_TIMESPAN(s->restart_max_delay_usec, USEC_PER_SEC),
prefix, FORMAT_TIMESPAN(s->restart_randomized_delay_usec, USEC_PER_SEC),
prefix, FORMAT_TIMESPAN(s->timeout_start_usec, USEC_PER_SEC),
prefix, FORMAT_TIMESPAN(s->timeout_stop_usec, USEC_PER_SEC),
prefix, service_timeout_failure_mode_to_string(s->timeout_start_failure_mode),
@@ -1741,7 +1755,8 @@ static usec_t service_coldplug_timeout(Service *s) {
return usec_add(UNIT(s)->state_change_timestamp.monotonic, service_timeout_abort_usec(s));
case SERVICE_AUTO_RESTART:
return usec_add(UNIT(s)->inactive_enter_timestamp.monotonic, service_restart_usec_next(s));
return usec_add(UNIT(s)->inactive_enter_timestamp.monotonic,
service_restart_usec_next_jittered(s));
case SERVICE_CLEANING:
return usec_add(UNIT(s)->state_change_timestamp.monotonic, s->exec_context.timeout_clean_usec);
@@ -2514,7 +2529,11 @@ static void service_enter_dead(Service *s, ServiceResult f, bool allow_restart)
if (s->restart_mode != SERVICE_RESTART_MODE_DIRECT)
service_set_state(s, restart_state);
restart_usec_next = service_restart_usec_next(s);
/* Do the randomized restart delay once and remember it so that it's stable across daemon-reload */
s->restart_randomized_delay_chosen_usec = s->restart_randomized_delay_usec > 0 ?
random_u64_range(s->restart_randomized_delay_usec) : 0;
restart_usec_next = service_restart_usec_next_jittered(s);
r = service_arm_timer(s, /* relative= */ true, restart_usec_next);
if (r < 0) {
@@ -2534,7 +2553,9 @@ static void service_enter_dead(Service *s, ServiceResult f, bool allow_restart)
log_unit_notice(UNIT(s), "Service dead, subsequent restarts will be executed with debug level logging.");
}
log_unit_debug(UNIT(s), "Next restart interval calculated as: %s", FORMAT_TIMESPAN(restart_usec_next, 0));
log_unit_debug(UNIT(s), "Next restart interval calculated as: %s (randomized delay: %s)",
FORMAT_TIMESPAN(restart_usec_next, 0),
FORMAT_TIMESPAN(s->restart_randomized_delay_chosen_usec, 0));
service_set_state(s, SERVICE_AUTO_RESTART);
} else {
@@ -3744,6 +3765,7 @@ static int service_serialize(Unit *u, FILE *f, FDSet *fds) {
(void) serialize_bool(f, "bus-name-good", s->bus_name_good);
(void) serialize_item_format(f, "n-restarts", "%u", s->n_restarts);
(void) serialize_usec(f, "restart-randomized-delay-chosen-usec", s->restart_randomized_delay_chosen_usec);
(void) serialize_bool(f, "forbid-restart", s->forbid_restart);
service_serialize_exec_command(u, f, s->control_command);
@@ -4197,6 +4219,9 @@ static int service_deserialize_item(Unit *u, const char *key, const char *value,
if (r < 0)
log_unit_debug_errno(u, r, "Failed to parse serialized restart counter '%s': %m", value);
} else if (streq(key, "restart-randomized-delay-chosen-usec")) {
(void) deserialize_usec(value, &s->restart_randomized_delay_chosen_usec);
} else if (streq(key, "forbid-restart")) {
r = parse_boolean(value);
if (r < 0)
src/core/service.h
+2
View File
@@ -155,6 +155,8 @@ typedef struct Service {
unsigned restart_steps;
usec_t restart_usec;
usec_t restart_max_delay_usec;
usec_t restart_randomized_delay_usec; /* configured upper bound for the randomized restart delay */
usec_t restart_randomized_delay_chosen_usec; /* the value actually picked for the pending auto-restart */
usec_t timeout_start_usec;
usec_t timeout_stop_usec;
usec_t timeout_abort_usec;
src/core/varlink-service.c
+1
View File
@@ -118,6 +118,7 @@ int service_context_build_json(sd_json_variant **ret, const char *name, void *us
JSON_BUILD_PAIR_FINITE_USEC("RestartUSec", s->restart_usec),
JSON_BUILD_PAIR_UNSIGNED_NON_ZERO("RestartSteps", s->restart_steps),
JSON_BUILD_PAIR_FINITE_USEC_NON_ZERO("RestartMaxDelayUSec", s->restart_max_delay_usec),
JSON_BUILD_PAIR_FINITE_USEC_NON_ZERO("RestartRandomizedDelayUSec", s->restart_randomized_delay_usec),
JSON_BUILD_PAIR_FINITE_USEC("TimeoutStartUSec", s->timeout_start_usec),
JSON_BUILD_PAIR_FINITE_USEC("TimeoutStopUSec", s->timeout_stop_usec),
JSON_BUILD_PAIR_ENUM("TimeoutStartFailureMode", service_timeout_failure_mode_to_string(s->timeout_start_failure_mode)),