From c0b64eef37b2d9ab5c74d56c087e4743cfd217b8 Mon Sep 17 00:00:00 2001 From: Yaping Li <202858510+YapingLi04@users.noreply.github.com> Date: Thu, 26 Mar 2026 19:57:46 -0700 Subject: [PATCH 1/3] report: add manager-level metrics to varlink Metrics API Added these metrics: - JobsQueued: number of jobs currently queued - SystemState: overall system state (running, degraded, etc.) - UnitsByLoadStateTotal: unit counts broken down by load state - UnitsTotal: total number of units Also bump METRICS_MAX from 1024 to 4096 to accommodate the new per-unit metrics that are now collected. --- src/core/varlink-metrics.c | 109 ++++++++++++++++++++++++++++++++++++- src/report/report.c | 2 +- 2 files changed, 109 insertions(+), 2 deletions(-) diff --git a/src/core/varlink-metrics.c b/src/core/varlink-metrics.c index 00af452d77..68560387f3 100644 --- a/src/core/varlink-metrics.c +++ b/src/core/varlink-metrics.c @@ -108,7 +108,7 @@ static int units_by_type_total_build_json(MetricFamilyContext *context, void *us static int units_by_state_total_build_json(MetricFamilyContext *context, void *userdata) { Manager *manager = ASSERT_PTR(userdata); - UnitActiveState counters[_UNIT_ACTIVE_STATE_MAX] = {}; + uint64_t counters[_UNIT_ACTIVE_STATE_MAX] = {}; Unit *unit; char *key; int r; @@ -143,14 +143,109 @@ static int units_by_state_total_build_json(MetricFamilyContext *context, void *u return 0; } +static int jobs_queued_build_json(MetricFamilyContext *context, void *userdata) { + Manager *manager = ASSERT_PTR(userdata); + + assert(context); + + return metric_build_send_unsigned( + context, + /* object= */ NULL, + hashmap_size(manager->jobs), + /* fields= */ NULL); +} + +static int system_state_build_json(MetricFamilyContext *context, void *userdata) { + Manager *manager = ASSERT_PTR(userdata); + + assert(context); + + return metric_build_send_string( + context, + /* object= */ NULL, + manager_state_to_string(manager_state(manager)), + /* fields= */ NULL); +} + +static int units_by_load_state_total_build_json(MetricFamilyContext *context, void *userdata) { + Manager *manager = ASSERT_PTR(userdata); + uint64_t counters[_UNIT_LOAD_STATE_MAX] = {}; + Unit *unit; + char *key; + int r; + + assert(context); + + HASHMAP_FOREACH_KEY(unit, key, manager->units) { + /* ignore aliases */ + if (key != unit->id) + continue; + + counters[unit->load_state]++; + } + + for (UnitLoadState state = 0; state < _UNIT_LOAD_STATE_MAX; state++) { + _cleanup_(sd_json_variant_unrefp) sd_json_variant *fields = NULL; + + r = sd_json_buildo(&fields, SD_JSON_BUILD_PAIR_STRING("load_state", unit_load_state_to_string(state))); + if (r < 0) + return r; + + r = metric_build_send_unsigned( + context, + /* object= */ NULL, + counters[state], + fields); + if (r < 0) + return r; + } + + return 0; +} + +static int units_total_build_json(MetricFamilyContext *context, void *userdata) { + Manager *manager = ASSERT_PTR(userdata); + uint64_t count = 0; + Unit *unit; + char *key; + + assert(context); + + HASHMAP_FOREACH_KEY(unit, key, manager->units) { + /* ignore aliases */ + if (key != unit->id) + continue; + + count++; + } + + return metric_build_send_unsigned( + context, + /* object= */ NULL, + count, + /* fields= */ NULL); +} + static const MetricFamily metric_family_table[] = { /* Keep metrics ordered alphabetically */ + { + .name = METRIC_IO_SYSTEMD_MANAGER_PREFIX "JobsQueued", + .description = "Number of jobs currently queued", + .type = METRIC_FAMILY_TYPE_GAUGE, + .generate = jobs_queued_build_json, + }, { .name = METRIC_IO_SYSTEMD_MANAGER_PREFIX "NRestarts", .description = "Per unit metric: number of restarts", .type = METRIC_FAMILY_TYPE_COUNTER, .generate = nrestarts_build_json, }, + { + .name = METRIC_IO_SYSTEMD_MANAGER_PREFIX "SystemState", + .description = "Overall system state", + .type = METRIC_FAMILY_TYPE_STRING, + .generate = system_state_build_json, + }, { .name = METRIC_IO_SYSTEMD_MANAGER_PREFIX "UnitActiveState", .description = "Per unit metric: active state", @@ -163,6 +258,12 @@ static const MetricFamily metric_family_table[] = { .type = METRIC_FAMILY_TYPE_STRING, .generate = unit_load_state_build_json, }, + { + .name = METRIC_IO_SYSTEMD_MANAGER_PREFIX "UnitsByLoadStateTotal", + .description = "Total number of units by load state", + .type = METRIC_FAMILY_TYPE_GAUGE, + .generate = units_by_load_state_total_build_json, + }, { .name = METRIC_IO_SYSTEMD_MANAGER_PREFIX "UnitsByStateTotal", .description = "Total number of units of different state", @@ -175,6 +276,12 @@ static const MetricFamily metric_family_table[] = { .type = METRIC_FAMILY_TYPE_GAUGE, .generate = units_by_type_total_build_json, }, + { + .name = METRIC_IO_SYSTEMD_MANAGER_PREFIX "UnitsTotal", + .description = "Total number of units", + .type = METRIC_FAMILY_TYPE_GAUGE, + .generate = units_total_build_json, + }, {} }; diff --git a/src/report/report.c b/src/report/report.c index 108b24e470..ca169c94a8 100644 --- a/src/report/report.c +++ b/src/report/report.c @@ -27,7 +27,7 @@ #include "varlink-idl-util.h" #include "verbs.h" -#define METRICS_OR_FACTS_MAX 1024U +#define METRICS_OR_FACTS_MAX 4096U #define METRICS_OR_FACTS_LINKS_MAX 128U #define TIMEOUT_USEC (30 * USEC_PER_SEC) /* 30 seconds */ From 4ec0520d9a06a9f2a162df22103d0117ded3b4ad Mon Sep 17 00:00:00 2001 From: Yaping Li <202858510+YapingLi04@users.noreply.github.com> Date: Thu, 19 Mar 2026 14:10:49 -0700 Subject: [PATCH 2/3] report: add per-service metrics to the varlink Metrics API Added these metrics: - ActiveTimestamp: active state transition timestamps (enter/exit) - InactiveExitTimestamp: when the unit last left inactive state - NRestarts: restart count - StateChangeTimestamp: last state change timestamp - StatusErrno: service errno status Per-service cgroup metrics (CpuUsage, MemoryUsage, IOReadBytes, IOReadOperations, TasksCurrent) are not included here as they are gathered by the kernel and will be served by a separate process that reads cgroup files directly, minimizing PID1 involvement. --- src/core/varlink-metrics.c | 136 +++++++++++++++++++++++++++++++++++++ 1 file changed, 136 insertions(+) diff --git a/src/core/varlink-metrics.c b/src/core/varlink-metrics.c index 68560387f3..82bc3cf4cb 100644 --- a/src/core/varlink-metrics.c +++ b/src/core/varlink-metrics.c @@ -11,6 +11,118 @@ #include "unit.h" #include "varlink-metrics.h" +static int active_timestamp_build_json(MetricFamilyContext *context, void *userdata) { + Manager *manager = ASSERT_PTR(userdata); + Unit *unit; + char *key; + int r; + + assert(context); + + _cleanup_(sd_json_variant_unrefp) sd_json_variant *enter_fields = NULL; + r = sd_json_buildo(&enter_fields, SD_JSON_BUILD_PAIR_STRING("event", "enter")); + if (r < 0) + return r; + + _cleanup_(sd_json_variant_unrefp) sd_json_variant *exit_fields = NULL; + r = sd_json_buildo(&exit_fields, SD_JSON_BUILD_PAIR_STRING("event", "exit")); + if (r < 0) + return r; + + HASHMAP_FOREACH_KEY(unit, key, manager->units) { + /* ignore aliases */ + if (key != unit->id) + continue; + + r = metric_build_send_unsigned( + context, + unit->id, + unit->active_enter_timestamp.realtime, + enter_fields); + if (r < 0) + return r; + + r = metric_build_send_unsigned( + context, + unit->id, + unit->active_exit_timestamp.realtime, + exit_fields); + if (r < 0) + return r; + } + + return 0; +} + +static int inactive_exit_timestamp_build_json(MetricFamilyContext *context, void *userdata) { + Manager *manager = ASSERT_PTR(userdata); + Unit *unit; + char *key; + int r; + + assert(context); + + HASHMAP_FOREACH_KEY(unit, key, manager->units) { + /* ignore aliases */ + if (key != unit->id) + continue; + + r = metric_build_send_unsigned( + context, + unit->id, + unit->inactive_exit_timestamp.realtime, + /* fields= */ NULL); + if (r < 0) + return r; + } + + return 0; +} + +static int state_change_timestamp_build_json(MetricFamilyContext *context, void *userdata) { + Manager *manager = ASSERT_PTR(userdata); + Unit *unit; + char *key; + int r; + + assert(context); + + HASHMAP_FOREACH_KEY(unit, key, manager->units) { + /* ignore aliases */ + if (key != unit->id) + continue; + + r = metric_build_send_unsigned( + context, + unit->id, + unit->state_change_timestamp.realtime, + /* fields= */ NULL); + if (r < 0) + return r; + } + + return 0; +} + +static int status_errno_build_json(MetricFamilyContext *context, void *userdata) { + Manager *manager = ASSERT_PTR(userdata); + int r; + + assert(context); + + LIST_FOREACH(units_by_type, unit, manager->units_by_type[UNIT_SERVICE]) { + r = metric_build_send_unsigned( + context, + unit->id, + (uint64_t) SERVICE(unit)->status_errno, + /* fields= */ NULL); + if (r < 0) + return r; + } + + return 0; +} + static int unit_active_state_build_json(MetricFamilyContext *context, void *userdata) { Manager *manager = ASSERT_PTR(userdata); Unit *unit; @@ -228,6 +340,18 @@ static int units_total_build_json(MetricFamilyContext *context, void *userdata) static const MetricFamily metric_family_table[] = { /* Keep metrics ordered alphabetically */ + { + .name = METRIC_IO_SYSTEMD_MANAGER_PREFIX "ActiveTimestamp", + .description = "Per unit metric: timestamp of active state transitions in microseconds; 0 indicates the transition has not occurred", + .type = METRIC_FAMILY_TYPE_GAUGE, + .generate = active_timestamp_build_json, + }, + { + .name = METRIC_IO_SYSTEMD_MANAGER_PREFIX "InactiveExitTimestamp", + .description = "Per unit metric: timestamp when the unit last exited the inactive state in microseconds; 0 indicates the transition has not occurred", + .type = METRIC_FAMILY_TYPE_GAUGE, + .generate = inactive_exit_timestamp_build_json, + }, { .name = METRIC_IO_SYSTEMD_MANAGER_PREFIX "JobsQueued", .description = "Number of jobs currently queued", @@ -240,6 +364,18 @@ static const MetricFamily metric_family_table[] = { .type = METRIC_FAMILY_TYPE_COUNTER, .generate = nrestarts_build_json, }, + { + .name = METRIC_IO_SYSTEMD_MANAGER_PREFIX "StateChangeTimestamp", + .description = "Per unit metric: timestamp of the last state change in microseconds; 0 indicates no state change has occurred", + .type = METRIC_FAMILY_TYPE_GAUGE, + .generate = state_change_timestamp_build_json, + }, + { + .name = METRIC_IO_SYSTEMD_MANAGER_PREFIX "StatusErrno", + .description = "Per service metric: errno status of the service", + .type = METRIC_FAMILY_TYPE_GAUGE, + .generate = status_errno_build_json, + }, { .name = METRIC_IO_SYSTEMD_MANAGER_PREFIX "SystemState", .description = "Overall system state", From 39ce1ee990fa9a703d557592c2a79753c723e8cc Mon Sep 17 00:00:00 2001 From: Yaping Li <202858510+YapingLi04@users.noreply.github.com> Date: Thu, 19 Mar 2026 14:10:59 -0700 Subject: [PATCH 3/3] report: follow up to review comments on #40619 Address review comments on https://github.com/systemd/systemd/pull/40619#discussion_r2810892362 --- src/network/networkd-varlink-metrics.c | 57 ++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/src/network/networkd-varlink-metrics.c b/src/network/networkd-varlink-metrics.c index d37f65b43d..50aacebbf8 100644 --- a/src/network/networkd-varlink-metrics.c +++ b/src/network/networkd-varlink-metrics.c @@ -1,7 +1,9 @@ /* SPDX-License-Identifier: LGPL-2.1-or-later */ +#include "sd-json.h" #include "sd-varlink.h" +#include "alloc-util.h" #include "argv-util.h" #include "errno-util.h" #include "fd-util.h" @@ -99,6 +101,55 @@ static int managed_interfaces_build_json(MetricFamilyContext *context, void *use return metric_build_send_unsigned(context, /* object= */ NULL, count, /* fields= */ NULL); } +static int required_for_online_build_json(MetricFamilyContext *context, void *userdata) { + Manager *manager = ASSERT_PTR(userdata); + Link *link; + int r; + + assert(context); + + HASHMAP_FOREACH(link, manager->links_by_index) { + if (!link->network) + continue; + + if (link->network->required_for_online == 0) { + r = metric_build_send_string( + context, + link->ifname, + "no", + /* fields= */ NULL); + } else { + LinkOperationalStateRange range; + link_required_operstate_for_online(link, &range); + + const char *min_str = link_operstate_to_string(range.min); + const char *max_str = link_operstate_to_string(range.max); + + if (range.min == range.max) + r = metric_build_send_string( + context, + link->ifname, + min_str, + /* fields= */ NULL); + else { + _cleanup_free_ char *value = NULL; + if (asprintf(&value, "%s:%s", min_str, max_str) < 0) + return -ENOMEM; + + r = metric_build_send_string( + context, + link->ifname, + value, + /* fields= */ NULL); + } + } + if (r < 0) + return r; + } + + return 0; +} + /* Keep metrics ordered alphabetically */ static const MetricFamily network_metric_family_table[] = { { @@ -143,6 +194,12 @@ static const MetricFamily network_metric_family_table[] = { .type = METRIC_FAMILY_TYPE_STRING, .generate = link_oper_state_build_json, }, + { + .name = METRIC_IO_SYSTEMD_NETWORK_PREFIX "RequiredForOnline", + .description = "Per interface metric: required operational state for online, or 'no' if not required", + .type = METRIC_FAMILY_TYPE_STRING, + .generate = required_for_online_build_json, + }, {} };