report: add per-service metrics to the varlink Metrics API

Added these metrics:

  - ActiveTimestamp: active state transition timestamps (enter/exit)
  - InactiveExitTimestamp: when the unit last left inactive state
  - NRestarts: restart count
  - StateChangeTimestamp: last state change timestamp
  - StatusErrno: service errno status

Per-service cgroup metrics (CpuUsage, MemoryUsage, IOReadBytes,
IOReadOperations, TasksCurrent) are not included here as they are
gathered by the kernel and will be served by a separate process that
reads cgroup files directly, minimizing PID1 involvement.

src/core/varlink-metrics.c

@@ -11,6 +11,118 @@
 #include "unit.h"
 #include "varlink-metrics.h"
 
+static int active_timestamp_build_json(MetricFamilyContext *context, void *userdata) {
+        Manager *manager = ASSERT_PTR(userdata);
+        Unit *unit;
+        char *key;
+        int r;
+
+        assert(context);
+
+        _cleanup_(sd_json_variant_unrefp) sd_json_variant *enter_fields = NULL;
+        r = sd_json_buildo(&enter_fields, SD_JSON_BUILD_PAIR_STRING("event", "enter"));
+        if (r < 0)
+                return r;
+
+        _cleanup_(sd_json_variant_unrefp) sd_json_variant *exit_fields = NULL;
+        r = sd_json_buildo(&exit_fields, SD_JSON_BUILD_PAIR_STRING("event", "exit"));
+        if (r < 0)
+                return r;
+
+        HASHMAP_FOREACH_KEY(unit, key, manager->units) {
+                /* ignore aliases */
+                if (key != unit->id)
+                        continue;
+
+                r = metric_build_send_unsigned(
+                                context,
+                                unit->id,
+                                unit->active_enter_timestamp.realtime,
+                                enter_fields);
+                if (r < 0)
+                        return r;
+
+                r = metric_build_send_unsigned(
+                                context,
+                                unit->id,
+                                unit->active_exit_timestamp.realtime,
+                                exit_fields);
+                if (r < 0)
+                        return r;
+        }
+
+        return 0;
+}
+
+static int inactive_exit_timestamp_build_json(MetricFamilyContext *context, void *userdata) {
+        Manager *manager = ASSERT_PTR(userdata);
+        Unit *unit;
+        char *key;
+        int r;
+
+        assert(context);
+
+        HASHMAP_FOREACH_KEY(unit, key, manager->units) {
+                /* ignore aliases */
+                if (key != unit->id)
+                        continue;
+
+                r = metric_build_send_unsigned(
+                                context,
+                                unit->id,
+                                unit->inactive_exit_timestamp.realtime,
+                                /* fields= */ NULL);
+                if (r < 0)
+                        return r;
+        }
+
+        return 0;
+}
+
+static int state_change_timestamp_build_json(MetricFamilyContext *context, void *userdata) {
+        Manager *manager = ASSERT_PTR(userdata);
+        Unit *unit;
+        char *key;
+        int r;
+
+        assert(context);
+
+        HASHMAP_FOREACH_KEY(unit, key, manager->units) {
+                /* ignore aliases */
+                if (key != unit->id)
+                        continue;
+
+                r = metric_build_send_unsigned(
+                                context,
+                                unit->id,
+                                unit->state_change_timestamp.realtime,
+                                /* fields= */ NULL);
+                if (r < 0)
+                        return r;
+        }
+
+        return 0;
+}
+
+static int status_errno_build_json(MetricFamilyContext *context, void *userdata) {
+        Manager *manager = ASSERT_PTR(userdata);
+        int r;
+
+        assert(context);
+
+        LIST_FOREACH(units_by_type, unit, manager->units_by_type[UNIT_SERVICE]) {
+                r = metric_build_send_unsigned(
+                                context,
+                                unit->id,
+                                (uint64_t) SERVICE(unit)->status_errno,
+                                /* fields= */ NULL);
+                if (r < 0)
+                        return r;
+        }
+
+        return 0;
+}
+
 static int unit_active_state_build_json(MetricFamilyContext *context, void *userdata) {
         Manager *manager = ASSERT_PTR(userdata);
         Unit *unit;
@@ -228,6 +340,18 @@ static int units_total_build_json(MetricFamilyContext *context, void *userdata)
 
 static const MetricFamily metric_family_table[] = {
         /* Keep metrics ordered alphabetically */
+        {
+                .name = METRIC_IO_SYSTEMD_MANAGER_PREFIX "ActiveTimestamp",
+                .description = "Per unit metric: timestamp of active state transitions in microseconds; 0 indicates the transition has not occurred",
+                .type = METRIC_FAMILY_TYPE_GAUGE,
+                .generate = active_timestamp_build_json,
+        },
+        {
+                .name = METRIC_IO_SYSTEMD_MANAGER_PREFIX "InactiveExitTimestamp",
+                .description = "Per unit metric: timestamp when the unit last exited the inactive state in microseconds; 0 indicates the transition has not occurred",
+                .type = METRIC_FAMILY_TYPE_GAUGE,
+                .generate = inactive_exit_timestamp_build_json,
+        },
         {
                 .name = METRIC_IO_SYSTEMD_MANAGER_PREFIX "JobsQueued",
                 .description = "Number of jobs currently queued",
@@ -240,6 +364,18 @@ static const MetricFamily metric_family_table[] = {
                 .type = METRIC_FAMILY_TYPE_COUNTER,
                 .generate = nrestarts_build_json,
         },
+        {
+                .name = METRIC_IO_SYSTEMD_MANAGER_PREFIX "StateChangeTimestamp",
+                .description = "Per unit metric: timestamp of the last state change in microseconds; 0 indicates no state change has occurred",
+                .type = METRIC_FAMILY_TYPE_GAUGE,
+                .generate = state_change_timestamp_build_json,
+        },
+        {
+                .name = METRIC_IO_SYSTEMD_MANAGER_PREFIX "StatusErrno",
+                .description = "Per service metric: errno status of the service",
+                .type = METRIC_FAMILY_TYPE_GAUGE,
+                .generate = status_errno_build_json,
+        },
         {
                 .name = METRIC_IO_SYSTEMD_MANAGER_PREFIX "SystemState",
                 .description = "Overall system state",