report-basic: cpu, memory, pressure, and disk metrics

Extend the io.systemd.Basic metric families with host-level runtime
metrics: cumulative per-state CPU seconds, memory in use (MemTotal minus
MemAvailable), swap in use, the kernel's 10s pressure stall averages
(cpu/memory/io x some/full). Partitions and stacked devices are excluded
from the disk counters, since their traffic is already accounted for in
the parent device.

Also extend the TEST-74 report subtest with checks for the new metrics.

Network metrics are not added, because the service runs with
PrivateNetwork=yes, and it doesn't make sense to report stats for 'lo'.

After some back-and-forth, I settled on "CPUUsage". We use this scheme of
capitalization in other metric family names: "IPv4Public", "IPv6Public",
IPv4AddressState", "IPv6AddressState", "OSRelease" "SMBIOS" "BootID",
"CPUsOnline", "MachineID", "MachineInfo", "OSRelease",
"SMBIOS.SystemManufacturer", "TPM2.Manufacturer", "TPM2.VendorString",
"IOReadBytes", "IOReadOperations", "CGroup". The exception is
"io.systemd.cgroup.CpuUsage". I think we should treat that one as a
mistake and consider changing it to conform.

src/basic/procfs-util.c

@@ -127,18 +127,8 @@ static uint64_t calc_gcd64(uint64_t a, uint64_t b) {
         return a;
 }
 
-int procfs_cpu_get_usage(nsec_t *ret) {
-        ProcfsCpuTicks ticks;
-        int r;
-
-        assert(ret);
-
-        r = procfs_cpu_get_ticks(&ticks);
-        if (r < 0)
-                return r;
-
+nsec_t procfs_ticks_to_nsec(uint64_t ticks) {
         uint64_t ticks_per_second = sysconf_clock_ticks_cached(),
-                sum = ticks.user + ticks.nice + ticks.system + ticks.irq + ticks.softirq,
                 gcd = calc_gcd64(NSEC_PER_SEC, ticks_per_second);
 
         /* Reduce NSEC_PER_SEC/ticks_per_second first to avoid overflow of ticks * NSEC_PER_SEC, and split
@@ -148,7 +138,20 @@ int procfs_cpu_get_usage(nsec_t *ret) {
         uint64_t a = NSEC_PER_SEC / gcd,
                  b = ticks_per_second / gcd;
 
-        *ret = (nsec_t) (sum / b * a + DIV_ROUND_UP(sum % b * a, b));
+        return ticks / b * a + DIV_ROUND_UP(ticks % b * a, b);
+}
+
+int procfs_cpu_get_usage(nsec_t *ret) {
+        int r;
+
+        assert(ret);
+
+        ProcfsCpuTicks ticks;
+        r = procfs_cpu_get_ticks(&ticks);
+        if (r < 0)
+                return r;
+
+        *ret = procfs_ticks_to_nsec(ticks.user + ticks.nice + ticks.system + ticks.irq + ticks.softirq);
         return 0;
 }
 

src/basic/procfs-util.h

@@ -9,6 +9,7 @@ int procfs_get_threads_max(uint64_t *ret);
 int procfs_tasks_set_limit(uint64_t limit);
 int procfs_tasks_get_current(uint64_t *ret);
 
+nsec_t procfs_ticks_to_nsec(uint64_t ticks);
 int procfs_cpu_get_usage(nsec_t *ret);
 
 typedef struct ProcfsCpuTicks {

src/report/report-basic.c

@@ -11,6 +11,7 @@
 
 #include "alloc-util.h"
 #include "architecture.h"
+#include "blockdev-list.h"
 #include "confidential-virt.h"
 #include "cpu-set-util.h"
 #include "env-file.h"
@@ -23,8 +24,13 @@
 #include "log.h"
 #include "metrics.h"
 #include "os-util.h"
+#include "path-util.h"
+#include "procfs-util.h"
+#include "psi-util.h"
 #include "report-basic.h"
 #include "string-util.h"
+#include "time-util.h"
+#include "utf8.h"
 #include "virt.h"
 
 static int architecture_generate(const MetricFamily *mf, sd_varlink *link, void *userdata) {
@@ -124,6 +130,94 @@ static int physical_memory_generate(const MetricFamily *mf, sd_varlink *link, vo
                         /* fields= */ NULL);
 }
 
+static int memory_used_generate(const MetricFamily *mf, sd_varlink *link, void *userdata) {
+        int r;
+
+        assert(mf && mf->name);
+        assert(link);
+
+        /* Memory currently in use, i.e. MemTotal − MemAvailable from /proc/meminfo. Note that
+         * PhysicalMemoryBytes is not necessarily the matching total: it is capped by the root cgroup's
+         * memory limit, while /proc/meminfo always shows the whole system. */
+
+        uint64_t used;
+        r = procfs_memory_get_used(&used);
+        if (r < 0) {
+                log_warning_errno(r, "Failed to read memory usage from /proc/meminfo, ignoring: %m");
+                return 0;
+        }
+
+        return metric_build_send_unsigned(
+                        mf,
+                        link,
+                        /* object= */ NULL,
+                        used,
+                        /* fields= */ NULL);
+}
+
+static int pressure_generate(const MetricFamily mf[static 2], sd_varlink *link, void *userdata) {
+        int r;
+
+        assert(mf && mf[0].name && mf[1].name);
+        assert(!mf[1].generate);
+        assert(link);
+
+        /* The kernel's smoothed pressure stall percentages over the last 10s, plus the cumulative stall
+         * time, as documented in https://docs.kernel.org/accounting/psi.html */
+
+        for (PressureResource resource = 0; resource < _PRESSURE_RESOURCE_MAX; resource++) {
+                const char *name = pressure_resource_to_string(resource);
+
+                _cleanup_free_ char *path = path_join("/proc/pressure", name);
+                if (!path)
+                        return -ENOMEM;
+
+                PressureType type;
+                FOREACH_ARGUMENT(type, PRESSURE_TYPE_SOME, PRESSURE_TYPE_FULL) {
+                        /* The "full" number for cpu pressure at the system level is always reported as 0,
+                         * so skip it. */
+                        if (resource == PRESSURE_CPU && type == PRESSURE_TYPE_FULL)
+                                continue;
+
+                        ResourcePressure rp;
+                        r = read_resource_pressure(path, type, &rp);
+                        if (r < 0) {
+                                bool absent = IN_SET(r, -ENOENT, -ENODATA) || ERRNO_IS_NEG_NOT_SUPPORTED(r);
+                                log_full_errno(absent ? LOG_DEBUG : LOG_WARNING, r,
+                                               "Failed to read %s pressure, ignoring: %m", name);
+                                break; /* If "some" can't be read, "full" won't work either */
+                        }
+
+                        _cleanup_(sd_json_variant_unrefp) sd_json_variant *fields = NULL;
+                        r = sd_json_buildo(&fields,
+                                           SD_JSON_BUILD_PAIR_STRING("resource", name),
+                                           SD_JSON_BUILD_PAIR_STRING("type", pressure_type_to_string(type)));
+                        if (r < 0)
+                                return r;
+
+                        r = metric_build_send_double(
+                                        mf,
+                                        link,
+                                        /* object= */ NULL,
+                                        (double) rp.avg10 / LOADAVG_FIXED_POINT_1_0,
+                                        fields);
+                        if (r < 0)
+                                return r;
+
+                        r = metric_build_send_double(
+                                        mf + 1,
+                                        link,
+                                        /* object= */ NULL,
+                                        (double) rp.total / USEC_PER_SEC,
+                                        fields);
+                        if (r < 0)
+                                return r;
+                }
+        }
+
+        return 0;
+}
+
 static int cpus_online_generate(const MetricFamily *mf, sd_varlink *link, void *userdata) {
         int r;
 
@@ -143,6 +237,52 @@ static int cpus_online_generate(const MetricFamily *mf, sd_varlink *link, void *
                         /* fields= */ NULL);
 }
 
+static int cpu_usage_generate(const MetricFamily *mf, sd_varlink *link, void *userdata) {
+        int r;
+
+        assert(mf && mf->name);
+        assert(link);
+
+        /* The aggregate "cpu" line of /proc/stat, converted from USER_HZ ticks to nanoseconds, matching
+         * the unit of io.systemd.CGroup.CpuUsage. "nice" time is folded into "user" and "irq"/"softirq"
+         * into "system", so that the published types together account for all CPU time. */
+
+        ProcfsCpuTicks ticks;
+        r = procfs_cpu_get_ticks(&ticks);
+        if (r < 0) {
+                log_warning_errno(r, "Failed to read CPU time from /proc/stat, ignoring: %m");
+                return 0;
+        }
+
+        static const char* const types[] = { "user", "system" };
+        const uint64_t values[] = {
+                ticks.user + ticks.nice,
+                ticks.system + ticks.irq + ticks.softirq,
+        };
+        assert_cc(ELEMENTSOF(types) == ELEMENTSOF(values));
+
+        for (size_t i = 0; i < ELEMENTSOF(values); i++) {
+                _cleanup_(sd_json_variant_unrefp) sd_json_variant *fields = NULL;
+
+                r = sd_json_buildo(&fields, SD_JSON_BUILD_PAIR_STRING("type", types[i]));
+                if (r < 0)
+                        return r;
+
+                uint64_t nsec = procfs_ticks_to_nsec(values[i]);
+
+                r = metric_build_send_unsigned(
+                                mf,
+                                link,
+                                /* object= */ NULL,
+                                nsec,
+                                fields);
+                if (r < 0)
+                        return r;
+        }
+
+        return 0;
+}
+
 static int load_average_generate(const MetricFamily *mf, sd_varlink *link, void *userdata) {
         enum {
                 LOAD_AVERAGE_FIELD_1MIN,
@@ -181,11 +321,14 @@ static int load_average_generate(const MetricFamily *mf, sd_varlink *link, void
         return 0;
 }
 
-static int swap_generate(const MetricFamily *mf, sd_varlink *link, void *userdata) {
-        assert(mf && mf->name);
+static int swap_generate(const MetricFamily mf[static 2], sd_varlink *link, void *userdata) {
+        int r;
+
+        assert(mf && mf[0].name && mf[1].name);
+        assert(!mf[1].generate);
         assert(link);
 
-        /* The total amount of configured swap space, in bytes. */
+        /* The total amount of configured swap space and the amount currently in use, in bytes. */
 
         struct sysinfo info;
         if (sysinfo(&info) < 0)
@@ -193,15 +336,69 @@ static int swap_generate(const MetricFamily *mf, sd_varlink *link, void *userdat
 
         /* Overflow is unrealistic (would need >16 EiB of swap), but use MUL_SAFE to make this obvious to
          * static analyzers. */
-        uint64_t swap;
-        assert_se(MUL_SAFE(&swap, (uint64_t) info.totalswap, (uint64_t) info.mem_unit));
+        uint64_t total, used;
+        assert_se(MUL_SAFE(&total, (uint64_t) info.totalswap, (uint64_t) info.mem_unit));
+        assert_se(MUL_SAFE(&used, (uint64_t) LESS_BY(info.totalswap, info.freeswap), (uint64_t) info.mem_unit));
 
-        return metric_build_send_unsigned(
-                        mf,
-                        link,
-                        /* object= */ NULL,
-                        swap,
-                        /* fields= */ NULL);
+        const uint64_t values[] = { total, used };
+        for (size_t i = 0; i < ELEMENTSOF(values); i++) {
+                r = metric_build_send_unsigned(
+                                mf + i,
+                                link,
+                                /* object= */ NULL,
+                                values[i],
+                                /* fields= */ NULL);
+                if (r < 0)
+                        return r;
+        }
+
+        return 0;
+}
+
+static int disk_io_generate(const MetricFamily mf[static 2], sd_varlink *link, void *userdata) {
+        int r;
+
+        assert(mf && mf[0].name && mf[1].name);
+        assert(!mf[1].generate);
+        assert(link);
+
+        /* Cumulative read/write byte counters of all whole physical block devices. Partitions are skipped,
+         * their traffic is already accounted for in the parent device. Stacked devices (dm, md, …) are
+         * skipped for the same reason: their traffic is accounted for again in the devices they are built
+         * on. */
+
+        BlockDevice *devices = NULL;
+        size_t n_devices = 0;
+        CLEANUP_ARRAY(devices, n_devices, block_device_array_free);
+
+        r = blockdev_list(BLOCKDEV_LIST_IGNORE_VIRTUAL | BLOCKDEV_LIST_IGNORE_PARTITIONS | BLOCKDEV_LIST_RW_STATS,
+                          &devices, &n_devices);
+        if (r < 0)
+                return r;
+
+        FOREACH_ARRAY(d, devices, n_devices) {
+                /* A non-UTF-8 name would make the JSON reply builder fail, taking down the whole method
+                 * call rather than just this device. */
+                if (!utf8_is_valid(d->node)) {
+                        log_warning("%s: block device name is not valid UTF-8, ignoring device.", d->node);
+                        continue;
+                }
+
+                const uint64_t values[] = { d->read_bytes, d->write_bytes };
+                for (size_t i = 0; i < ELEMENTSOF(values); i++)
+                        if (values[i] != UINT64_MAX) {
+                                r = metric_build_send_unsigned(
+                                                mf + i,
+                                                link,
+                                                /* object= */ d->node,
+                                                values[i],
+                                                /* fields= */ NULL);
+                                if (r < 0)
+                                        return r;
+                        }
+        }
+
+        return 0;
 }
 
 enum {
@@ -547,6 +744,26 @@ static const MetricFamily metric_family_table[] = {
                 METRIC_FAMILY_TYPE_GAUGE,
                 .generate = cpus_online_generate,
         },
+        {
+                METRIC_IO_SYSTEMD_BASIC_PREFIX "CPUUsage",
+                "Aggregate CPU usage across all CPUs in nanoseconds (type=user|system)",
+                METRIC_FAMILY_TYPE_COUNTER,
+                .generate = cpu_usage_generate,
+        },
+        {
+                METRIC_IO_SYSTEMD_BASIC_PREFIX "DiskReadBytes",
+                "Per block device metric: cumulative number of bytes read "
+                "(partitions, stacked and virtual devices are excluded)",
+                METRIC_FAMILY_TYPE_COUNTER,
+                .generate = disk_io_generate,
+        },
+        {
+                METRIC_IO_SYSTEMD_BASIC_PREFIX "DiskWriteBytes",
+                "Per block device metric: cumulative number of bytes written "
+                "(partitions, stacked and virtual devices are excluded)",
+                METRIC_FAMILY_TYPE_COUNTER,
+        },
+        /* Keep those ↑ in sync with disk_io_generate(). */
         {
                 METRIC_IO_SYSTEMD_BASIC_PREFIX "Hostname",
                 "System hostname",
@@ -592,6 +809,12 @@ static const MetricFamily metric_family_table[] = {
         MACHINE_INFO_STANDARD_FIELD("LOCATION"),
         MACHINE_INFO_STANDARD_FIELD("TAGS"),
         /* Keep those ↑ in sync with machine_info_generate(). */
+        {
+                METRIC_IO_SYSTEMD_BASIC_PREFIX "MemoryUsedBytes",
+                "Memory currently in use (MemTotal minus MemAvailable) in bytes",
+                METRIC_FAMILY_TYPE_GAUGE,
+                .generate = memory_used_generate,
+        },
         {
                 METRIC_IO_SYSTEMD_BASIC_PREFIX "OSRelease.NAME",
                 "Operating system human-readable name (PRETTY_NAME= or NAME= field from os-release)",
@@ -616,6 +839,18 @@ static const MetricFamily metric_family_table[] = {
                 METRIC_FAMILY_TYPE_GAUGE,
                 .generate = physical_memory_generate,
         },
+        {
+                METRIC_IO_SYSTEMD_BASIC_PREFIX "PressureAvg10",
+                "Pressure stall percentage over the last 10s (resource=cpu|memory|io, type=some|full)",
+                METRIC_FAMILY_TYPE_GAUGE,
+                .generate = pressure_generate,
+        },
+        {
+                METRIC_IO_SYSTEMD_BASIC_PREFIX "PressureStallSeconds",
+                "Cumulative time stalled on a resource, in seconds (resource=cpu|memory|io, type=some|full)",
+                METRIC_FAMILY_TYPE_COUNTER,
+        },
+        /* Keep those ↑ in sync with pressure_generate(). */
         {
                 /* NB: Here we use the naming of the field as per SMBIOS specification, i.e. undo the weird
                  * renaming that Linux did on the fields. When new fields are added here, please make sure to
@@ -650,6 +885,12 @@ static const MetricFamily metric_family_table[] = {
                 METRIC_FAMILY_TYPE_GAUGE,
                 .generate = swap_generate,
         },
+        {
+                METRIC_IO_SYSTEMD_BASIC_PREFIX "SwapUsedBytes",
+                "Swap space currently in use in bytes",
+                METRIC_FAMILY_TYPE_GAUGE,
+        },
+        /* Keep those ↑ in sync with swap_generate(). */
         {
                 METRIC_IO_SYSTEMD_BASIC_PREFIX "TPM2.Manufacturer",
                 "TPM2 device manufacturer (ID_TPM2_MANUFACTURER property of the tpmrm0 device)",

test/units/TEST-74-AUX-UTILS.report.sh

@@ -2,6 +2,7 @@
 # SPDX-License-Identifier: LGPL-2.1-or-later
 set -eux
 set -o pipefail
+shopt -s nullglob
 
 # Unset $PAGER so we don't have to use --no-pager everywhere
 export PAGER=
@@ -77,20 +78,87 @@ id2="$(. /etc/os-release; echo "$ID")"
 basic_metrics="$(varlinkctl call --more /run/systemd/report/io.systemd.Basic io.systemd.Metrics.List {})"
 # NB: '| tostring' turns the numeric value into a raw string, so 'jq -r' prints it cleanly
 # (numbers, unlike strings, are otherwise still emitted with the json-seq record separator).
-basic_value() { echo "$basic_metrics" | jq --seq -r "select(.name == \"$1\") | .value | tostring"; }
+basic_value() { echo "$basic_metrics" | jq --seq -r "select(.name == \"io.systemd.Basic.$1\") | .value | tostring"; }
+# Like basic_value, but the value must be a JSON number (jq's 'numbers' filter emits the value
+# only if it is one, so a non-empty result confirms both presence and type), and an optional
+# second argument gives a jq condition further restricting the match, e.g. on .object or .fields.
+basic_number() { echo "$basic_metrics" | jq --seq -r "select(.name == \"io.systemd.Basic.$1\" and ${2:-true}) | .value | numbers | tostring"; }
 
-# The three classic load average fields must be present and numeric (jq's 'numbers' filter
-# emits the value only if it is a JSON number, so a non-empty result confirms both).
+# The three classic load average fields must be present and numeric.
 for field in LoadAverage1Min LoadAverage5Min LoadAverage15Min; do
-    loadavg="$(echo "$basic_metrics" | jq --seq -r "select(.name == \"io.systemd.Basic.$field\") | .value | numbers | tostring")"
-    test -n "$loadavg"
+    test -n "$(basic_number "$field")"
 done
 
 # SwapBytes must match the total the kernel reports in /proc/meminfo (which is in kB).
-swap_reported="$(basic_value io.systemd.Basic.SwapBytes)"
+swap_reported="$(basic_value SwapBytes)"
 swap_expected=$(( $(awk '/^SwapTotal:/ { print $2; found=1 } END { if (!found) print 0 }' /proc/meminfo) * 1024 ))
 [ "$swap_reported" = "$swap_expected" ]
 
+# SwapUsedBytes must be numeric and cannot exceed the configured total.
+swap_used="$(basic_number SwapUsedBytes)"
+test -n "$swap_used"
+[ "$swap_used" -le "$swap_reported" ]
+
+# Those fields must be reported with a numeric nanoseconds value.
+for type in user system; do
+    test -n "$(basic_number CPUUsage ".fields.type == \"$type\"")"
+done
+
+# MemoryUsedBytes is MemTotal − MemAvailable, so it cannot exceed the MemTotal reported by the
+# same file. (PhysicalMemoryBytes is not a valid bound here: it is capped by the root cgroup's
+# memory limit, so it can legitimately be smaller than the value computed from /proc/meminfo.)
+mem_used="$(basic_number MemoryUsedBytes)"
+mem_total=$(( $(awk '/^MemTotal:/ { print $2 }' /proc/meminfo) * 1024 ))
+test -n "$mem_used"
+[ "$mem_used" -le "$mem_total" ]
+
+# Pressure metrics are only reported if the kernel supports PSI.
+if cat /proc/pressure/cpu >/dev/null 2>&1; then
+    for family in PressureAvg10 PressureStallSeconds; do
+        for resource in cpu memory io; do
+            test -n "$(basic_number "$family" ".fields.resource == \"$resource\" and .fields.type == \"some\"")"
+            # "full" is deliberately not emitted for cpu, where the kernel always reports 0.
+            full="$(basic_number "$family" ".fields.resource == \"$resource\" and .fields.type == \"full\"")"
+            if [ "$resource" = cpu ]; then
+                test -z "$full"
+            else
+                test -n "$full"
+            fi
+        done
+    done
+
+    # The avg10 values are percentages, so they must lie in the 0…100 range. This would catch a
+    # wrong fixed-point conversion, which would inflate the value 2048x.
+    echo "$basic_metrics" | jq --seq -r 'select(.name == "io.systemd.Basic.PressureAvg10") | .value | tostring' |
+        awk '{ if ($1 < 0 || $1 > 100) exit 1 }'
+fi
+
+# Every whole physical block device (not partitions, not virtual or stacked devices) must be
+# reported, with byte values derived from the 512-byte sector counters in sysfs. Those counters
+# only grow, so the values sampled above cannot exceed what sysfs shows now.
+for dev in /sys/block/*; do
+    devpath="$(udevadm info --query=property --value --property=DEVPATH "$dev" || :)"
+    devname="$(udevadm info --query=property --value --property=DEVNAME "$dev" || :)"
+    [[ -n "$devpath" && -n "$devname" ]] || continue
+
+    [[ "$devpath" =~ "/devices/virtual/" ]] && continue
+
+    read_bytes="$(basic_number DiskReadBytes ".object == \"${devname}\"")"
+    write_bytes="$(basic_number DiskWriteBytes ".object == \"${devname}\"")"
+    test -n "$read_bytes"
+    test -n "$write_bytes"
+
+    read -r _ _ read_sectors _ _ _ write_sectors _ <"$dev/stat"
+    [ "$read_bytes" -le "$(( read_sectors * 512 ))" ]
+    [ "$write_bytes" -le "$(( write_sectors * 512 ))" ]
+done
+
+# Every new metric family must be described, even those whose values depend on the environment.
+basic_describe="$(varlinkctl call --more /run/systemd/report/io.systemd.Basic io.systemd.Metrics.Describe {})"
+for name in CPUUsage DiskReadBytes DiskWriteBytes MemoryUsedBytes PressureAvg10 PressureStallSeconds SwapUsedBytes; do
+    echo "$basic_describe" | grep -F "io.systemd.Basic.$name" >/dev/null
+done
+
 # io.systemd.Manager.Version should be non-empty and match what `systemctl --version` reports
 metrics_version="$(varlinkctl call --more /run/systemd/report/io.systemd.Manager io.systemd.Metrics.List {} | jq --seq -r 'select(.name == "io.systemd.Manager.Version") | .value')"
 [ -n "$metrics_version" ]