diff --git a/src/basic/procfs-util.c b/src/basic/procfs-util.c index af3f14ae3f..4e80e873a8 100644 --- a/src/basic/procfs-util.c +++ b/src/basic/procfs-util.c @@ -127,18 +127,8 @@ static uint64_t calc_gcd64(uint64_t a, uint64_t b) { return a; } -int procfs_cpu_get_usage(nsec_t *ret) { - ProcfsCpuTicks ticks; - int r; - - assert(ret); - - r = procfs_cpu_get_ticks(&ticks); - if (r < 0) - return r; - +nsec_t procfs_ticks_to_nsec(uint64_t ticks) { uint64_t ticks_per_second = sysconf_clock_ticks_cached(), - sum = ticks.user + ticks.nice + ticks.system + ticks.irq + ticks.softirq, gcd = calc_gcd64(NSEC_PER_SEC, ticks_per_second); /* Reduce NSEC_PER_SEC/ticks_per_second first to avoid overflow of ticks * NSEC_PER_SEC, and split @@ -148,7 +138,20 @@ int procfs_cpu_get_usage(nsec_t *ret) { uint64_t a = NSEC_PER_SEC / gcd, b = ticks_per_second / gcd; - *ret = (nsec_t) (sum / b * a + DIV_ROUND_UP(sum % b * a, b)); + return ticks / b * a + DIV_ROUND_UP(ticks % b * a, b); +} + +int procfs_cpu_get_usage(nsec_t *ret) { + int r; + + assert(ret); + + ProcfsCpuTicks ticks; + r = procfs_cpu_get_ticks(&ticks); + if (r < 0) + return r; + + *ret = procfs_ticks_to_nsec(ticks.user + ticks.nice + ticks.system + ticks.irq + ticks.softirq); return 0; } diff --git a/src/basic/procfs-util.h b/src/basic/procfs-util.h index 15a07871d1..22218ac7c0 100644 --- a/src/basic/procfs-util.h +++ b/src/basic/procfs-util.h @@ -9,6 +9,7 @@ int procfs_get_threads_max(uint64_t *ret); int procfs_tasks_set_limit(uint64_t limit); int procfs_tasks_get_current(uint64_t *ret); +nsec_t procfs_ticks_to_nsec(uint64_t ticks); int procfs_cpu_get_usage(nsec_t *ret); typedef struct ProcfsCpuTicks { diff --git a/src/report/report-basic.c b/src/report/report-basic.c index cc3cdd8b24..7cb36d6885 100644 --- a/src/report/report-basic.c +++ b/src/report/report-basic.c @@ -11,6 +11,7 @@ #include "alloc-util.h" #include "architecture.h" +#include "blockdev-list.h" #include "confidential-virt.h" #include "cpu-set-util.h" #include "env-file.h" @@ -23,8 +24,13 @@ #include "log.h" #include "metrics.h" #include "os-util.h" +#include "path-util.h" +#include "procfs-util.h" +#include "psi-util.h" #include "report-basic.h" #include "string-util.h" +#include "time-util.h" +#include "utf8.h" #include "virt.h" static int architecture_generate(const MetricFamily *mf, sd_varlink *link, void *userdata) { @@ -124,6 +130,94 @@ static int physical_memory_generate(const MetricFamily *mf, sd_varlink *link, vo /* fields= */ NULL); } +static int memory_used_generate(const MetricFamily *mf, sd_varlink *link, void *userdata) { + int r; + + assert(mf && mf->name); + assert(link); + + /* Memory currently in use, i.e. MemTotal − MemAvailable from /proc/meminfo. Note that + * PhysicalMemoryBytes is not necessarily the matching total: it is capped by the root cgroup's + * memory limit, while /proc/meminfo always shows the whole system. */ + + uint64_t used; + r = procfs_memory_get_used(&used); + if (r < 0) { + log_warning_errno(r, "Failed to read memory usage from /proc/meminfo, ignoring: %m"); + return 0; + } + + return metric_build_send_unsigned( + mf, + link, + /* object= */ NULL, + used, + /* fields= */ NULL); +} + +static int pressure_generate(const MetricFamily mf[static 2], sd_varlink *link, void *userdata) { + int r; + + assert(mf && mf[0].name && mf[1].name); + assert(!mf[1].generate); + assert(link); + + /* The kernel's smoothed pressure stall percentages over the last 10s, plus the cumulative stall + * time, as documented in https://docs.kernel.org/accounting/psi.html */ + + for (PressureResource resource = 0; resource < _PRESSURE_RESOURCE_MAX; resource++) { + const char *name = pressure_resource_to_string(resource); + + _cleanup_free_ char *path = path_join("/proc/pressure", name); + if (!path) + return -ENOMEM; + + PressureType type; + FOREACH_ARGUMENT(type, PRESSURE_TYPE_SOME, PRESSURE_TYPE_FULL) { + /* The "full" number for cpu pressure at the system level is always reported as 0, + * so skip it. */ + if (resource == PRESSURE_CPU && type == PRESSURE_TYPE_FULL) + continue; + + ResourcePressure rp; + r = read_resource_pressure(path, type, &rp); + if (r < 0) { + bool absent = IN_SET(r, -ENOENT, -ENODATA) || ERRNO_IS_NEG_NOT_SUPPORTED(r); + log_full_errno(absent ? LOG_DEBUG : LOG_WARNING, r, + "Failed to read %s pressure, ignoring: %m", name); + break; /* If "some" can't be read, "full" won't work either */ + } + + _cleanup_(sd_json_variant_unrefp) sd_json_variant *fields = NULL; + r = sd_json_buildo(&fields, + SD_JSON_BUILD_PAIR_STRING("resource", name), + SD_JSON_BUILD_PAIR_STRING("type", pressure_type_to_string(type))); + if (r < 0) + return r; + + r = metric_build_send_double( + mf, + link, + /* object= */ NULL, + (double) rp.avg10 / LOADAVG_FIXED_POINT_1_0, + fields); + if (r < 0) + return r; + + r = metric_build_send_double( + mf + 1, + link, + /* object= */ NULL, + (double) rp.total / USEC_PER_SEC, + fields); + if (r < 0) + return r; + } + } + + return 0; +} + static int cpus_online_generate(const MetricFamily *mf, sd_varlink *link, void *userdata) { int r; @@ -143,6 +237,52 @@ static int cpus_online_generate(const MetricFamily *mf, sd_varlink *link, void * /* fields= */ NULL); } +static int cpu_usage_generate(const MetricFamily *mf, sd_varlink *link, void *userdata) { + int r; + + assert(mf && mf->name); + assert(link); + + /* The aggregate "cpu" line of /proc/stat, converted from USER_HZ ticks to nanoseconds, matching + * the unit of io.systemd.CGroup.CpuUsage. "nice" time is folded into "user" and "irq"/"softirq" + * into "system", so that the published types together account for all CPU time. */ + + ProcfsCpuTicks ticks; + r = procfs_cpu_get_ticks(&ticks); + if (r < 0) { + log_warning_errno(r, "Failed to read CPU time from /proc/stat, ignoring: %m"); + return 0; + } + + static const char* const types[] = { "user", "system" }; + const uint64_t values[] = { + ticks.user + ticks.nice, + ticks.system + ticks.irq + ticks.softirq, + }; + assert_cc(ELEMENTSOF(types) == ELEMENTSOF(values)); + + for (size_t i = 0; i < ELEMENTSOF(values); i++) { + _cleanup_(sd_json_variant_unrefp) sd_json_variant *fields = NULL; + + r = sd_json_buildo(&fields, SD_JSON_BUILD_PAIR_STRING("type", types[i])); + if (r < 0) + return r; + + uint64_t nsec = procfs_ticks_to_nsec(values[i]); + + r = metric_build_send_unsigned( + mf, + link, + /* object= */ NULL, + nsec, + fields); + if (r < 0) + return r; + } + + return 0; +} + static int load_average_generate(const MetricFamily *mf, sd_varlink *link, void *userdata) { enum { LOAD_AVERAGE_FIELD_1MIN, @@ -181,11 +321,14 @@ static int load_average_generate(const MetricFamily *mf, sd_varlink *link, void return 0; } -static int swap_generate(const MetricFamily *mf, sd_varlink *link, void *userdata) { - assert(mf && mf->name); +static int swap_generate(const MetricFamily mf[static 2], sd_varlink *link, void *userdata) { + int r; + + assert(mf && mf[0].name && mf[1].name); + assert(!mf[1].generate); assert(link); - /* The total amount of configured swap space, in bytes. */ + /* The total amount of configured swap space and the amount currently in use, in bytes. */ struct sysinfo info; if (sysinfo(&info) < 0) @@ -193,15 +336,69 @@ static int swap_generate(const MetricFamily *mf, sd_varlink *link, void *userdat /* Overflow is unrealistic (would need >16 EiB of swap), but use MUL_SAFE to make this obvious to * static analyzers. */ - uint64_t swap; - assert_se(MUL_SAFE(&swap, (uint64_t) info.totalswap, (uint64_t) info.mem_unit)); + uint64_t total, used; + assert_se(MUL_SAFE(&total, (uint64_t) info.totalswap, (uint64_t) info.mem_unit)); + assert_se(MUL_SAFE(&used, (uint64_t) LESS_BY(info.totalswap, info.freeswap), (uint64_t) info.mem_unit)); - return metric_build_send_unsigned( - mf, - link, - /* object= */ NULL, - swap, - /* fields= */ NULL); + const uint64_t values[] = { total, used }; + for (size_t i = 0; i < ELEMENTSOF(values); i++) { + r = metric_build_send_unsigned( + mf + i, + link, + /* object= */ NULL, + values[i], + /* fields= */ NULL); + if (r < 0) + return r; + } + + return 0; +} + +static int disk_io_generate(const MetricFamily mf[static 2], sd_varlink *link, void *userdata) { + int r; + + assert(mf && mf[0].name && mf[1].name); + assert(!mf[1].generate); + assert(link); + + /* Cumulative read/write byte counters of all whole physical block devices. Partitions are skipped, + * their traffic is already accounted for in the parent device. Stacked devices (dm, md, …) are + * skipped for the same reason: their traffic is accounted for again in the devices they are built + * on. */ + + BlockDevice *devices = NULL; + size_t n_devices = 0; + CLEANUP_ARRAY(devices, n_devices, block_device_array_free); + + r = blockdev_list(BLOCKDEV_LIST_IGNORE_VIRTUAL | BLOCKDEV_LIST_IGNORE_PARTITIONS | BLOCKDEV_LIST_RW_STATS, + &devices, &n_devices); + if (r < 0) + return r; + + FOREACH_ARRAY(d, devices, n_devices) { + /* A non-UTF-8 name would make the JSON reply builder fail, taking down the whole method + * call rather than just this device. */ + if (!utf8_is_valid(d->node)) { + log_warning("%s: block device name is not valid UTF-8, ignoring device.", d->node); + continue; + } + + const uint64_t values[] = { d->read_bytes, d->write_bytes }; + for (size_t i = 0; i < ELEMENTSOF(values); i++) + if (values[i] != UINT64_MAX) { + r = metric_build_send_unsigned( + mf + i, + link, + /* object= */ d->node, + values[i], + /* fields= */ NULL); + if (r < 0) + return r; + } + } + + return 0; } enum { @@ -547,6 +744,26 @@ static const MetricFamily metric_family_table[] = { METRIC_FAMILY_TYPE_GAUGE, .generate = cpus_online_generate, }, + { + METRIC_IO_SYSTEMD_BASIC_PREFIX "CPUUsage", + "Aggregate CPU usage across all CPUs in nanoseconds (type=user|system)", + METRIC_FAMILY_TYPE_COUNTER, + .generate = cpu_usage_generate, + }, + { + METRIC_IO_SYSTEMD_BASIC_PREFIX "DiskReadBytes", + "Per block device metric: cumulative number of bytes read " + "(partitions, stacked and virtual devices are excluded)", + METRIC_FAMILY_TYPE_COUNTER, + .generate = disk_io_generate, + }, + { + METRIC_IO_SYSTEMD_BASIC_PREFIX "DiskWriteBytes", + "Per block device metric: cumulative number of bytes written " + "(partitions, stacked and virtual devices are excluded)", + METRIC_FAMILY_TYPE_COUNTER, + }, + /* Keep those ↑ in sync with disk_io_generate(). */ { METRIC_IO_SYSTEMD_BASIC_PREFIX "Hostname", "System hostname", @@ -592,6 +809,12 @@ static const MetricFamily metric_family_table[] = { MACHINE_INFO_STANDARD_FIELD("LOCATION"), MACHINE_INFO_STANDARD_FIELD("TAGS"), /* Keep those ↑ in sync with machine_info_generate(). */ + { + METRIC_IO_SYSTEMD_BASIC_PREFIX "MemoryUsedBytes", + "Memory currently in use (MemTotal minus MemAvailable) in bytes", + METRIC_FAMILY_TYPE_GAUGE, + .generate = memory_used_generate, + }, { METRIC_IO_SYSTEMD_BASIC_PREFIX "OSRelease.NAME", "Operating system human-readable name (PRETTY_NAME= or NAME= field from os-release)", @@ -616,6 +839,18 @@ static const MetricFamily metric_family_table[] = { METRIC_FAMILY_TYPE_GAUGE, .generate = physical_memory_generate, }, + { + METRIC_IO_SYSTEMD_BASIC_PREFIX "PressureAvg10", + "Pressure stall percentage over the last 10s (resource=cpu|memory|io, type=some|full)", + METRIC_FAMILY_TYPE_GAUGE, + .generate = pressure_generate, + }, + { + METRIC_IO_SYSTEMD_BASIC_PREFIX "PressureStallSeconds", + "Cumulative time stalled on a resource, in seconds (resource=cpu|memory|io, type=some|full)", + METRIC_FAMILY_TYPE_COUNTER, + }, + /* Keep those ↑ in sync with pressure_generate(). */ { /* NB: Here we use the naming of the field as per SMBIOS specification, i.e. undo the weird * renaming that Linux did on the fields. When new fields are added here, please make sure to @@ -650,6 +885,12 @@ static const MetricFamily metric_family_table[] = { METRIC_FAMILY_TYPE_GAUGE, .generate = swap_generate, }, + { + METRIC_IO_SYSTEMD_BASIC_PREFIX "SwapUsedBytes", + "Swap space currently in use in bytes", + METRIC_FAMILY_TYPE_GAUGE, + }, + /* Keep those ↑ in sync with swap_generate(). */ { METRIC_IO_SYSTEMD_BASIC_PREFIX "TPM2.Manufacturer", "TPM2 device manufacturer (ID_TPM2_MANUFACTURER property of the tpmrm0 device)", diff --git a/test/units/TEST-74-AUX-UTILS.report.sh b/test/units/TEST-74-AUX-UTILS.report.sh index 627e7fef65..625095c32e 100755 --- a/test/units/TEST-74-AUX-UTILS.report.sh +++ b/test/units/TEST-74-AUX-UTILS.report.sh @@ -2,6 +2,7 @@ # SPDX-License-Identifier: LGPL-2.1-or-later set -eux set -o pipefail +shopt -s nullglob # Unset $PAGER so we don't have to use --no-pager everywhere export PAGER= @@ -77,20 +78,87 @@ id2="$(. /etc/os-release; echo "$ID")" basic_metrics="$(varlinkctl call --more /run/systemd/report/io.systemd.Basic io.systemd.Metrics.List {})" # NB: '| tostring' turns the numeric value into a raw string, so 'jq -r' prints it cleanly # (numbers, unlike strings, are otherwise still emitted with the json-seq record separator). -basic_value() { echo "$basic_metrics" | jq --seq -r "select(.name == \"$1\") | .value | tostring"; } +basic_value() { echo "$basic_metrics" | jq --seq -r "select(.name == \"io.systemd.Basic.$1\") | .value | tostring"; } +# Like basic_value, but the value must be a JSON number (jq's 'numbers' filter emits the value +# only if it is one, so a non-empty result confirms both presence and type), and an optional +# second argument gives a jq condition further restricting the match, e.g. on .object or .fields. +basic_number() { echo "$basic_metrics" | jq --seq -r "select(.name == \"io.systemd.Basic.$1\" and ${2:-true}) | .value | numbers | tostring"; } -# The three classic load average fields must be present and numeric (jq's 'numbers' filter -# emits the value only if it is a JSON number, so a non-empty result confirms both). +# The three classic load average fields must be present and numeric. for field in LoadAverage1Min LoadAverage5Min LoadAverage15Min; do - loadavg="$(echo "$basic_metrics" | jq --seq -r "select(.name == \"io.systemd.Basic.$field\") | .value | numbers | tostring")" - test -n "$loadavg" + test -n "$(basic_number "$field")" done # SwapBytes must match the total the kernel reports in /proc/meminfo (which is in kB). -swap_reported="$(basic_value io.systemd.Basic.SwapBytes)" +swap_reported="$(basic_value SwapBytes)" swap_expected=$(( $(awk '/^SwapTotal:/ { print $2; found=1 } END { if (!found) print 0 }' /proc/meminfo) * 1024 )) [ "$swap_reported" = "$swap_expected" ] +# SwapUsedBytes must be numeric and cannot exceed the configured total. +swap_used="$(basic_number SwapUsedBytes)" +test -n "$swap_used" +[ "$swap_used" -le "$swap_reported" ] + +# Those fields must be reported with a numeric nanoseconds value. +for type in user system; do + test -n "$(basic_number CPUUsage ".fields.type == \"$type\"")" +done + +# MemoryUsedBytes is MemTotal − MemAvailable, so it cannot exceed the MemTotal reported by the +# same file. (PhysicalMemoryBytes is not a valid bound here: it is capped by the root cgroup's +# memory limit, so it can legitimately be smaller than the value computed from /proc/meminfo.) +mem_used="$(basic_number MemoryUsedBytes)" +mem_total=$(( $(awk '/^MemTotal:/ { print $2 }' /proc/meminfo) * 1024 )) +test -n "$mem_used" +[ "$mem_used" -le "$mem_total" ] + +# Pressure metrics are only reported if the kernel supports PSI. +if cat /proc/pressure/cpu >/dev/null 2>&1; then + for family in PressureAvg10 PressureStallSeconds; do + for resource in cpu memory io; do + test -n "$(basic_number "$family" ".fields.resource == \"$resource\" and .fields.type == \"some\"")" + # "full" is deliberately not emitted for cpu, where the kernel always reports 0. + full="$(basic_number "$family" ".fields.resource == \"$resource\" and .fields.type == \"full\"")" + if [ "$resource" = cpu ]; then + test -z "$full" + else + test -n "$full" + fi + done + done + + # The avg10 values are percentages, so they must lie in the 0…100 range. This would catch a + # wrong fixed-point conversion, which would inflate the value 2048x. + echo "$basic_metrics" | jq --seq -r 'select(.name == "io.systemd.Basic.PressureAvg10") | .value | tostring' | + awk '{ if ($1 < 0 || $1 > 100) exit 1 }' +fi + +# Every whole physical block device (not partitions, not virtual or stacked devices) must be +# reported, with byte values derived from the 512-byte sector counters in sysfs. Those counters +# only grow, so the values sampled above cannot exceed what sysfs shows now. +for dev in /sys/block/*; do + devpath="$(udevadm info --query=property --value --property=DEVPATH "$dev" || :)" + devname="$(udevadm info --query=property --value --property=DEVNAME "$dev" || :)" + [[ -n "$devpath" && -n "$devname" ]] || continue + + [[ "$devpath" =~ "/devices/virtual/" ]] && continue + + read_bytes="$(basic_number DiskReadBytes ".object == \"${devname}\"")" + write_bytes="$(basic_number DiskWriteBytes ".object == \"${devname}\"")" + test -n "$read_bytes" + test -n "$write_bytes" + + read -r _ _ read_sectors _ _ _ write_sectors _ <"$dev/stat" + [ "$read_bytes" -le "$(( read_sectors * 512 ))" ] + [ "$write_bytes" -le "$(( write_sectors * 512 ))" ] +done + +# Every new metric family must be described, even those whose values depend on the environment. +basic_describe="$(varlinkctl call --more /run/systemd/report/io.systemd.Basic io.systemd.Metrics.Describe {})" +for name in CPUUsage DiskReadBytes DiskWriteBytes MemoryUsedBytes PressureAvg10 PressureStallSeconds SwapUsedBytes; do + echo "$basic_describe" | grep -F "io.systemd.Basic.$name" >/dev/null +done + # io.systemd.Manager.Version should be non-empty and match what `systemctl --version` reports metrics_version="$(varlinkctl call --more /run/systemd/report/io.systemd.Manager io.systemd.Metrics.List {} | jq --seq -r 'select(.name == "io.systemd.Manager.Version") | .value')" [ -n "$metrics_version" ]