From 0a38e6b9a37ba16d6dd6e50323c5441914b40af9 Mon Sep 17 00:00:00 2001
From: Lennart Poettering <lennart@poettering.net>
Date: Tue, 8 Jan 2019 18:23:40 +0100
Subject: [PATCH 01/29] fileio: add an openat() flavour for fopen()

This adds xfopenat() which is to fopen() what xopendirat() is to
opendir(), i.e. the "at" counterpart to fopen().

(Similar to the xopendir() case, we prefix this with "x", in case libc
gains this natively eventually.)
---
 src/basic/fileio.c | 75 ++++++++++++++++++++++++++++++++++++++++++++++
 src/basic/fileio.h |  1 +
 2 files changed, 76 insertions(+)

diff --git a/src/basic/fileio.c b/src/basic/fileio.c
index 714a00cd9a..044b31d584 100644
--- a/src/basic/fileio.c
+++ b/src/basic/fileio.c
@@ -680,6 +680,81 @@ DIR *xopendirat(int fd, const char *name, int flags) {
         return d;
 }
 
+static int mode_to_flags(const char *mode) {
+        const char *p;
+        int flags;
+
+        if ((p = startswith(mode, "r+")))
+                flags = O_RDWR;
+        else if ((p = startswith(mode, "r")))
+                flags = O_RDONLY;
+        else if ((p = startswith(mode, "w+")))
+                flags = O_RDWR|O_CREAT|O_TRUNC;
+        else if ((p = startswith(mode, "w")))
+                flags = O_WRONLY|O_CREAT|O_TRUNC;
+        else if ((p = startswith(mode, "a+")))
+                flags = O_RDWR|O_CREAT|O_APPEND;
+        else if ((p = startswith(mode, "a")))
+                flags = O_WRONLY|O_CREAT|O_APPEND;
+        else
+                return -EINVAL;
+
+        for (; *p != 0; p++) {
+
+                switch (*p) {
+
+                case 'e':
+                        flags |= O_CLOEXEC;
+                        break;
+
+                case 'x':
+                        flags |= O_EXCL;
+                        break;
+
+                case 'm':
+                        /* ignore this here, fdopen() might care later though */
+                        break;
+
+                case 'c': /* not sure what to do about this one */
+                default:
+                        return -EINVAL;
+                }
+        }
+
+        return flags;
+}
+
+int xfopenat(int dir_fd, const char *path, const char *mode, int flags, FILE **ret) {
+        FILE *f;
+
+        /* A combination of fopen() with openat() */
+
+        if (dir_fd == AT_FDCWD && flags == 0) {
+                f = fopen(path, mode);
+                if (!f)
+                        return -errno;
+        } else {
+                int fd, mode_flags;
+
+                mode_flags = mode_to_flags(mode);
+                if (mode_flags < 0)
+                        return mode_flags;
+
+                fd = openat(dir_fd, path, mode_flags | flags);
+                if (fd < 0)
+                        return -errno;
+
+                f = fdopen(fd, mode);
+                if (!f) {
+                        safe_close(fd);
+                        return -errno;
+                }
+        }
+
+        *ret = f;
+        return 0;
+}
+
 static int search_and_fopen_internal(const char *path, const char *mode, const char *root, char **search, FILE **_f) {
         char **i;
 
diff --git a/src/basic/fileio.h b/src/basic/fileio.h
index 31bfef33ac..d5bfe1f0fa 100644
--- a/src/basic/fileio.h
+++ b/src/basic/fileio.h
@@ -69,6 +69,7 @@ int executable_is_script(const char *path, char **interpreter);
 int get_proc_field(const char *filename, const char *pattern, const char *terminator, char **field);
 
 DIR *xopendirat(int dirfd, const char *name, int flags);
+int xfopenat(int dir_fd, const char *path, const char *mode, int flags, FILE **ret);
 
 int search_and_fopen(const char *path, const char *mode, const char *root, const char **search, FILE **_f);
 int search_and_fopen_nulstr(const char *path, const char *mode, const char *root, const char *search, FILE **_f);

From 8241f785f414dfa1f1af053451eafa53e215b1ac Mon Sep 17 00:00:00 2001
From: Lennart Poettering <lennart@poettering.net>
Date: Tue, 8 Jan 2019 18:29:36 +0100
Subject: [PATCH 02/29] fileio: add 'dir_fd' parameter to read_full_file_full()

Let's introduce an "at" version of read_full_file().
---
 src/basic/fileio.c             | 6 ++++--
 src/basic/fileio.h             | 5 +++--
 src/network/netdev/macsec.c    | 2 +-
 src/network/netdev/wireguard.c | 2 +-
 4 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/src/basic/fileio.c b/src/basic/fileio.c
index 044b31d584..ef27c3f913 100644
--- a/src/basic/fileio.c
+++ b/src/basic/fileio.c
@@ -543,17 +543,19 @@ finalize:
         return r;
 }
 
-int read_full_file_full(const char *filename, ReadFullFileFlags flags, char **contents, size_t *size) {
+int read_full_file_full(int dir_fd, const char *filename, ReadFullFileFlags flags, char **contents, size_t *size) {
         _cleanup_fclose_ FILE *f = NULL;
         int r;
 
         assert(filename);
         assert(contents);
 
-        r = fopen_unlocked(filename, "re", &f);
+        r = xfopenat(dir_fd, filename, "re", 0, &f);
         if (r < 0)
                 return r;
 
+        (void) __fsetlocking(f, FSETLOCKING_BYCALLER);
+
         return read_full_stream_full(f, filename, flags, contents, size);
 }
 
diff --git a/src/basic/fileio.h b/src/basic/fileio.h
index d5bfe1f0fa..209a06f826 100644
--- a/src/basic/fileio.h
+++ b/src/basic/fileio.h
@@ -6,6 +6,7 @@
 #include <stddef.h>
 #include <stdio.h>
 #include <sys/stat.h>
+#include <sys/fcntl.h>
 #include <sys/types.h>
 
 #include "macro.h"
@@ -52,9 +53,9 @@ static inline int write_string_file(const char *fn, const char *line, WriteStrin
 int write_string_filef(const char *fn, WriteStringFileFlags flags, const char *format, ...) _printf_(3, 4);
 
 int read_one_line_file(const char *filename, char **line);
-int read_full_file_full(const char *filename, ReadFullFileFlags flags, char **contents, size_t *size);
+int read_full_file_full(int dir_fd, const char *filename, ReadFullFileFlags flags, char **contents, size_t *size);
 static inline int read_full_file(const char *filename, char **contents, size_t *size) {
-        return read_full_file_full(filename, 0, contents, size);
+        return read_full_file_full(AT_FDCWD, filename, 0, contents, size);
 }
 int read_full_virtual_file(const char *filename, char **ret_contents, size_t *ret_size);
 int read_full_stream_full(FILE *f, const char *filename, ReadFullFileFlags flags, char **contents, size_t *size);
diff --git a/src/network/netdev/macsec.c b/src/network/netdev/macsec.c
index 25dc23ff03..c731596f83 100644
--- a/src/network/netdev/macsec.c
+++ b/src/network/netdev/macsec.c
@@ -981,7 +981,7 @@ static int macsec_read_key_file(NetDev *netdev, SecurityAssociation *sa) {
 
         (void) warn_file_is_world_accessible(sa->key_file, NULL, NULL, 0);
 
-        r = read_full_file_full(sa->key_file, READ_FULL_FILE_SECURE | READ_FULL_FILE_UNHEX, (char **) &key, &key_len);
+        r = read_full_file_full(AT_FDCWD, sa->key_file, READ_FULL_FILE_SECURE | READ_FULL_FILE_UNHEX, (char **) &key, &key_len);
         if (r < 0)
                 return log_netdev_error_errno(netdev, r,
                                               "Failed to read key from '%s', ignoring: %m",
diff --git a/src/network/netdev/wireguard.c b/src/network/netdev/wireguard.c
index 4a1228584d..820a6cfa0f 100644
--- a/src/network/netdev/wireguard.c
+++ b/src/network/netdev/wireguard.c
@@ -902,7 +902,7 @@ static int wireguard_read_key_file(const char *filename, uint8_t dest[static WG_
 
         (void) warn_file_is_world_accessible(filename, NULL, NULL, 0);
 
-        r = read_full_file_full(filename, READ_FULL_FILE_SECURE | READ_FULL_FILE_UNBASE64, &key, &key_len);
+        r = read_full_file_full(AT_FDCWD, filename, READ_FULL_FILE_SECURE | READ_FULL_FILE_UNBASE64, &key, &key_len);
         if (r < 0)
                 return r;
 

From 95244ceb9c827ce6762a57baf36aa229b594fd0f Mon Sep 17 00:00:00 2001
From: Lennart Poettering <lennart@poettering.net>
Date: Fri, 17 May 2019 15:52:35 +0200
Subject: [PATCH 03/29] fileio: add WRITE_STRING_FILE_MODE_0600 flag for
 writing files

usually we want to create new files with mode 0666 (modulated by the
umask). Sometimes we want more restrictive access though, let's add an
explicit flag support for that.

(Note that we don't bother with arbitrary access modes to keep things
simple: just "open as umask permits" and "private to me", nothing else)
---
 src/basic/fileio.c | 43 +++++++++++++++++++++----------------------
 src/basic/fileio.h |  1 +
 2 files changed, 22 insertions(+), 22 deletions(-)

diff --git a/src/basic/fileio.c b/src/basic/fileio.c
index ef27c3f913..fe0c4f4707 100644
--- a/src/basic/fileio.c
+++ b/src/basic/fileio.c
@@ -136,16 +136,21 @@ static int write_string_file_atomic(
         assert(fn);
         assert(line);
 
+        /* Note that we'd really like to use O_TMPFILE here, but can't really, since we want replacement
+         * semantics here, and O_TMPFILE can't offer that. i.e. rename() replaces but linkat() doesn't. */
+
         r = fopen_temporary(fn, &f, &p);
         if (r < 0)
                 return r;
 
-        (void) fchmod_umask(fileno(f), 0644);
-
         r = write_string_stream_ts(f, line, flags, ts);
         if (r < 0)
                 goto fail;
 
+        r = fchmod_umask(fileno(f), FLAGS_SET(flags, WRITE_STRING_FILE_MODE_0600) ? 0600 : 0644);
+        if (r < 0)
+                goto fail;
+
         if (rename(p, fn) < 0) {
                 r = -errno;
                 goto fail;
@@ -165,7 +170,7 @@ int write_string_file_ts(
                 struct timespec *ts) {
 
         _cleanup_fclose_ FILE *f = NULL;
-        int q, r;
+        int q, r, fd;
 
         assert(fn);
         assert(line);
@@ -190,26 +195,20 @@ int write_string_file_ts(
         } else
                 assert(!ts);
 
-        if (flags & WRITE_STRING_FILE_CREATE) {
-                r = fopen_unlocked(fn, "we", &f);
-                if (r < 0)
-                        goto fail;
-        } else {
-                int fd;
+        /* We manually build our own version of fopen(..., "we") that works without O_CREAT and with O_NOFOLLOW if needed. */
+        fd = open(fn, O_WRONLY|O_CLOEXEC|O_NOCTTY |
+                  (FLAGS_SET(flags, WRITE_STRING_FILE_NOFOLLOW) ? O_NOFOLLOW : 0) |
+                  (FLAGS_SET(flags, WRITE_STRING_FILE_CREATE) ? O_CREAT : 0),
+                  (FLAGS_SET(flags, WRITE_STRING_FILE_MODE_0600) ? 0600 : 0666));
+        if (fd < 0) {
+                r = -errno;
+                goto fail;
+        }
 
-                /* We manually build our own version of fopen(..., "we") that
-                 * works without O_CREAT */
-                fd = open(fn, O_WRONLY|O_CLOEXEC|O_NOCTTY | ((flags & WRITE_STRING_FILE_NOFOLLOW) ? O_NOFOLLOW : 0));
-                if (fd < 0) {
-                        r = -errno;
-                        goto fail;
-                }
-
-                r = fdopen_unlocked(fd, "w", &f);
-                if (r < 0) {
-                        safe_close(fd);
-                        goto fail;
-                }
+        r = fdopen_unlocked(fd, "w", &f);
+        if (r < 0) {
+                safe_close(fd);
+                goto fail;
         }
 
         if (flags & WRITE_STRING_FILE_DISABLE_BUFFER)
diff --git a/src/basic/fileio.h b/src/basic/fileio.h
index 209a06f826..e6fea2afd4 100644
--- a/src/basic/fileio.h
+++ b/src/basic/fileio.h
@@ -23,6 +23,7 @@ typedef enum {
         WRITE_STRING_FILE_DISABLE_BUFFER    = 1 << 5,
         WRITE_STRING_FILE_NOFOLLOW          = 1 << 6,
         WRITE_STRING_FILE_MKDIR_0755        = 1 << 7,
+        WRITE_STRING_FILE_MODE_0600         = 1 << 8,
 
         /* And before you wonder, why write_string_file_atomic_label_ts() is a separate function instead of just one
            more flag here: it's about linking: we don't want to pull -lselinux into all users of write_string_file()

From 07737617a18d33f3310329384423a53dd761218f Mon Sep 17 00:00:00 2001
From: Lennart Poettering <lennart@poettering.net>
Date: Thu, 3 Jan 2019 12:19:31 +0100
Subject: [PATCH 04/29] json: beef up strv parser to also accept a single
 string instead of an array of strings

Let's be permissive in what we accept and take a single string instead
of an array of strings, when a string is requested, too.
---
 src/shared/json.c | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/src/shared/json.c b/src/shared/json.c
index 7393c100cd..c3efb11bbb 100644
--- a/src/shared/json.c
+++ b/src/shared/json.c
@@ -3420,6 +3420,16 @@ int json_dispatch_strv(const char *name, JsonVariant *variant, JsonDispatchFlags
                 return 0;
         }
 
+        /* Let's be flexible here: accept a single string in place of a single-item array */
+        if (json_variant_is_string(variant)) {
+                l = strv_new(json_variant_string(variant));
+                if (!l)
+                        return log_oom();
+
+                strv_free_and_replace(*s, l);
+                return 0;
+        }
+
         if (!json_variant_is_array(variant))
                 return json_log(variant, SYNTHETIC_ERRNO(EINVAL), flags, "JSON field '%s' is not an array.", strna(name));
 

From e787b211a5aa9c06cb319f50479665d43a8f426e Mon Sep 17 00:00:00 2001
From: Lennart Poettering <lennart@poettering.net>
Date: Thu, 4 Jul 2019 17:38:17 +0200
Subject: [PATCH 05/29] json: add new json_variant_is_blank_{object,array}()
 helpers

---
 src/shared/json.c | 13 +++++++++++++
 src/shared/json.h |  2 ++
 2 files changed, 15 insertions(+)

diff --git a/src/shared/json.c b/src/shared/json.c
index c3efb11bbb..84ad6dd02b 100644
--- a/src/shared/json.c
+++ b/src/shared/json.c
@@ -946,6 +946,19 @@ mismatch:
         return false;
 }
 
+bool json_variant_is_blank_object(JsonVariant *v) {
+        /* Returns true if the specified object is null or empty */
+        return !v ||
+                json_variant_is_null(v) ||
+                (json_variant_is_object(v) && json_variant_elements(v) == 0);
+}
+
+bool json_variant_is_blank_array(JsonVariant *v) {
+        return !v ||
+                json_variant_is_null(v) ||
+                (json_variant_is_array(v) && json_variant_elements(v) == 0);
+}
+
 JsonVariantType json_variant_type(JsonVariant *v) {
 
         if (!v)
diff --git a/src/shared/json.h b/src/shared/json.h
index 1f9c620ebb..59962049d7 100644
--- a/src/shared/json.h
+++ b/src/shared/json.h
@@ -120,6 +120,8 @@ static inline bool json_variant_is_null(JsonVariant *v) {
 }
 
 bool json_variant_is_negative(JsonVariant *v);
+bool json_variant_is_blank_object(JsonVariant *v);
+bool json_variant_is_blank_array(JsonVariant *v);
 
 size_t json_variant_elements(JsonVariant *v);
 JsonVariant *json_variant_by_index(JsonVariant *v, size_t index);

From f2ff34ff2aaafd313a5c62b4b9f13ba6777731e5 Mon Sep 17 00:00:00 2001
From: Lennart Poettering <lennart@poettering.net>
Date: Thu, 3 Jan 2019 17:51:12 +0100
Subject: [PATCH 06/29] json: add new API json_variant_filter() for dropping
 fields from objects

---
 src/shared/json.c | 53 +++++++++++++++++++++++++++++++++++++++++++++++
 src/shared/json.h |  2 ++
 2 files changed, 55 insertions(+)

diff --git a/src/shared/json.c b/src/shared/json.c
index 84ad6dd02b..32f0424510 100644
--- a/src/shared/json.c
+++ b/src/shared/json.c
@@ -1622,6 +1622,59 @@ void json_variant_dump(JsonVariant *v, JsonFormatFlags flags, FILE *f, const cha
                 fputc('\n', f); /* In case of SSE add a second newline */
 }
 
+int json_variant_filter(JsonVariant **v, char **to_remove) {
+        _cleanup_(json_variant_unrefp) JsonVariant *w = NULL;
+        _cleanup_free_ JsonVariant **array = NULL;
+        size_t i, n = 0, k = 0;
+        int r;
+
+        assert(v);
+
+        if (json_variant_is_blank_object(*v))
+                return 0;
+        if (!json_variant_is_object(*v))
+                return -EINVAL;
+
+        if (strv_isempty(to_remove))
+                return 0;
+
+        for (i = 0; i < json_variant_elements(*v); i += 2) {
+                JsonVariant *p;
+
+                p = json_variant_by_index(*v, i);
+                if (!json_variant_has_type(p, JSON_VARIANT_STRING))
+                        return -EINVAL;
+
+                if (strv_contains(to_remove, json_variant_string(p))) {
+                        if (!array) {
+                                array = new(JsonVariant*, json_variant_elements(*v) - 2);
+                                if (!array)
+                                        return -ENOMEM;
+
+                                for (k = 0; k < i; k++)
+                                        array[k] = json_variant_by_index(*v, k);
+                        }
+
+                        n++;
+                } else if (array) {
+                        array[k++] = p;
+                        array[k++] = json_variant_by_index(*v, i + 1);
+                }
+        }
+
+        if (n == 0)
+                return 0;
+
+        r = json_variant_new_object(&w, array, k);
+        if (r < 0)
+                return r;
+
+        json_variant_unref(*v);
+        *v = TAKE_PTR(w);
+
+        return (int) n;
+}
+
 static int json_variant_copy(JsonVariant **nv, JsonVariant *v) {
         JsonVariantType t;
         JsonVariant *c;
diff --git a/src/shared/json.h b/src/shared/json.h
index 59962049d7..772ee67d7b 100644
--- a/src/shared/json.h
+++ b/src/shared/json.h
@@ -167,6 +167,8 @@ typedef enum JsonFormatFlags {
 int json_variant_format(JsonVariant *v, JsonFormatFlags flags, char **ret);
 void json_variant_dump(JsonVariant *v, JsonFormatFlags flags, FILE *f, const char *prefix);
 
+int json_variant_filter(JsonVariant **v, char **to_remove);
+
 int json_parse(const char *string, JsonVariant **ret, unsigned *ret_line, unsigned *ret_column);
 int json_parse_continue(const char **p, JsonVariant **ret, unsigned *ret_line, unsigned *ret_column);
 int json_parse_file(FILE *f, const char *path, JsonVariant **ret, unsigned *ret_line, unsigned *ret_column);

From 78a41236e40fe0c1ea55f1c2d63933c6fcb7b8ae Mon Sep 17 00:00:00 2001
From: Lennart Poettering <lennart@poettering.net>
Date: Fri, 4 Jan 2019 13:24:32 +0100
Subject: [PATCH 07/29] json: add new json_variant_set_field() helper

---
 src/shared/json.c | 69 +++++++++++++++++++++++++++++++++++++++++++++++
 src/shared/json.h |  2 ++
 2 files changed, 71 insertions(+)

diff --git a/src/shared/json.c b/src/shared/json.c
index 32f0424510..d85cc9f116 100644
--- a/src/shared/json.c
+++ b/src/shared/json.c
@@ -1675,6 +1675,75 @@ int json_variant_filter(JsonVariant **v, char **to_remove) {
         return (int) n;
 }
 
+int json_variant_set_field(JsonVariant **v, const char *field, JsonVariant *value) {
+        _cleanup_(json_variant_unrefp) JsonVariant *field_variant = NULL, *w = NULL;
+        _cleanup_free_ JsonVariant **array = NULL;
+        size_t i, k = 0;
+        int r;
+
+        assert(v);
+        assert(field);
+
+        if (json_variant_is_blank_object(*v)) {
+                array = new(JsonVariant*, 2);
+                if (!array)
+                        return -ENOMEM;
+
+        } else {
+                if (!json_variant_is_object(*v))
+                        return -EINVAL;
+
+                for (i = 0; i < json_variant_elements(*v); i += 2) {
+                        JsonVariant *p;
+
+                        p = json_variant_by_index(*v, i);
+                        if (!json_variant_is_string(p))
+                                return -EINVAL;
+
+                        if (streq(json_variant_string(p), field)) {
+
+                                if (!array) {
+                                        array = new(JsonVariant*, json_variant_elements(*v));
+                                        if (!array)
+                                                return -ENOMEM;
+
+                                        for (k = 0; k < i; k++)
+                                                array[k] = json_variant_by_index(*v, k);
+                                }
+
+                        } else if (array) {
+                                array[k++] = p;
+                                array[k++] = json_variant_by_index(*v, i + 1);
+                        }
+                }
+
+                if (!array) {
+                        array = new(JsonVariant*, json_variant_elements(*v) + 2);
+                        if (!array)
+                                return -ENOMEM;
+
+                        for (k = 0; k < json_variant_elements(*v); k++)
+                                array[k] = json_variant_by_index(*v, k);
+                }
+        }
+
+        r = json_variant_new_string(&field_variant, field);
+        if (r < 0)
+                return r;
+
+        array[k++] = field_variant;
+        array[k++] = value;
+
+        r = json_variant_new_object(&w, array, k);
+        if (r < 0)
+                return r;
+
+        json_variant_unref(*v);
+        *v = TAKE_PTR(w);
+
+        return 1;
+}
+
 static int json_variant_copy(JsonVariant **nv, JsonVariant *v) {
         JsonVariantType t;
         JsonVariant *c;
diff --git a/src/shared/json.h b/src/shared/json.h
index 772ee67d7b..0a9b2e111a 100644
--- a/src/shared/json.h
+++ b/src/shared/json.h
@@ -169,6 +169,8 @@ void json_variant_dump(JsonVariant *v, JsonFormatFlags flags, FILE *f, const cha
 
 int json_variant_filter(JsonVariant **v, char **to_remove);
 
+int json_variant_set_field(JsonVariant **v, const char *field, JsonVariant *value);
+
 int json_parse(const char *string, JsonVariant **ret, unsigned *ret_line, unsigned *ret_column);
 int json_parse_continue(const char **p, JsonVariant **ret, unsigned *ret_line, unsigned *ret_column);
 int json_parse_file(FILE *f, const char *path, JsonVariant **ret, unsigned *ret_line, unsigned *ret_column);

From 83bc6cb792330760efd02f1c9df80d3bcfae1453 Mon Sep 17 00:00:00 2001
From: Lennart Poettering <lennart@poettering.net>
Date: Tue, 8 Jan 2019 18:33:32 +0100
Subject: [PATCH 08/29] json: add a new "sensitive" flags for JsonVariant
 objects

An object marked with this flag will be erased from memory when it is
freed. This is useful for dealing with sensitive data (key material,
passphrases) encoded in JSON objects.
---
 src/shared/json.c | 76 +++++++++++++++++++++++++++++++++++++++++++++--
 src/shared/json.h |  2 ++
 2 files changed, 75 insertions(+), 3 deletions(-)

diff --git a/src/shared/json.c b/src/shared/json.c
index d85cc9f116..b9a4fe4e23 100644
--- a/src/shared/json.c
+++ b/src/shared/json.c
@@ -71,6 +71,9 @@ struct JsonVariant {
         /* While comparing two arrays, we use this for marking what we already have seen */
         bool is_marked:1;
 
+        /* Ersase from memory when freeing */
+        bool sensitive:1;
+
         /* The current 'depth' of the JsonVariant, i.e. how many levels of member variants this has */
         uint16_t depth;
 
@@ -650,7 +653,46 @@ int json_variant_new_object(JsonVariant **ret, JsonVariant **array, size_t n) {
         return 0;
 }
 
-static void json_variant_free_inner(JsonVariant *v) {
+static size_t json_variant_size(JsonVariant* v) {
+
+        if (!json_variant_is_regular(v))
+                return 0;
+
+        if (v->is_reference)
+                return offsetof(JsonVariant, reference) + sizeof(JsonVariant*);
+
+        switch (v->type) {
+
+        case JSON_VARIANT_STRING:
+                return offsetof(JsonVariant, string) + strlen(v->string) + 1;
+
+        case JSON_VARIANT_REAL:
+                return offsetof(JsonVariant, value) + sizeof(long double);
+
+        case JSON_VARIANT_UNSIGNED:
+                return offsetof(JsonVariant, value) + sizeof(uintmax_t);
+
+        case JSON_VARIANT_INTEGER:
+                return offsetof(JsonVariant, value) + sizeof(intmax_t);
+
+        case JSON_VARIANT_BOOLEAN:
+                return offsetof(JsonVariant, value) + sizeof(bool);
+
+        case JSON_VARIANT_ARRAY:
+        case JSON_VARIANT_OBJECT:
+                return offsetof(JsonVariant, n_elements) + sizeof(size_t);
+
+        case JSON_VARIANT_NULL:
+                return offsetof(JsonVariant, value);
+
+        default:
+                assert_not_reached("unexpected type");
+        }
+}
+
+static void json_variant_free_inner(JsonVariant *v, bool force_sensitive) {
+        bool sensitive;
+
         assert(v);
 
         if (!json_variant_is_regular(v))
@@ -658,7 +700,12 @@ static void json_variant_free_inner(JsonVariant *v) {
 
         json_source_unref(v->source);
 
+        sensitive = v->sensitive || force_sensitive;
+
         if (v->is_reference) {
+                if (sensitive)
+                        json_variant_sensitive(v->reference);
+
                 json_variant_unref(v->reference);
                 return;
         }
@@ -667,8 +714,11 @@ static void json_variant_free_inner(JsonVariant *v) {
                 size_t i;
 
                 for (i = 0; i < v->n_elements; i++)
-                        json_variant_free_inner(v + 1 + i);
+                        json_variant_free_inner(v + 1 + i, sensitive);
         }
+
+        if (sensitive)
+                explicit_bzero_safe(v, json_variant_size(v));
 }
 
 JsonVariant *json_variant_ref(JsonVariant *v) {
@@ -700,7 +750,7 @@ JsonVariant *json_variant_unref(JsonVariant *v) {
                 v->n_ref--;
 
                 if (v->n_ref == 0) {
-                        json_variant_free_inner(v);
+                        json_variant_free_inner(v, false);
                         free(v);
                 }
         }
@@ -1242,6 +1292,26 @@ bool json_variant_equal(JsonVariant *a, JsonVariant *b) {
         }
 }
 
+void json_variant_sensitive(JsonVariant *v) {
+        assert(v);
+
+        /* Marks a variant as "sensitive", so that it is erased from memory when it is destroyed. This is a
+         * one-way operation: as soon as it is marked this way it remains marked this way until it's
+         * destoryed. A magic variant is never sensitive though, even when asked, since it's too
+         * basic. Similar, const string variant are never sensitive either, after all they are included in
+         * the source code as they are, which is not suitable for inclusion of secrets.
+         *
+         * Note that this flag has a recursive effect: when we destroy an object or array we'll propagate the
+         * flag to all contained variants. And if those are then destroyed this is propagated further down,
+         * and so on. */
+
+        v = json_variant_normalize(v);
+        if (!json_variant_is_regular(v))
+                return;
+
+        v->sensitive = true;
+}
+
 int json_variant_get_source(JsonVariant *v, const char **ret_source, unsigned *ret_line, unsigned *ret_column) {
         assert_return(v, -EINVAL);
 
diff --git a/src/shared/json.h b/src/shared/json.h
index 0a9b2e111a..8e589a0b84 100644
--- a/src/shared/json.h
+++ b/src/shared/json.h
@@ -130,6 +130,8 @@ JsonVariant *json_variant_by_key_full(JsonVariant *v, const char *key, JsonVaria
 
 bool json_variant_equal(JsonVariant *a, JsonVariant *b);
 
+void json_variant_sensitive(JsonVariant *v);
+
 struct json_variant_foreach_state {
         JsonVariant *variant;
         size_t idx;

From f325aaf3417590a29d1c362a118532ea11fb020d Mon Sep 17 00:00:00 2001
From: Lennart Poettering <lennart@poettering.net>
Date: Tue, 8 Jan 2019 18:46:47 +0100
Subject: [PATCH 09/29] json: add json_parse_file_at() helper

This is an "at" function, similar to json_parse_file().
---
 src/shared/json.c | 4 ++--
 src/shared/json.h | 7 ++++++-
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/src/shared/json.c b/src/shared/json.c
index b9a4fe4e23..377a2c455a 100644
--- a/src/shared/json.c
+++ b/src/shared/json.c
@@ -2833,7 +2833,7 @@ int json_parse_continue(const char **p, JsonVariant **ret, unsigned *ret_line, u
         return json_parse_internal(p, NULL, ret, ret_line, ret_column, true);
 }
 
-int json_parse_file(FILE *f, const char *path, JsonVariant **ret, unsigned *ret_line, unsigned *ret_column) {
+int json_parse_file_at(FILE *f, int dir_fd, const char *path, JsonVariant **ret, unsigned *ret_line, unsigned *ret_column) {
         _cleanup_(json_source_unrefp) JsonSource *source = NULL;
         _cleanup_free_ char *text = NULL;
         const char *p;
@@ -2842,7 +2842,7 @@ int json_parse_file(FILE *f, const char *path, JsonVariant **ret, unsigned *ret_
         if (f)
                 r = read_full_stream(f, &text, NULL);
         else if (path)
-                r = read_full_file(path, &text, NULL);
+                r = read_full_file_full(dir_fd, path, 0, &text, NULL);
         else
                 return -EINVAL;
         if (r < 0)
diff --git a/src/shared/json.h b/src/shared/json.h
index 8e589a0b84..dd1aff450a 100644
--- a/src/shared/json.h
+++ b/src/shared/json.h
@@ -1,6 +1,7 @@
 /* SPDX-License-Identifier: LGPL-2.1+ */
 #pragma once
 
+#include <fcntl.h>
 #include <stdbool.h>
 #include <stddef.h>
 #include <stdint.h>
@@ -175,7 +176,11 @@ int json_variant_set_field(JsonVariant **v, const char *field, JsonVariant *valu
 
 int json_parse(const char *string, JsonVariant **ret, unsigned *ret_line, unsigned *ret_column);
 int json_parse_continue(const char **p, JsonVariant **ret, unsigned *ret_line, unsigned *ret_column);
-int json_parse_file(FILE *f, const char *path, JsonVariant **ret, unsigned *ret_line, unsigned *ret_column);
+int json_parse_file_at(FILE *f, int dir_fd, const char *path, JsonVariant **ret, unsigned *ret_line, unsigned *ret_column);
+
+static inline int json_parse_file(FILE *f, const char *path, JsonVariant **ret, unsigned *ret_line, unsigned *ret_column) {
+        return json_parse_file_at(f, AT_FDCWD, path, ret, ret_line, ret_column);
+}
 
 enum {
         _JSON_BUILD_STRING,

From d642f640bf39f9086a24920d97808fa40159a372 Mon Sep 17 00:00:00 2001
From: Lennart Poettering <lennart@poettering.net>
Date: Tue, 8 Jan 2019 18:48:48 +0100
Subject: [PATCH 10/29] json: add flags parameter to json_parse_file(), for
 parsing "sensitive" data

This will call json_variant_sensitive() internally while parsing for
each allocated sub-variant. This is better than calling it a posteriori
at the end, because partially parsed variants will always be properly
erased from memory this way.
---
 src/fuzz/fuzz-json.c    |  2 +-
 src/nspawn/nspawn-oci.c |  2 +-
 src/shared/json.c       | 21 ++++++++++++++-------
 src/shared/json.h       | 14 +++++++++-----
 src/shared/varlink.c    |  2 +-
 src/test/test-json.c    | 12 ++++++------
 6 files changed, 32 insertions(+), 21 deletions(-)

diff --git a/src/fuzz/fuzz-json.c b/src/fuzz/fuzz-json.c
index ce7b69dbb9..c01e2a570c 100644
--- a/src/fuzz/fuzz-json.c
+++ b/src/fuzz/fuzz-json.c
@@ -18,7 +18,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
         f = fmemopen_unlocked((char*) data, size, "re");
         assert_se(f);
 
-        if (json_parse_file(f, NULL, &v, NULL, NULL) < 0)
+        if (json_parse_file(f, NULL, 0, &v, NULL, NULL) < 0)
                 return 0;
 
         g = open_memstream_unlocked(&out, &out_size);
diff --git a/src/nspawn/nspawn-oci.c b/src/nspawn/nspawn-oci.c
index 4519c74b95..ba8b142c99 100644
--- a/src/nspawn/nspawn-oci.c
+++ b/src/nspawn/nspawn-oci.c
@@ -2214,7 +2214,7 @@ int oci_load(FILE *f, const char *bundle, Settings **ret) {
 
         path = strjoina(bundle, "/config.json");
 
-        r = json_parse_file(f, path, &oci, &line, &column);
+        r = json_parse_file(f, path, 0, &oci, &line, &column);
         if (r < 0) {
                 if (line != 0 && column != 0)
                         return log_error_errno(r, "Failed to parse '%s' at %u:%u: %m", path, line, column);
diff --git a/src/shared/json.c b/src/shared/json.c
index 377a2c455a..98fa067ef4 100644
--- a/src/shared/json.c
+++ b/src/shared/json.c
@@ -2479,6 +2479,7 @@ static void json_stack_release(JsonStack *s) {
 static int json_parse_internal(
                 const char **input,
                 JsonSource *source,
+                JsonParseFlags flags,
                 JsonVariant **ret,
                 unsigned *line,
                 unsigned *column,
@@ -2797,6 +2798,12 @@ static int json_parse_internal(
                 }
 
                 if (add) {
+                        /* If we are asked to make this parsed object sensitive, then let's apply this
+                         * immediately after allocating each variant, so that when we abort half-way
+                         * everything we already allocated that is then freed is correctly marked. */
+                        if (FLAGS_SET(flags, JSON_PARSE_SENSITIVE))
+                                json_variant_sensitive(add);
+
                         (void) json_variant_set_source(&add, source, line_token, column_token);
 
                         if (!GREEDY_REALLOC(current->elements, current->n_elements_allocated, current->n_elements + 1)) {
@@ -2825,15 +2832,15 @@ finish:
         return r;
 }
 
-int json_parse(const char *input, JsonVariant **ret, unsigned *ret_line, unsigned *ret_column) {
-        return json_parse_internal(&input, NULL, ret, ret_line, ret_column, false);
+int json_parse(const char *input, JsonParseFlags flags, JsonVariant **ret, unsigned *ret_line, unsigned *ret_column) {
+        return json_parse_internal(&input, NULL, flags, ret, ret_line, ret_column, false);
 }
 
-int json_parse_continue(const char **p, JsonVariant **ret, unsigned *ret_line, unsigned *ret_column) {
-        return json_parse_internal(p, NULL, ret, ret_line, ret_column, true);
+int json_parse_continue(const char **p, JsonParseFlags flags, JsonVariant **ret, unsigned *ret_line, unsigned *ret_column) {
+        return json_parse_internal(p, NULL, flags, ret, ret_line, ret_column, true);
 }
 
-int json_parse_file_at(FILE *f, int dir_fd, const char *path, JsonVariant **ret, unsigned *ret_line, unsigned *ret_column) {
+int json_parse_file_at(FILE *f, int dir_fd, const char *path, JsonParseFlags flags, JsonVariant **ret, unsigned *ret_line, unsigned *ret_column) {
         _cleanup_(json_source_unrefp) JsonSource *source = NULL;
         _cleanup_free_ char *text = NULL;
         const char *p;
@@ -2855,7 +2862,7 @@ int json_parse_file_at(FILE *f, int dir_fd, const char *path, JsonVariant **ret,
         }
 
         p = text;
-        return json_parse_internal(&p, source, ret, ret_line, ret_column, false);
+        return json_parse_internal(&p, source, flags, ret, ret_line, ret_column, false);
 }
 
 int json_buildv(JsonVariant **ret, va_list ap) {
@@ -3093,7 +3100,7 @@ int json_buildv(JsonVariant **ret, va_list ap) {
                                 /* Note that we don't care for current->n_suppress here, we should generate parsing
                                  * errors even in suppressed object properties */
 
-                                r = json_parse(l, &add, NULL, NULL);
+                                r = json_parse(l, 0, &add, NULL, NULL);
                                 if (r < 0)
                                         goto finish;
                         } else
diff --git a/src/shared/json.h b/src/shared/json.h
index dd1aff450a..d3e44d1b04 100644
--- a/src/shared/json.h
+++ b/src/shared/json.h
@@ -174,12 +174,16 @@ int json_variant_filter(JsonVariant **v, char **to_remove);
 
 int json_variant_set_field(JsonVariant **v, const char *field, JsonVariant *value);
 
-int json_parse(const char *string, JsonVariant **ret, unsigned *ret_line, unsigned *ret_column);
-int json_parse_continue(const char **p, JsonVariant **ret, unsigned *ret_line, unsigned *ret_column);
-int json_parse_file_at(FILE *f, int dir_fd, const char *path, JsonVariant **ret, unsigned *ret_line, unsigned *ret_column);
+typedef enum JsonParseFlags {
+        JSON_PARSE_SENSITIVE = 1 << 0, /* mark variant as "sensitive", i.e. something containing secret key material or such */
+} JsonParseFlags;
 
-static inline int json_parse_file(FILE *f, const char *path, JsonVariant **ret, unsigned *ret_line, unsigned *ret_column) {
-        return json_parse_file_at(f, AT_FDCWD, path, ret, ret_line, ret_column);
+int json_parse(const char *string, JsonParseFlags flags, JsonVariant **ret, unsigned *ret_line, unsigned *ret_column);
+int json_parse_continue(const char **p, JsonParseFlags flags, JsonVariant **ret, unsigned *ret_line, unsigned *ret_column);
+int json_parse_file_at(FILE *f, int dir_fd, const char *path, JsonParseFlags flags, JsonVariant **ret, unsigned *ret_line, unsigned *ret_column);
+
+static inline int json_parse_file(FILE *f, const char *path, JsonParseFlags flags, JsonVariant **ret, unsigned *ret_line, unsigned *ret_column) {
+        return json_parse_file_at(f, AT_FDCWD, path, flags, ret, ret_line, ret_column);
 }
 
 enum {
diff --git a/src/shared/varlink.c b/src/shared/varlink.c
index ee4fb9e843..e1d956e649 100644
--- a/src/shared/varlink.c
+++ b/src/shared/varlink.c
@@ -577,7 +577,7 @@ static int varlink_parse_message(Varlink *v) {
 
         varlink_log(v, "New incoming message: %s", begin);
 
-        r = json_parse(begin, &v->current, NULL, NULL);
+        r = json_parse(begin, 0, &v->current, NULL, NULL);
         if (r < 0)
                 return r;
 
diff --git a/src/test/test-json.c b/src/test/test-json.c
index a6613043b9..90b91f6310 100644
--- a/src/test/test-json.c
+++ b/src/test/test-json.c
@@ -82,7 +82,7 @@ static void test_variant(const char *data, Test test) {
         _cleanup_free_ char *s = NULL;
         int r;
 
-        r = json_parse(data, &v, NULL, NULL);
+        r = json_parse(data, 0, &v, NULL, NULL);
         assert_se(r == 0);
         assert_se(v);
 
@@ -93,7 +93,7 @@ static void test_variant(const char *data, Test test) {
 
         log_info("formatted normally: %s\n", s);
 
-        r = json_parse(data, &w, NULL, NULL);
+        r = json_parse(data, JSON_PARSE_SENSITIVE, &w, NULL, NULL);
         assert_se(r == 0);
         assert_se(w);
         assert_se(json_variant_has_type(v, json_variant_type(w)));
@@ -110,7 +110,7 @@ static void test_variant(const char *data, Test test) {
 
         log_info("formatted prettily:\n%s", s);
 
-        r = json_parse(data, &w, NULL, NULL);
+        r = json_parse(data, 0, &w, NULL, NULL);
         assert_se(r == 0);
         assert_se(w);
 
@@ -302,7 +302,7 @@ static void test_build(void) {
 
         assert_se(json_variant_format(a, 0, &s) >= 0);
         log_info("GOT: %s\n", s);
-        assert_se(json_parse(s, &b, NULL, NULL) >= 0);
+        assert_se(json_parse(s, 0, &b, NULL, NULL) >= 0);
         assert_se(json_variant_equal(a, b));
 
         a = json_variant_unref(a);
@@ -313,7 +313,7 @@ static void test_build(void) {
         s = mfree(s);
         assert_se(json_variant_format(a, 0, &s) >= 0);
         log_info("GOT: %s\n", s);
-        assert_se(json_parse(s, &b, NULL, NULL) >= 0);
+        assert_se(json_parse(s, 0, &b, NULL, NULL) >= 0);
         assert_se(json_variant_format(b, 0, &t) >= 0);
         log_info("GOT: %s\n", t);
 
@@ -365,7 +365,7 @@ static void test_source(void) {
 
         assert_se(f = fmemopen_unlocked((void*) data, strlen(data), "r"));
 
-        assert_se(json_parse_file(f, "waldo", &v, NULL, NULL) >= 0);
+        assert_se(json_parse_file(f, "waldo", 0, &v, NULL, NULL) >= 0);
 
         printf("--- non-pretty begin ---\n");
         json_variant_dump(v, 0, stdout, NULL);

From ba23dbf1ebd9ac347e43540ee682c2744c52f3c3 Mon Sep 17 00:00:00 2001
From: Lennart Poettering <lennart@poettering.net>
Date: Tue, 23 Apr 2019 15:25:42 +0200
Subject: [PATCH 11/29] json: optionally, make string checks stricter when
 dispatching strings

---
 src/shared/json.c | 9 +++++++++
 src/shared/json.h | 5 +++--
 2 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/src/shared/json.c b/src/shared/json.c
index 98fa067ef4..b3de2ad718 100644
--- a/src/shared/json.c
+++ b/src/shared/json.c
@@ -3611,6 +3611,9 @@ int json_dispatch_string(const char *name, JsonVariant *variant, JsonDispatchFla
         if (!json_variant_is_string(variant))
                 return json_log(variant, flags, SYNTHETIC_ERRNO(EINVAL), "JSON field '%s' is not a string.", strna(name));
 
+        if ((flags & JSON_SAFE) && !string_is_safe(json_variant_string(variant)))
+                return json_log(variant, flags, SYNTHETIC_ERRNO(EINVAL), "JSON field '%s' contains unsafe characters, refusing.", strna(name));
+
         r = free_and_strdup(s, json_variant_string(variant));
         if (r < 0)
                 return json_log(variant, flags, r, "Failed to allocate string: %m");
@@ -3634,6 +3637,9 @@ int json_dispatch_strv(const char *name, JsonVariant *variant, JsonDispatchFlags
 
         /* Let's be flexible here: accept a single string in place of a single-item array */
         if (json_variant_is_string(variant)) {
+                if ((flags & JSON_SAFE) && !string_is_safe(json_variant_string(variant)))
+                        return json_log(variant, flags, SYNTHETIC_ERRNO(EINVAL), "JSON field '%s' contains unsafe characters, refusing.", strna(name));
+
                 l = strv_new(json_variant_string(variant));
                 if (!l)
                         return log_oom();
@@ -3649,6 +3655,9 @@ int json_dispatch_strv(const char *name, JsonVariant *variant, JsonDispatchFlags
                 if (!json_variant_is_string(e))
                         return json_log(e, flags, SYNTHETIC_ERRNO(EINVAL), "JSON array element is not a string.");
 
+                if ((flags & JSON_SAFE) && !string_is_safe(json_variant_string(e)))
+                        return json_log(e, flags, SYNTHETIC_ERRNO(EINVAL), "JSON field '%s' contains unsafe characters, refusing.", strna(name));
+
                 r = strv_extend(&l, json_variant_string(e));
                 if (r < 0)
                         return json_log(e, flags, r, "Failed to append array element: %m");
diff --git a/src/shared/json.h b/src/shared/json.h
index d3e44d1b04..1ce0f90c29 100644
--- a/src/shared/json.h
+++ b/src/shared/json.h
@@ -230,10 +230,11 @@ typedef enum JsonDispatchFlags {
         JSON_PERMISSIVE = 1 << 0, /* Shall parsing errors be considered fatal for this property? */
         JSON_MANDATORY  = 1 << 1, /* Should existence of this property be mandatory? */
         JSON_LOG        = 1 << 2, /* Should the parser log about errors? */
+        JSON_SAFE       = 1 << 3, /* Don't accept "unsafe" strings in json_dispatch_string() + json_dispatch_string() */
 
         /* The following two may be passed into log_json() in addition to the three above */
-        JSON_DEBUG      = 1 << 3, /* Indicates that this log message is a debug message */
-        JSON_WARNING    = 1 << 4, /* Indicates that this log message is a warning message */
+        JSON_DEBUG      = 1 << 4, /* Indicates that this log message is a debug message */
+        JSON_WARNING    = 1 << 5, /* Indicates that this log message is a warning message */
 } JsonDispatchFlags;
 
 typedef int (*JsonDispatchCallback)(const char *name, JsonVariant *variant, JsonDispatchFlags flags, void *userdata);

From 22f14d6b0287dd83fb282631e44eb49c1dac725f Mon Sep 17 00:00:00 2001
From: Lennart Poettering <lennart@poettering.net>
Date: Tue, 23 Apr 2019 18:20:01 +0200
Subject: [PATCH 12/29] json: add json_variant_strv() helper that converts a
 json variant to an strv

Only works for arrays of strings, of course.
---
 src/shared/json.c | 60 +++++++++++++++++++++++++++++++++++++++++++++++
 src/shared/json.h |  2 ++
 2 files changed, 62 insertions(+)

diff --git a/src/shared/json.c b/src/shared/json.c
index b3de2ad718..e761037d34 100644
--- a/src/shared/json.c
+++ b/src/shared/json.c
@@ -1814,6 +1814,66 @@ int json_variant_set_field(JsonVariant **v, const char *field, JsonVariant *valu
         return 1;
 }
 
+int json_variant_strv(JsonVariant *v, char ***ret) {
+        char **l = NULL;
+        size_t n, i;
+        bool sensitive;
+        int r;
+
+        assert(ret);
+
+        if (!v || json_variant_is_null(v)) {
+                l = new0(char*, 1);
+                if (!l)
+                        return -ENOMEM;
+
+                *ret = l;
+                return 0;
+        }
+
+        if (!json_variant_is_array(v))
+                return -EINVAL;
+
+        sensitive = v->sensitive;
+
+        n = json_variant_elements(v);
+        l = new(char*, n+1);
+        if (!l)
+                return -ENOMEM;
+
+        for (i = 0; i < n; i++) {
+                JsonVariant *e;
+
+                assert_se(e = json_variant_by_index(v, i));
+                sensitive = sensitive || e->sensitive;
+
+                if (!json_variant_is_string(e)) {
+                        l[i] = NULL;
+                        r = -EINVAL;
+                        goto fail;
+                }
+
+                l[i] = strdup(json_variant_string(e));
+                if (!l[i]) {
+                        r = -ENOMEM;
+                        goto fail;
+                }
+        }
+
+        l[i] = NULL;
+        *ret = TAKE_PTR(l);
+
+        return 0;
+
+fail:
+        if (sensitive)
+                strv_free_erase(l);
+        else
+                strv_free(l);
+
+        return r;
+}
+
 static int json_variant_copy(JsonVariant **nv, JsonVariant *v) {
         JsonVariantType t;
         JsonVariant *c;
diff --git a/src/shared/json.h b/src/shared/json.h
index 1ce0f90c29..667c57c9f7 100644
--- a/src/shared/json.h
+++ b/src/shared/json.h
@@ -174,6 +174,8 @@ int json_variant_filter(JsonVariant **v, char **to_remove);
 
 int json_variant_set_field(JsonVariant **v, const char *field, JsonVariant *value);
 
+int json_variant_strv(JsonVariant *v, char ***ret);
+
 typedef enum JsonParseFlags {
         JSON_PARSE_SENSITIVE = 1 << 0, /* mark variant as "sensitive", i.e. something containing secret key material or such */
 } JsonParseFlags;

From a7f8c9ce60f9487b40f47189ea6a1cc590f3e188 Mon Sep 17 00:00:00 2001
From: Lennart Poettering <lennart@poettering.net>
Date: Thu, 25 Apr 2019 12:57:54 +0200
Subject: [PATCH 13/29] nspawn-oci: use new json_variant_strv() helper

---
 src/nspawn/nspawn-oci.c | 17 +++--------------
 1 file changed, 3 insertions(+), 14 deletions(-)

diff --git a/src/nspawn/nspawn-oci.c b/src/nspawn/nspawn-oci.c
index ba8b142c99..782c03c539 100644
--- a/src/nspawn/nspawn-oci.c
+++ b/src/nspawn/nspawn-oci.c
@@ -172,24 +172,13 @@ static int oci_env(const char *name, JsonVariant *v, JsonDispatchFlags flags, vo
 static int oci_args(const char *name, JsonVariant *v, JsonDispatchFlags flags, void *userdata) {
         _cleanup_strv_free_ char **l = NULL;
         char ***value = userdata;
-        JsonVariant *e;
         int r;
 
         assert(value);
 
-        JSON_VARIANT_ARRAY_FOREACH(e, v) {
-                const char *n;
-
-                if (!json_variant_is_string(e))
-                        return json_log(v, flags, SYNTHETIC_ERRNO(EINVAL),
-                                        "Argument is not a string.");
-
-                assert_se(n = json_variant_string(e));
-
-                r = strv_extend(&l, n);
-                if (r < 0)
-                        return log_oom();
-        }
+        r = json_variant_strv(v, &l);
+        if (r < 0)
+                return json_log(v, flags, r, "Cannot parse arguments as list of strings: %m");
 
         if (strv_isempty(l))
                 return json_log(v, flags, SYNTHETIC_ERRNO(EINVAL),

From 15f1fb3e3e3fd603c245d97c62daa5292b289789 Mon Sep 17 00:00:00 2001
From: Lennart Poettering <lennart@poettering.net>
Date: Thu, 4 Jul 2019 17:42:00 +0200
Subject: [PATCH 14/29] json: add json_variant_set_field_string() and
 json_variant_set_field_unsigned()

---
 src/shared/json.c | 22 ++++++++++++++++++++++
 src/shared/json.h |  2 ++
 2 files changed, 24 insertions(+)

diff --git a/src/shared/json.c b/src/shared/json.c
index e761037d34..294e4335ee 100644
--- a/src/shared/json.c
+++ b/src/shared/json.c
@@ -1814,6 +1814,28 @@ int json_variant_set_field(JsonVariant **v, const char *field, JsonVariant *valu
         return 1;
 }
 
+int json_variant_set_field_string(JsonVariant **v, const char *field, const char *value) {
+        _cleanup_(json_variant_unrefp) JsonVariant *m = NULL;
+        int r;
+
+        r = json_variant_new_string(&m, value);
+        if (r < 0)
+                return r;
+
+        return json_variant_set_field(v, field, m);
+}
+
+int json_variant_set_field_unsigned(JsonVariant **v, const char *field, uintmax_t u) {
+        _cleanup_(json_variant_unrefp) JsonVariant *m = NULL;
+        int r;
+
+        r = json_variant_new_unsigned(&m, u);
+        if (r < 0)
+                return r;
+
+        return json_variant_set_field(v, field, m);
+}
+
 int json_variant_strv(JsonVariant *v, char ***ret) {
         char **l = NULL;
         size_t n, i;
diff --git a/src/shared/json.h b/src/shared/json.h
index 667c57c9f7..41bdb95cc0 100644
--- a/src/shared/json.h
+++ b/src/shared/json.h
@@ -173,6 +173,8 @@ void json_variant_dump(JsonVariant *v, JsonFormatFlags flags, FILE *f, const cha
 int json_variant_filter(JsonVariant **v, char **to_remove);
 
 int json_variant_set_field(JsonVariant **v, const char *field, JsonVariant *value);
+int json_variant_set_field_string(JsonVariant **v, const char *field, const char *value);
+int json_variant_set_field_unsigned(JsonVariant **v, const char *field, uintmax_t value);
 
 int json_variant_strv(JsonVariant *v, char ***ret);
 

From ca409a59c8eedb57692bceef00bfb248dbd92a76 Mon Sep 17 00:00:00 2001
From: Lennart Poettering <lennart@poettering.net>
Date: Thu, 4 Jul 2019 17:42:32 +0200
Subject: [PATCH 15/29] json: add json_variant_merge() helper

---
 src/shared/json.c | 63 +++++++++++++++++++++++++++++++++++++++++++++++
 src/shared/json.h |  2 ++
 2 files changed, 65 insertions(+)

diff --git a/src/shared/json.c b/src/shared/json.c
index 294e4335ee..6e2f3e1c32 100644
--- a/src/shared/json.c
+++ b/src/shared/json.c
@@ -1836,6 +1836,69 @@ int json_variant_set_field_unsigned(JsonVariant **v, const char *field, uintmax_
         return json_variant_set_field(v, field, m);
 }
 
+int json_variant_merge(JsonVariant **v, JsonVariant *m) {
+        _cleanup_(json_variant_unrefp) JsonVariant *w = NULL;
+        _cleanup_free_ JsonVariant **array = NULL;
+        size_t v_elements, m_elements, i, k;
+        bool v_blank, m_blank;
+        int r;
+
+        m = json_variant_dereference(m);
+
+        v_blank = json_variant_is_blank_object(*v);
+        m_blank = json_variant_is_blank_object(m);
+
+        if (!v_blank && !json_variant_is_object(*v))
+                return -EINVAL;
+        if (!m_blank && !json_variant_is_object(m))
+                return -EINVAL;
+
+        if (m_blank)
+                return 0; /* nothing to do */
+
+        if (v_blank) {
+                json_variant_unref(*v);
+                *v = json_variant_ref(m);
+                return 1;
+        }
+
+        v_elements = json_variant_elements(*v);
+        m_elements = json_variant_elements(m);
+        if (v_elements > SIZE_MAX - m_elements) /* overflow check */
+                return -ENOMEM;
+
+        array = new(JsonVariant*, v_elements + m_elements);
+        if (!array)
+                return -ENOMEM;
+
+        k = 0;
+        for (i = 0; i < v_elements; i += 2) {
+                JsonVariant *u;
+
+                u = json_variant_by_index(*v, i);
+                if (!json_variant_is_string(u))
+                        return -EINVAL;
+
+                if (json_variant_by_key(m, json_variant_string(u)))
+                        continue; /* skip if exists in second variant */
+
+                array[k++] = u;
+                array[k++] = json_variant_by_index(*v, i + 1);
+        }
+
+        for (i = 0; i < m_elements; i++)
+                array[k++] = json_variant_by_index(m, i);
+
+        r = json_variant_new_object(&w, array, k);
+        if (r < 0)
+                return r;
+
+        json_variant_unref(*v);
+        *v = TAKE_PTR(w);
+
+        return 1;
+}
+
 int json_variant_strv(JsonVariant *v, char ***ret) {
         char **l = NULL;
         size_t n, i;
diff --git a/src/shared/json.h b/src/shared/json.h
index 41bdb95cc0..9ad08c625f 100644
--- a/src/shared/json.h
+++ b/src/shared/json.h
@@ -176,6 +176,8 @@ int json_variant_set_field(JsonVariant **v, const char *field, JsonVariant *valu
 int json_variant_set_field_string(JsonVariant **v, const char *field, const char *value);
 int json_variant_set_field_unsigned(JsonVariant **v, const char *field, uintmax_t value);
 
+int json_variant_merge(JsonVariant **v, JsonVariant *m);
+
 int json_variant_strv(JsonVariant *v, char ***ret);
 
 typedef enum JsonParseFlags {

From b7fc90a2e63f6a90cd94b7fcd21997bb647eb454 Mon Sep 17 00:00:00 2001
From: Lennart Poettering <lennart@poettering.net>
Date: Wed, 15 May 2019 16:55:04 +0200
Subject: [PATCH 16/29] json: add concept of normalization

Let's add a concept of normalization: as preparation for signing json
records let's add a mechanism to bring JSON records into a well-defined
order so that we can safely validate JSON records.

This adds two booleans to each JsonVariant object: "sorted" and
"normalized". The latter indicates whether a variant is fully sorted
(i.e. all keys of objects listed in alphabetical order) recursively down
the tree. The former is a weaker property: it only checks whether the
keys of the object itself are sorted. All variants which are
"normalized" are also "sorted", but not vice versa.

The knowledge of the "sorted" property is then used to optimize
searching for keys in the variant by using bisection.

Both properties are determined at the moment the variants are allocated.
Since our objects are immutable this is safe.
---
 src/shared/json.c    | 237 +++++++++++++++++++++++++++++++++++++++----
 src/shared/json.h    |   5 +
 src/test/test-json.c |  92 ++++++++++++++++-
 3 files changed, 314 insertions(+), 20 deletions(-)

diff --git a/src/shared/json.c b/src/shared/json.c
index 6e2f3e1c32..a5bf525c87 100644
--- a/src/shared/json.c
+++ b/src/shared/json.c
@@ -71,9 +71,15 @@ struct JsonVariant {
         /* While comparing two arrays, we use this for marking what we already have seen */
         bool is_marked:1;
 
-        /* Ersase from memory when freeing */
+        /* Erase from memory when freeing */
         bool sensitive:1;
 
+        /* If this is an object the fields are strictly ordered by name */
+        bool sorted:1;
+
+        /* If in addition to this object all objects referenced by it are also ordered strictly by name */
+        bool normalized:1;
+
         /* The current 'depth' of the JsonVariant, i.e. how many levels of member variants this has */
         uint16_t depth;
 
@@ -215,10 +221,10 @@ static uint16_t json_variant_depth(JsonVariant *v) {
         return v->depth;
 }
 
-static JsonVariant *json_variant_normalize(JsonVariant *v) {
+static JsonVariant *json_variant_formalize(JsonVariant *v) {
 
-        /* Converts json variants to their normalized form, i.e. fully dereferenced and wherever possible converted to
-         * the "magic" version if there is one */
+        /* Converts json variant pointers to their normalized form, i.e. fully dereferenced and wherever
+         * possible converted to the "magic" version if there is one */
 
         if (!v)
                 return NULL;
@@ -259,9 +265,9 @@ static JsonVariant *json_variant_normalize(JsonVariant *v) {
         }
 }
 
-static JsonVariant *json_variant_conservative_normalize(JsonVariant *v) {
+static JsonVariant *json_variant_conservative_formalize(JsonVariant *v) {
 
-        /* Much like json_variant_normalize(), but won't simplify if the variant has a source/line location attached to
+        /* Much like json_variant_formalize(), but won't simplify if the variant has a source/line location attached to
          * it, in order not to lose context */
 
         if (!v)
@@ -273,7 +279,7 @@ static JsonVariant *json_variant_conservative_normalize(JsonVariant *v) {
         if (v->source || v->line > 0 || v->column > 0)
                 return v;
 
-        return json_variant_normalize(v);
+        return json_variant_formalize(v);
 }
 
 static int json_variant_new(JsonVariant **ret, JsonVariantType type, size_t space) {
@@ -451,7 +457,7 @@ static void json_variant_set(JsonVariant *a, JsonVariant *b) {
         case JSON_VARIANT_ARRAY:
         case JSON_VARIANT_OBJECT:
                 a->is_reference = true;
-                a->reference = json_variant_ref(json_variant_conservative_normalize(b));
+                a->reference = json_variant_ref(json_variant_conservative_formalize(b));
                 break;
 
         case JSON_VARIANT_NULL:
@@ -476,6 +482,7 @@ static void json_variant_copy_source(JsonVariant *v, JsonVariant *from) {
 
 int json_variant_new_array(JsonVariant **ret, JsonVariant **array, size_t n) {
         _cleanup_(json_variant_unrefp) JsonVariant *v = NULL;
+        bool normalized = true;
 
         assert_return(ret, -EINVAL);
         if (n == 0) {
@@ -511,8 +518,13 @@ int json_variant_new_array(JsonVariant **ret, JsonVariant **array, size_t n) {
 
                 json_variant_set(w, c);
                 json_variant_copy_source(w, c);
+
+                if (!json_variant_is_normalized(c))
+                        normalized = false;
         }
 
+        v->normalized = normalized;
+
         *ret = TAKE_PTR(v);
         return 0;
 }
@@ -550,6 +562,8 @@ int json_variant_new_array_bytes(JsonVariant **ret, const void *p, size_t n) {
                 };
         }
 
+        v->normalized = true;
+
         *ret = v;
         return 0;
 }
@@ -601,12 +615,16 @@ int json_variant_new_array_strv(JsonVariant **ret, char **l) {
                         memcpy(w->string, l[v->n_elements], k+1);
         }
 
+        v->normalized = true;
+
         *ret = TAKE_PTR(v);
         return 0;
 }
 
 int json_variant_new_object(JsonVariant **ret, JsonVariant **array, size_t n) {
         _cleanup_(json_variant_unrefp) JsonVariant *v = NULL;
+        const char *prev = NULL;
+        bool sorted = true, normalized = true;
 
         assert_return(ret, -EINVAL);
         if (n == 0) {
@@ -630,9 +648,20 @@ int json_variant_new_object(JsonVariant **ret, JsonVariant **array, size_t n) {
                         *c = array[v->n_elements];
                 uint16_t d;
 
-                if ((v->n_elements & 1) == 0 &&
-                    !json_variant_is_string(c))
-                        return -EINVAL; /* Every second one needs to be a string, as it is the key name */
+                if ((v->n_elements & 1) == 0) {
+                        const char *k;
+
+                        if (!json_variant_is_string(c))
+                                return -EINVAL; /* Every second one needs to be a string, as it is the key name */
+
+                        assert_se(k = json_variant_string(c));
+
+                        if (prev && strcmp(k, prev) <= 0)
+                                sorted = normalized = false;
+
+                        prev = k;
+                } else if (!json_variant_is_normalized(c))
+                        normalized = false;
 
                 d = json_variant_depth(c);
                 if (d >= DEPTH_MAX) /* Refuse too deep nesting */
@@ -649,6 +678,9 @@ int json_variant_new_object(JsonVariant **ret, JsonVariant **array, size_t n) {
                 json_variant_copy_source(w, c);
         }
 
+        v->normalized = normalized;
+        v->sorted = sorted;
+
         *ret = TAKE_PTR(v);
         return 0;
 }
@@ -1131,7 +1163,7 @@ JsonVariant *json_variant_by_index(JsonVariant *v, size_t idx) {
         if (idx >= v->n_elements)
                 return NULL;
 
-        return json_variant_conservative_normalize(v + 1 + idx);
+        return json_variant_conservative_formalize(v + 1 + idx);
 
 mismatch:
         log_debug("Element in non-array/non-object JSON variant requested by index, returning NULL.");
@@ -1154,6 +1186,37 @@ JsonVariant *json_variant_by_key_full(JsonVariant *v, const char *key, JsonVaria
         if (v->is_reference)
                 return json_variant_by_key(v->reference, key);
 
+        if (v->sorted) {
+                size_t a = 0, b = v->n_elements/2;
+
+                /* If the variant is sorted we can use bisection to find the entry we need in O(log(n)) time */
+
+                while (b > a) {
+                        JsonVariant *p;
+                        const char *f;
+                        int c;
+
+                        i = (a + b) / 2;
+                        p = json_variant_dereference(v + 1 + i*2);
+
+                        assert_se(f = json_variant_string(p));
+
+                        c = strcmp(key, f);
+                        if (c == 0) {
+                                if (ret_key)
+                                        *ret_key = json_variant_conservative_formalize(v + 1 + i*2);
+
+                                return json_variant_conservative_formalize(v + 1 + i*2 + 1);
+                        } else if (c < 0)
+                                b = i;
+                        else
+                                a = i + 1;
+                }
+
+                goto not_found;
+        }
+
+        /* The variant is not sorted, hence search for the field linearly */
         for (i = 0; i < v->n_elements; i += 2) {
                 JsonVariant *p;
 
@@ -1165,9 +1228,9 @@ JsonVariant *json_variant_by_key_full(JsonVariant *v, const char *key, JsonVaria
                 if (streq(json_variant_string(p), key)) {
 
                         if (ret_key)
-                                *ret_key = json_variant_conservative_normalize(v + 1 + i);
+                                *ret_key = json_variant_conservative_formalize(v + 1 + i);
 
-                        return json_variant_conservative_normalize(v + 1 + i + 1);
+                        return json_variant_conservative_formalize(v + 1 + i + 1);
                 }
         }
 
@@ -1192,8 +1255,8 @@ JsonVariant *json_variant_by_key(JsonVariant *v, const char *key) {
 bool json_variant_equal(JsonVariant *a, JsonVariant *b) {
         JsonVariantType t;
 
-        a = json_variant_normalize(a);
-        b = json_variant_normalize(b);
+        a = json_variant_formalize(a);
+        b = json_variant_formalize(b);
 
         if (a == b)
                 return true;
@@ -1305,7 +1368,7 @@ void json_variant_sensitive(JsonVariant *v) {
          * flag to all contained variants. And if those are then destroyed this is propagated further down,
          * and so on. */
 
-        v = json_variant_normalize(v);
+        v = json_variant_formalize(v);
         if (!json_variant_is_regular(v))
                 return;
 
@@ -2022,7 +2085,7 @@ static int json_variant_copy(JsonVariant **nv, JsonVariant *v) {
                 c->n_ref = 1;
                 c->type = t;
                 c->is_reference = true;
-                c->reference = json_variant_ref(json_variant_normalize(v));
+                c->reference = json_variant_ref(json_variant_formalize(v));
 
                 *nv = c;
                 return 0;
@@ -3824,6 +3887,144 @@ int json_dispatch_variant(const char *name, JsonVariant *variant, JsonDispatchFl
         return 0;
 }
 
+static int json_cmp_strings(const void *x, const void *y) {
+        JsonVariant *const *a = x, *const *b = y;
+
+        if (!json_variant_is_string(*a) || !json_variant_is_string(*b))
+                return CMP(*a, *b);
+
+        return strcmp(json_variant_string(*a), json_variant_string(*b));
+}
+
+int json_variant_sort(JsonVariant **v) {
+        _cleanup_free_ JsonVariant **a = NULL;
+        JsonVariant *n = NULL;
+        size_t i, m;
+        int r;
+
+        assert(v);
+
+        if (json_variant_is_sorted(*v))
+                return 0;
+
+        if (!json_variant_is_object(*v))
+                return -EMEDIUMTYPE;
+
+        /* Sorts they key/value pairs in an object variant */
+
+        m = json_variant_elements(*v);
+        a = new(JsonVariant*, m);
+        if (!a)
+                return -ENOMEM;
+
+        for (i = 0; i < m; i++)
+                a[i] = json_variant_by_index(*v, i);
+
+        qsort(a, m/2, sizeof(JsonVariant*)*2, json_cmp_strings);
+
+        r = json_variant_new_object(&n, a, m);
+        if (r < 0)
+                return r;
+        if (!n->sorted) /* Check if this worked. This will fail if there are multiple identical keys used. */
+                return -ENOTUNIQ;
+
+        json_variant_unref(*v);
+        *v = n;
+
+        return 1;
+}
+
+int json_variant_normalize(JsonVariant **v) {
+        _cleanup_free_ JsonVariant **a = NULL;
+        JsonVariant *n = NULL;
+        size_t i, j, m;
+        int r;
+
+        assert(v);
+
+        if (json_variant_is_normalized(*v))
+                return 0;
+
+        if (!json_variant_is_object(*v) && !json_variant_is_array(*v))
+                return -EMEDIUMTYPE;
+
+        /* Sorts the key/value pairs in an object variant anywhere down the tree in the specified variant */
+
+        m = json_variant_elements(*v);
+        a = new(JsonVariant*, m);
+        if (!a)
+                return -ENOMEM;
+
+        for (i = 0; i < m; i++) {
+                a[i] = json_variant_ref(json_variant_by_index(*v, i));
+
+                r = json_variant_normalize(a + i);
+                if (r < 0)
+                        goto finish;
+        }
+
+        qsort(a, m/2, sizeof(JsonVariant*)*2, json_cmp_strings);
+
+        if (json_variant_is_object(*v))
+                r = json_variant_new_object(&n, a, m);
+        else {
+                assert(json_variant_is_array(*v));
+                r = json_variant_new_array(&n, a, m);
+        }
+        if (r < 0)
+                goto finish;
+        if (!n->normalized) { /* Let's see if normalization worked. It will fail if there are multiple
+                               * identical keys used in the same object anywhere, or if there are floating
+                               * point numbers used (see below) */
+                r = -ENOTUNIQ;
+                goto finish;
+        }
+
+        json_variant_unref(*v);
+        *v = n;
+
+        r = 1;
+
+finish:
+        for (j = 0; j < i; j++)
+                json_variant_unref(a[j]);
+
+        return r;
+}
+
+bool json_variant_is_normalized(JsonVariant *v) {
+
+        /* For now, let's consider anything containing numbers not expressible as integers as
+         * non-normalized. That's because we cannot sensibly compare them due to accuracy issues, nor even
+         * store them if they are too large. */
+        if (json_variant_is_real(v) && !json_variant_is_integer(v) && !json_variant_is_unsigned(v))
+                return false;
+
+        /* The concept only applies to variants that include other variants, i.e. objects and arrays. All
+         * others are normalized anyway. */
+        if (!json_variant_is_object(v) && !json_variant_is_array(v))
+                return true;
+
+        /* Empty objects/arrays don't include any other variant, hence are always normalized too */
+        if (json_variant_elements(v) == 0)
+                return true;
+
+        return v->normalized; /* For everything else there's an explicit boolean we maintain */
+}
+
+bool json_variant_is_sorted(JsonVariant *v) {
+
+        /* Returns true if all key/value pairs of an object are properly sorted. Note that this only applies
+         * to objects, not arrays. */
+
+        if (!json_variant_is_object(v))
+                return true;
+        if (json_variant_elements(v) <= 1)
+                return true;
+
+        return v->sorted;
+}
+
 static const char* const json_variant_type_table[_JSON_VARIANT_TYPE_MAX] = {
         [JSON_VARIANT_STRING] = "string",
         [JSON_VARIANT_INTEGER] = "integer",
diff --git a/src/shared/json.h b/src/shared/json.h
index 9ad08c625f..432a66fe41 100644
--- a/src/shared/json.h
+++ b/src/shared/json.h
@@ -123,6 +123,8 @@ static inline bool json_variant_is_null(JsonVariant *v) {
 bool json_variant_is_negative(JsonVariant *v);
 bool json_variant_is_blank_object(JsonVariant *v);
 bool json_variant_is_blank_array(JsonVariant *v);
+bool json_variant_is_normalized(JsonVariant *v);
+bool json_variant_is_sorted(JsonVariant *v);
 
 size_t json_variant_elements(JsonVariant *v);
 JsonVariant *json_variant_by_index(JsonVariant *v, size_t index);
@@ -180,6 +182,9 @@ int json_variant_merge(JsonVariant **v, JsonVariant *m);
 
 int json_variant_strv(JsonVariant *v, char ***ret);
 
+int json_variant_sort(JsonVariant **v);
+int json_variant_normalize(JsonVariant **v);
+
 typedef enum JsonParseFlags {
         JSON_PARSE_SENSITIVE = 1 << 0, /* mark variant as "sensitive", i.e. something containing secret key material or such */
 } JsonParseFlags;
diff --git a/src/test/test-json.c b/src/test/test-json.c
index 90b91f6310..c72cd74274 100644
--- a/src/test/test-json.c
+++ b/src/test/test-json.c
@@ -415,6 +415,93 @@ static void test_depth(void) {
         fputs("\n", stdout);
 }
 
+static void test_normalize(void) {
+        _cleanup_(json_variant_unrefp) JsonVariant *v = NULL, *w = NULL;
+        _cleanup_free_ char *t = NULL;
+
+        assert_se(json_build(&v, JSON_BUILD_OBJECT(
+                                             JSON_BUILD_PAIR("b", JSON_BUILD_STRING("x")),
+                                             JSON_BUILD_PAIR("c", JSON_BUILD_STRING("y")),
+                                             JSON_BUILD_PAIR("a", JSON_BUILD_STRING("z")))) >= 0);
+
+        assert_se(!json_variant_is_sorted(v));
+        assert_se(!json_variant_is_normalized(v));
+
+        assert_se(json_variant_format(v, 0, &t) >= 0);
+        assert_se(streq(t, "{\"b\":\"x\",\"c\":\"y\",\"a\":\"z\"}"));
+        t = mfree(t);
+
+        assert_se(json_build(&w, JSON_BUILD_OBJECT(
+                                             JSON_BUILD_PAIR("bar", JSON_BUILD_STRING("zzz")),
+                                             JSON_BUILD_PAIR("foo", JSON_BUILD_VARIANT(v)))) >= 0);
+
+        assert_se(json_variant_is_sorted(w));
+        assert_se(!json_variant_is_normalized(w));
+
+        assert_se(json_variant_format(w, 0, &t) >= 0);
+        assert_se(streq(t, "{\"bar\":\"zzz\",\"foo\":{\"b\":\"x\",\"c\":\"y\",\"a\":\"z\"}}"));
+        t = mfree(t);
+
+        assert_se(json_variant_sort(&v) >= 0);
+        assert_se(json_variant_is_sorted(v));
+        assert_se(json_variant_is_normalized(v));
+
+        assert_se(json_variant_format(v, 0, &t) >= 0);
+        assert_se(streq(t, "{\"a\":\"z\",\"b\":\"x\",\"c\":\"y\"}"));
+        t = mfree(t);
+
+        assert_se(json_variant_normalize(&w) >= 0);
+        assert_se(json_variant_is_sorted(w));
+        assert_se(json_variant_is_normalized(w));
+
+        assert_se(json_variant_format(w, 0, &t) >= 0);
+        assert_se(streq(t, "{\"bar\":\"zzz\",\"foo\":{\"a\":\"z\",\"b\":\"x\",\"c\":\"y\"}}"));
+        t = mfree(t);
+}
+
+static void test_bisect(void) {
+        _cleanup_(json_variant_unrefp) JsonVariant *v = NULL;
+        char c;
+
+        /* Tests the bisection logic in json_variant_by_key() */
+
+        for (c = 'z'; c >= 'a'; c--) {
+
+                if ((c % 3) == 0)
+                        continue;
+
+                _cleanup_(json_variant_unrefp) JsonVariant *w = NULL;
+                assert_se(json_variant_new_stringn(&w, (char[4]) { '<', c, c, '>' }, 4) >= 0);
+                assert_se(json_variant_set_field(&v, (char[2]) { c, 0 }, w) >= 0);
+        }
+
+        json_variant_dump(v, JSON_FORMAT_COLOR|JSON_FORMAT_PRETTY, NULL, NULL);
+
+        assert_se(!json_variant_is_sorted(v));
+        assert_se(!json_variant_is_normalized(v));
+        assert_se(json_variant_normalize(&v) >= 0);
+        assert_se(json_variant_is_sorted(v));
+        assert_se(json_variant_is_normalized(v));
+
+        json_variant_dump(v, JSON_FORMAT_COLOR|JSON_FORMAT_PRETTY, NULL, NULL);
+
+        for (c = 'a'; c <= 'z'; c++) {
+                JsonVariant *k;
+                const char *z;
+
+                k = json_variant_by_key(v, (char[2]) { c, 0 });
+                assert_se(!k == ((c % 3) == 0));
+
+                if (!k)
+                        continue;
+
+                assert_se(json_variant_is_string(k));
+
+                z = (char[5]){ '<', c, c, '>', 0};
+                assert_se(streq(json_variant_string(k), z));
+        }
+}
+
 int main(int argc, char *argv[]) {
         test_setup_logging(LOG_DEBUG);
 
@@ -462,10 +549,11 @@ int main(int argc, char *argv[]) {
         test_variant("[ 0, -0, 0.0, -0.0, 0.000, -0.000, 0e0, -0e0, 0e+0, -0e-0, 0e-0, -0e000, 0e+000 ]", test_zeroes);
 
         test_build();
-
         test_source();
-
         test_depth();
 
+        test_normalize();
+        test_bisect();
+
         return 0;
 }

From cc164891da2922c52028420197cb0408c184378e Mon Sep 17 00:00:00 2001
From: Lennart Poettering <lennart@poettering.net>
Date: Thu, 4 Jul 2019 17:56:00 +0200
Subject: [PATCH 17/29] json: add new helper json_variant_new_base64()

---
 src/shared/json.c | 14 ++++++++++++++
 src/shared/json.h |  1 +
 2 files changed, 15 insertions(+)

diff --git a/src/shared/json.c b/src/shared/json.c
index a5bf525c87..7ea1313970 100644
--- a/src/shared/json.c
+++ b/src/shared/json.c
@@ -411,6 +411,20 @@ int json_variant_new_stringn(JsonVariant **ret, const char *s, size_t n) {
         return 0;
 }
 
+int json_variant_new_base64(JsonVariant **ret, const void *p, size_t n) {
+        _cleanup_free_ char *s = NULL;
+        ssize_t k;
+
+        assert_return(ret, -EINVAL);
+        assert_return(n == 0 || p, -EINVAL);
+
+        k = base64mem(p, n, &s);
+        if (k < 0)
+                return k;
+
+        return json_variant_new_stringn(ret, s, k);
+}
+
 static void json_variant_set(JsonVariant *a, JsonVariant *b) {
         assert(a);
 
diff --git a/src/shared/json.h b/src/shared/json.h
index 432a66fe41..e95e3baed1 100644
--- a/src/shared/json.h
+++ b/src/shared/json.h
@@ -55,6 +55,7 @@ typedef enum JsonVariantType {
 } JsonVariantType;
 
 int json_variant_new_stringn(JsonVariant **ret, const char *s, size_t n);
+int json_variant_new_base64(JsonVariant **ret, const void *p, size_t n);
 int json_variant_new_integer(JsonVariant **ret, intmax_t i);
 int json_variant_new_unsigned(JsonVariant **ret, uintmax_t u);
 int json_variant_new_real(JsonVariant **ret, long double d);

From 21e2151107712b07e000de79037e786fe7f21482 Mon Sep 17 00:00:00 2001
From: Lennart Poettering <lennart@poettering.net>
Date: Thu, 4 Jul 2019 17:56:16 +0200
Subject: [PATCH 18/29] json: add new helper json_variant_append_array()

---
 src/shared/json.c | 43 +++++++++++++++++++++++++++++++++++++++++++
 src/shared/json.h |  2 ++
 2 files changed, 45 insertions(+)

diff --git a/src/shared/json.c b/src/shared/json.c
index 7ea1313970..3ccda282cf 100644
--- a/src/shared/json.c
+++ b/src/shared/json.c
@@ -1976,6 +1976,49 @@ int json_variant_merge(JsonVariant **v, JsonVariant *m) {
         return 1;
 }
 
+int json_variant_append_array(JsonVariant **v, JsonVariant *element) {
+        _cleanup_(json_variant_unrefp) JsonVariant *nv = NULL;
+        bool blank;
+        int r;
+
+        assert(v);
+        assert(element);
+
+
+        if (!*v || json_variant_is_null(*v))
+                blank = true;
+        else if (!json_variant_is_array(*v))
+                return -EINVAL;
+        else
+                blank = json_variant_elements(*v) == 0;
+
+        if (blank)
+                r = json_variant_new_array(&nv, (JsonVariant*[]) { element }, 1);
+        else {
+                _cleanup_free_ JsonVariant **array = NULL;
+                size_t i;
+
+                array = new(JsonVariant*, json_variant_elements(*v) + 1);
+                if (!array)
+                        return -ENOMEM;
+
+                for (i = 0; i < json_variant_elements(*v); i++)
+                        array[i] = json_variant_by_index(*v, i);
+
+                array[i] = element;
+
+                r = json_variant_new_array(&nv, array, i + 1);
+        }
+
+        if (r < 0)
+                return r;
+
+        json_variant_unref(*v);
+        *v = TAKE_PTR(nv);
+
+        return 0;
+}
+
 int json_variant_strv(JsonVariant *v, char ***ret) {
         char **l = NULL;
         size_t n, i;
diff --git a/src/shared/json.h b/src/shared/json.h
index e95e3baed1..c1999a7411 100644
--- a/src/shared/json.h
+++ b/src/shared/json.h
@@ -179,6 +179,8 @@ int json_variant_set_field(JsonVariant **v, const char *field, JsonVariant *valu
 int json_variant_set_field_string(JsonVariant **v, const char *field, const char *value);
 int json_variant_set_field_unsigned(JsonVariant **v, const char *field, uintmax_t value);
 
+int json_variant_append_array(JsonVariant **v, JsonVariant *element);
+
 int json_variant_merge(JsonVariant **v, JsonVariant *m);
 
 int json_variant_strv(JsonVariant *v, char ***ret);

From 886b0c93a8bc5461ebaa9d129545d449a3b7c8ff Mon Sep 17 00:00:00 2001
From: Lennart Poettering <lennart@poettering.net>
Date: Thu, 4 Jul 2019 17:56:42 +0200
Subject: [PATCH 19/29] json: allow putting together base64 fields with
 json_build()

---
 src/shared/json.c | 30 ++++++++++++++++++++++++++++++
 src/shared/json.h |  2 ++
 2 files changed, 32 insertions(+)

diff --git a/src/shared/json.c b/src/shared/json.c
index 3ccda282cf..5d3c6aad91 100644
--- a/src/shared/json.c
+++ b/src/shared/json.c
@@ -3462,6 +3462,36 @@ int json_buildv(JsonVariant **ret, va_list ap) {
                         break;
                 }
 
+                case _JSON_BUILD_BASE64: {
+                        const void *p;
+                        size_t n;
+
+                        if (!IN_SET(current->expect, EXPECT_TOPLEVEL, EXPECT_OBJECT_VALUE, EXPECT_ARRAY_ELEMENT)) {
+                                r = -EINVAL;
+                                goto finish;
+                        }
+
+                        p = va_arg(ap, const void *);
+                        n = va_arg(ap, size_t);
+
+                        if (current->n_suppress == 0) {
+                                r = json_variant_new_base64(&add, p, n);
+                                if (r < 0)
+                                        goto finish;
+                        }
+
+                        n_subtract = 1;
+
+                        if (current->expect == EXPECT_TOPLEVEL)
+                                current->expect = EXPECT_END;
+                        else if (current->expect == EXPECT_OBJECT_VALUE)
+                                current->expect = EXPECT_OBJECT_KEY;
+                        else
+                                assert(current->expect == EXPECT_ARRAY_ELEMENT);
+
+                        break;
+                }
+
                 case _JSON_BUILD_OBJECT_BEGIN:
 
                         if (!IN_SET(current->expect, EXPECT_TOPLEVEL, EXPECT_OBJECT_VALUE, EXPECT_ARRAY_ELEMENT)) {
diff --git a/src/shared/json.h b/src/shared/json.h
index c1999a7411..9bb3e8cd78 100644
--- a/src/shared/json.h
+++ b/src/shared/json.h
@@ -216,6 +216,7 @@ enum {
         _JSON_BUILD_VARIANT,
         _JSON_BUILD_LITERAL,
         _JSON_BUILD_STRV,
+        _JSON_BUILD_BASE64,
         _JSON_BUILD_MAX,
 };
 
@@ -232,6 +233,7 @@ enum {
 #define JSON_BUILD_VARIANT(v) _JSON_BUILD_VARIANT, ({ JsonVariant *_x = v; _x; })
 #define JSON_BUILD_LITERAL(l) _JSON_BUILD_LITERAL, ({ const char *_x = l; _x; })
 #define JSON_BUILD_STRV(l) _JSON_BUILD_STRV, ({ char **_x = l; _x; })
+#define JSON_BUILD_BASE64(p, n) _JSON_BUILD_BASE64, ({ const void *_x = p; _x; }), ({ size_t _y = n; _y; })
 
 int json_build(JsonVariant **ret, ...);
 int json_buildv(JsonVariant **ret, va_list ap);

From aafa52ab8370e87ff98834a8ca00a2cbdc69370d Mon Sep 17 00:00:00 2001
From: Lennart Poettering <lennart@poettering.net>
Date: Thu, 23 May 2019 20:59:04 +0200
Subject: [PATCH 20/29] json: add ability to generate empty arrays/objects in
 json builder

---
 src/shared/json.h | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/shared/json.h b/src/shared/json.h
index 9bb3e8cd78..abcf17e86d 100644
--- a/src/shared/json.h
+++ b/src/shared/json.h
@@ -226,7 +226,9 @@ enum {
 #define JSON_BUILD_REAL(d) _JSON_BUILD_REAL, ({ long double _x = d; _x; })
 #define JSON_BUILD_BOOLEAN(b) _JSON_BUILD_BOOLEAN, ({ bool _x = b; _x; })
 #define JSON_BUILD_ARRAY(...) _JSON_BUILD_ARRAY_BEGIN, __VA_ARGS__, _JSON_BUILD_ARRAY_END
+#define JSON_BUILD_EMPTY_ARRAY _JSON_BUILD_ARRAY_BEGIN, _JSON_BUILD_ARRAY_END
 #define JSON_BUILD_OBJECT(...) _JSON_BUILD_OBJECT_BEGIN, __VA_ARGS__, _JSON_BUILD_OBJECT_END
+#define JSON_BUILD_EMPTY_OBJECT _JSON_BUILD_OBJECT_BEGIN, _JSON_BUILD_OBJECT_END
 #define JSON_BUILD_PAIR(n, ...) _JSON_BUILD_PAIR, ({ const char *_x = n; _x; }), __VA_ARGS__
 #define JSON_BUILD_PAIR_CONDITION(c, n, ...) _JSON_BUILD_PAIR_CONDITION, ({ bool _x = c; _x; }), ({ const char *_x = n; _x; }), __VA_ARGS__
 #define JSON_BUILD_NULL _JSON_BUILD_NULL

From 3dd1b600b8b76f480d27872f49a52b83a4960004 Mon Sep 17 00:00:00 2001
From: Lennart Poettering <lennart@poettering.net>
Date: Fri, 24 May 2019 10:44:56 +0200
Subject: [PATCH 21/29] json: permit 'null' as a way to reset tri-states to
 default

---
 src/shared/json.c | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/shared/json.c b/src/shared/json.c
index 5d3c6aad91..38b2bb14e4 100644
--- a/src/shared/json.c
+++ b/src/shared/json.c
@@ -3826,6 +3826,11 @@ int json_dispatch_tristate(const char *name, JsonVariant *variant, JsonDispatchF
         assert(variant);
         assert(b);
 
+        if (json_variant_is_null(variant)) {
+                *b = -1;
+                return 0;
+        }
+
         if (!json_variant_is_boolean(variant))
                 return json_log(variant, flags, SYNTHETIC_ERRNO(EINVAL), "JSON field '%s' is not a boolean.", strna(name));
 

From 0ac0787e30f0ba0b2fa370fbd87c19d62c0130c6 Mon Sep 17 00:00:00 2001
From: Lennart Poettering <lennart@poettering.net>
Date: Tue, 28 May 2019 15:05:53 +0200
Subject: [PATCH 22/29] json: add explicit log call for ENOMEM

---
 src/shared/json.h | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/shared/json.h b/src/shared/json.h
index abcf17e86d..40e74e03a4 100644
--- a/src/shared/json.h
+++ b/src/shared/json.h
@@ -311,6 +311,9 @@ int json_log_internal(JsonVariant *variant, int level, int error, const char *fi
                         : -ERRNO_VALUE(_e);                             \
         })
 
+#define json_log_oom(variant, flags) \
+        json_log(variant, flags, SYNTHETIC_ERRNO(ENOMEM), "Out of memory.")
+
 #define JSON_VARIANT_STRING_CONST(x) _JSON_VARIANT_STRING_CONST(UNIQ, (x))
 
 #define _JSON_VARIANT_STRING_CONST(xq, x)                               \

From 0b1f2e8a061721b37100380bd3c9ee4b97cb35cf Mon Sep 17 00:00:00 2001
From: Lennart Poettering <lennart@poettering.net>
Date: Wed, 29 May 2019 12:24:40 +0200
Subject: [PATCH 23/29] json: add new flag for forcing a flush after dumping
 json data to file

This is particularly useful when no trailing \n is generated, i.e. stdio
doesn't flush the output on its own.
---
 src/shared/json.c | 3 +++
 src/shared/json.h | 1 +
 2 files changed, 4 insertions(+)

diff --git a/src/shared/json.c b/src/shared/json.c
index 38b2bb14e4..7abbd1917d 100644
--- a/src/shared/json.c
+++ b/src/shared/json.c
@@ -1767,6 +1767,9 @@ void json_variant_dump(JsonVariant *v, JsonFormatFlags flags, FILE *f, const cha
                 fputc('\n', f);
         if (flags & JSON_FORMAT_SSE)
                 fputc('\n', f); /* In case of SSE add a second newline */
+
+        if (flags & JSON_FORMAT_FLUSH)
+                fflush(f);
 }
 
 int json_variant_filter(JsonVariant **v, char **to_remove) {
diff --git a/src/shared/json.h b/src/shared/json.h
index 40e74e03a4..749c25253f 100644
--- a/src/shared/json.h
+++ b/src/shared/json.h
@@ -168,6 +168,7 @@ typedef enum JsonFormatFlags {
         JSON_FORMAT_SOURCE     = 1 << 4, /* prefix with source filename/line/column */
         JSON_FORMAT_SSE        = 1 << 5, /* prefix/suffix with W3C server-sent events */
         JSON_FORMAT_SEQ        = 1 << 6, /* prefix/suffix with RFC 7464 application/json-seq */
+        JSON_FORMAT_FLUSH      = 1 << 7, /* call fflush() after dumping JSON */
 } JsonFormatFlags;
 
 int json_variant_format(JsonVariant *v, JsonFormatFlags flags, char **ret);

From faca141c5fb3b663216358d1798b38cd74b55c1b Mon Sep 17 00:00:00 2001
From: Lennart Poettering <lennart@poettering.net>
Date: Fri, 28 Jun 2019 20:05:21 +0200
Subject: [PATCH 24/29] json: add json_variant_unbase64() helper

---
 src/shared/json.c | 8 ++++++++
 src/shared/json.h | 2 ++
 2 files changed, 10 insertions(+)

diff --git a/src/shared/json.c b/src/shared/json.c
index 7abbd1917d..d1a477c36d 100644
--- a/src/shared/json.c
+++ b/src/shared/json.c
@@ -4120,6 +4120,14 @@ bool json_variant_is_sorted(JsonVariant *v) {
         return v->sorted;
 }
 
+int json_variant_unbase64(JsonVariant *v, void **ret, size_t *ret_size) {
+
+        if (!json_variant_is_string(v))
+                return -EINVAL;
+
+        return unbase64mem(json_variant_string(v), (size_t) -1, ret, ret_size);
+}
+
 static const char* const json_variant_type_table[_JSON_VARIANT_TYPE_MAX] = {
         [JSON_VARIANT_STRING] = "string",
         [JSON_VARIANT_INTEGER] = "integer",
diff --git a/src/shared/json.h b/src/shared/json.h
index 749c25253f..393e35a647 100644
--- a/src/shared/json.h
+++ b/src/shared/json.h
@@ -324,5 +324,7 @@ int json_log_internal(JsonVariant *variant, int level, int error, const char *fi
                 (JsonVariant*) ((uintptr_t) UNIQ_T(json_string_const, xq) + 1); \
         })
 
+int json_variant_unbase64(JsonVariant *v, void **ret, size_t *ret_size);
+
 const char *json_variant_type_to_string(JsonVariantType t);
 JsonVariantType json_variant_type_from_string(const char *s);

From a832b08e6e1bb93829b488615006bba768c50121 Mon Sep 17 00:00:00 2001
From: Lennart Poettering <lennart@poettering.net>
Date: Thu, 4 Jul 2019 18:27:02 +0200
Subject: [PATCH 25/29] json: add json_variant_set_field_integer() and
 json_variant_set_field_boolean() helpers

---
 src/shared/json.c | 22 ++++++++++++++++++++++
 src/shared/json.h |  2 ++
 2 files changed, 24 insertions(+)

diff --git a/src/shared/json.c b/src/shared/json.c
index d1a477c36d..7c62136703 100644
--- a/src/shared/json.c
+++ b/src/shared/json.c
@@ -1905,6 +1905,17 @@ int json_variant_set_field_string(JsonVariant **v, const char *field, const char
         return json_variant_set_field(v, field, m);
 }
 
+int json_variant_set_field_integer(JsonVariant **v, const char *field, intmax_t i) {
+        _cleanup_(json_variant_unrefp) JsonVariant *m = NULL;
+        int r;
+
+        r = json_variant_new_integer(&m, i);
+        if (r < 0)
+                return r;
+
+        return json_variant_set_field(v, field, m);
+}
+
 int json_variant_set_field_unsigned(JsonVariant **v, const char *field, uintmax_t u) {
         _cleanup_(json_variant_unrefp) JsonVariant *m = NULL;
         int r;
@@ -1916,6 +1927,17 @@ int json_variant_set_field_unsigned(JsonVariant **v, const char *field, uintmax_
         return json_variant_set_field(v, field, m);
 }
 
+int json_variant_set_field_boolean(JsonVariant **v, const char *field, bool b) {
+        _cleanup_(json_variant_unrefp) JsonVariant *m = NULL;
+        int r;
+
+        r = json_variant_new_boolean(&m, b);
+        if (r < 0)
+                return r;
+
+        return json_variant_set_field(v, field, m);
+}
+
 int json_variant_merge(JsonVariant **v, JsonVariant *m) {
         _cleanup_(json_variant_unrefp) JsonVariant *w = NULL;
         _cleanup_free_ JsonVariant **array = NULL;
diff --git a/src/shared/json.h b/src/shared/json.h
index 393e35a647..bb72b43015 100644
--- a/src/shared/json.h
+++ b/src/shared/json.h
@@ -178,7 +178,9 @@ int json_variant_filter(JsonVariant **v, char **to_remove);
 
 int json_variant_set_field(JsonVariant **v, const char *field, JsonVariant *value);
 int json_variant_set_field_string(JsonVariant **v, const char *field, const char *value);
+int json_variant_set_field_integer(JsonVariant **v, const char *field, intmax_t value);
 int json_variant_set_field_unsigned(JsonVariant **v, const char *field, uintmax_t value);
+int json_variant_set_field_boolean(JsonVariant **v, const char *field, bool b);
 
 int json_variant_append_array(JsonVariant **v, JsonVariant *element);
 

From a42ef715a2c65a8af0ebec907ae9de8acdc8f500 Mon Sep 17 00:00:00 2001
From: Lennart Poettering <lennart@poettering.net>
Date: Thu, 4 Jul 2019 18:27:12 +0200
Subject: [PATCH 26/29] json: add more dispatch helpers

---
 src/shared/json.c | 76 +++++++++++++++++++++++++++++++++++++++++++++++
 src/shared/json.h |  4 +++
 2 files changed, 80 insertions(+)

diff --git a/src/shared/json.c b/src/shared/json.c
index 7c62136703..722c2cfac3 100644
--- a/src/shared/json.c
+++ b/src/shared/json.c
@@ -23,6 +23,7 @@
 #include "string-util.h"
 #include "strv.h"
 #include "terminal-util.h"
+#include "user-util.h"
 #include "utf8.h"
 
 /* Refuse putting together variants with a larger depth than 4K by default (as a protection against overflowing stacks
@@ -4004,6 +4005,81 @@ int json_dispatch_variant(const char *name, JsonVariant *variant, JsonDispatchFl
         return 0;
 }
 
+int json_dispatch_uid_gid(const char *name, JsonVariant *variant, JsonDispatchFlags flags, void *userdata) {
+        uid_t *uid = userdata;
+        uintmax_t k;
+
+        assert_cc(sizeof(uid_t) == sizeof(uint32_t));
+        assert_cc(sizeof(gid_t) == sizeof(uint32_t));
+
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wtype-limits"
+        assert_cc(((uid_t) -1 < (uid_t) 0) == ((gid_t) -1 < (gid_t) 0));
+#pragma GCC diagnostic pop
+
+        if (json_variant_is_null(variant)) {
+                *uid = UID_INVALID;
+                return 0;
+        }
+
+        if (!json_variant_is_unsigned(variant))
+                return json_log(variant, flags, SYNTHETIC_ERRNO(EINVAL), "JSON field '%s' is not a integer.", strna(name));
+
+        k = json_variant_unsigned(variant);
+        if (k > UINT32_MAX || !uid_is_valid(k))
+                return json_log(variant, flags, SYNTHETIC_ERRNO(EINVAL), "JSON field '%s' is not a valid UID/GID.", strna(name));
+
+        *uid = k;
+        return 0;
+}
+
+int json_dispatch_user_group_name(const char *name, JsonVariant *variant, JsonDispatchFlags flags, void *userdata) {
+        char **s = userdata;
+        const char *n;
+        int r;
+
+        if (json_variant_is_null(variant)) {
+                *s = mfree(*s);
+                return 0;
+        }
+
+        if (!json_variant_is_string(variant))
+                return json_log(variant, flags, SYNTHETIC_ERRNO(EINVAL), "JSON field '%s' is not a string.", strna(name));
+
+        n = json_variant_string(variant);
+        if (!valid_user_group_name(n))
+                return json_log(variant, flags, SYNTHETIC_ERRNO(EINVAL), "JSON field '%s' is not a valid user/group name.", strna(name));
+
+        r = free_and_strdup(s, n);
+        if (r < 0)
+                return json_log(variant, flags, r, "Failed to allocate string: %m");
+
+        return 0;
+}
+
+int json_dispatch_id128(const char *name, JsonVariant *variant, JsonDispatchFlags flags, void *userdata) {
+        sd_id128_t *uuid = userdata;
+        int r;
+
+        if (json_variant_is_null(variant)) {
+                *uuid = SD_ID128_NULL;
+                return 0;
+        }
+
+        if (!json_variant_is_string(variant))
+                return json_log(variant, flags, SYNTHETIC_ERRNO(EINVAL), "JSON field '%s' is not a string.", strna(name));
+
+        r = sd_id128_from_string(json_variant_string(variant), uuid);
+        if (r < 0)
+                return json_log(variant, flags, r, "JSON field '%s' is not a valid UID.", strna(name));
+
+        return 0;
+}
+
+int json_dispatch_unsupported(const char *name, JsonVariant *variant, JsonDispatchFlags flags, void *userdata) {
+        return json_log(variant, flags, SYNTHETIC_ERRNO(EINVAL), "JSON field '%s' is not allowed in this object.", strna(name));
+}
+
 static int json_cmp_strings(const void *x, const void *y) {
         JsonVariant *const *a = x, *const *b = y;
 
diff --git a/src/shared/json.h b/src/shared/json.h
index bb72b43015..508e9c5053 100644
--- a/src/shared/json.h
+++ b/src/shared/json.h
@@ -279,6 +279,10 @@ int json_dispatch_integer(const char *name, JsonVariant *variant, JsonDispatchFl
 int json_dispatch_unsigned(const char *name, JsonVariant *variant, JsonDispatchFlags flags, void *userdata);
 int json_dispatch_uint32(const char *name, JsonVariant *variant, JsonDispatchFlags flags, void *userdata);
 int json_dispatch_int32(const char *name, JsonVariant *variant, JsonDispatchFlags flags, void *userdata);
+int json_dispatch_uid_gid(const char *name, JsonVariant *variant, JsonDispatchFlags flags, void *userdata);
+int json_dispatch_user_group_name(const char *name, JsonVariant *variant, JsonDispatchFlags flags, void *userdata);
+int json_dispatch_id128(const char *name, JsonVariant *variant, JsonDispatchFlags flags, void *userdata);
+int json_dispatch_unsupported(const char *name, JsonVariant *variant, JsonDispatchFlags flags, void *userdata);
 
 assert_cc(sizeof(uintmax_t) == sizeof(uint64_t));
 #define json_dispatch_uint64 json_dispatch_unsigned

From e4defdc4b02f915a07b72571191b8415b4e4b828 Mon Sep 17 00:00:00 2001
From: Lennart Poettering <lennart@poettering.net>
Date: Mon, 29 Jul 2019 09:12:28 +0200
Subject: [PATCH 27/29] json: teach json_build() to build arrays from C arrays
 of JsonVariant

---
 src/shared/json.c | 30 ++++++++++++++++++++++++++++++
 src/shared/json.h |  2 ++
 2 files changed, 32 insertions(+)

diff --git a/src/shared/json.c b/src/shared/json.c
index 722c2cfac3..25e711964f 100644
--- a/src/shared/json.c
+++ b/src/shared/json.c
@@ -3377,6 +3377,36 @@ int json_buildv(JsonVariant **ret, va_list ap) {
 
                         break;
 
+                case _JSON_BUILD_VARIANT_ARRAY: {
+                        JsonVariant **array;
+                        size_t n;
+
+                        if (!IN_SET(current->expect, EXPECT_TOPLEVEL, EXPECT_OBJECT_VALUE, EXPECT_ARRAY_ELEMENT)) {
+                                r = -EINVAL;
+                                goto finish;
+                        }
+
+                        array = va_arg(ap, JsonVariant**);
+                        n = va_arg(ap, size_t);
+
+                        if (current->n_suppress == 0) {
+                                r = json_variant_new_array(&add, array, n);
+                                if (r < 0)
+                                        goto finish;
+                        }
+
+                        n_subtract = 1;
+
+                        if (current->expect == EXPECT_TOPLEVEL)
+                                current->expect = EXPECT_END;
+                        else if (current->expect == EXPECT_OBJECT_VALUE)
+                                current->expect = EXPECT_OBJECT_KEY;
+                        else
+                                assert(current->expect == EXPECT_ARRAY_ELEMENT);
+
+                        break;
+                }
+
                 case _JSON_BUILD_LITERAL: {
                         const char *l;
 
diff --git a/src/shared/json.h b/src/shared/json.h
index 508e9c5053..d1a6eb5eb9 100644
--- a/src/shared/json.h
+++ b/src/shared/json.h
@@ -217,6 +217,7 @@ enum {
         _JSON_BUILD_PAIR_CONDITION,
         _JSON_BUILD_NULL,
         _JSON_BUILD_VARIANT,
+        _JSON_BUILD_VARIANT_ARRAY,
         _JSON_BUILD_LITERAL,
         _JSON_BUILD_STRV,
         _JSON_BUILD_BASE64,
@@ -236,6 +237,7 @@ enum {
 #define JSON_BUILD_PAIR_CONDITION(c, n, ...) _JSON_BUILD_PAIR_CONDITION, ({ bool _x = c; _x; }), ({ const char *_x = n; _x; }), __VA_ARGS__
 #define JSON_BUILD_NULL _JSON_BUILD_NULL
 #define JSON_BUILD_VARIANT(v) _JSON_BUILD_VARIANT, ({ JsonVariant *_x = v; _x; })
+#define JSON_BUILD_VARIANT_ARRAY(v, n) _JSON_BUILD_VARIANT_ARRAY, ({ JsonVariant **_x = v; _x; }), ({ size_t _y = n; _y; })
 #define JSON_BUILD_LITERAL(l) _JSON_BUILD_LITERAL, ({ const char *_x = l; _x; })
 #define JSON_BUILD_STRV(l) _JSON_BUILD_STRV, ({ char **_x = l; _x; })
 #define JSON_BUILD_BASE64(p, n) _JSON_BUILD_BASE64, ({ const void *_x = p; _x; }), ({ size_t _y = n; _y; })

From 19a209cc710a0368588d8541ed40ba8886599bd1 Mon Sep 17 00:00:00 2001
From: Lennart Poettering <lennart@poettering.net>
Date: Wed, 7 Aug 2019 12:45:50 +0200
Subject: [PATCH 28/29] json: add const string dispatcher

This adds json_dispatch_const_string() which is similar to
json_dispatch_string() but doesn't store a strdup()'ed copy of the
string, but a pointer directly into the JSON record.

This should simplify cases where the json variant sticks around long
enough anyway.
---
 src/shared/json.c | 21 +++++++++++++++++++++
 src/shared/json.h |  1 +
 2 files changed, 22 insertions(+)

diff --git a/src/shared/json.c b/src/shared/json.c
index 25e711964f..2327cbcc9c 100644
--- a/src/shared/json.c
+++ b/src/shared/json.c
@@ -3977,6 +3977,27 @@ int json_dispatch_string(const char *name, JsonVariant *variant, JsonDispatchFla
         return 0;
 }
 
+int json_dispatch_const_string(const char *name, JsonVariant *variant, JsonDispatchFlags flags, void *userdata) {
+        const char **s = userdata;
+
+        assert(variant);
+        assert(s);
+
+        if (json_variant_is_null(variant)) {
+                *s = NULL;
+                return 0;
+        }
+
+        if (!json_variant_is_string(variant))
+                return json_log(variant, flags, SYNTHETIC_ERRNO(EINVAL), "JSON field '%s' is not a string.", strna(name));
+
+        if ((flags & JSON_SAFE) && !string_is_safe(json_variant_string(variant)))
+                return json_log(variant, flags, SYNTHETIC_ERRNO(EINVAL), "JSON field '%s' contains unsafe characters, refusing.", strna(name));
+
+        *s = json_variant_string(variant);
+        return 0;
+}
+
 int json_dispatch_strv(const char *name, JsonVariant *variant, JsonDispatchFlags flags, void *userdata) {
         _cleanup_strv_free_ char **l = NULL;
         char ***s = userdata;
diff --git a/src/shared/json.h b/src/shared/json.h
index d1a6eb5eb9..a89381dc51 100644
--- a/src/shared/json.h
+++ b/src/shared/json.h
@@ -273,6 +273,7 @@ typedef struct JsonDispatch {
 int json_dispatch(JsonVariant *v, const JsonDispatch table[], JsonDispatchCallback bad, JsonDispatchFlags flags, void *userdata);
 
 int json_dispatch_string(const char *name, JsonVariant *variant, JsonDispatchFlags flags, void *userdata);
+int json_dispatch_const_string(const char *name, JsonVariant *variant, JsonDispatchFlags flags, void *userdata);
 int json_dispatch_strv(const char *name, JsonVariant *variant, JsonDispatchFlags flags, void *userdata);
 int json_dispatch_boolean(const char *name, JsonVariant *variant, JsonDispatchFlags flags, void *userdata);
 int json_dispatch_tristate(const char *name, JsonVariant *variant, JsonDispatchFlags flags, void *userdata);

From 2d8143048bc67470c76e8975cdecfb6b9823ecce Mon Sep 17 00:00:00 2001
From: Lennart Poettering <lennart@poettering.net>
Date: Thu, 15 Aug 2019 09:36:04 +0200
Subject: [PATCH 29/29] json: add new output flag JSON_PRETTY_AUTO

This takes inspiration from JSON_COLOR_AUTO: it will automatically map
to JSON_PRETTY if connected to a TTY and JSON_NEWLINE otherwise.
---
 src/shared/json.c |  3 +++
 src/shared/json.h | 17 +++++++++--------
 2 files changed, 12 insertions(+), 8 deletions(-)

diff --git a/src/shared/json.c b/src/shared/json.c
index 2327cbcc9c..869aa279ee 100644
--- a/src/shared/json.c
+++ b/src/shared/json.c
@@ -1757,6 +1757,9 @@ void json_variant_dump(JsonVariant *v, JsonFormatFlags flags, FILE *f, const cha
         if (((flags & (JSON_FORMAT_COLOR_AUTO|JSON_FORMAT_COLOR)) == JSON_FORMAT_COLOR_AUTO) && colors_enabled())
                 flags |= JSON_FORMAT_COLOR;
 
+        if (((flags & (JSON_FORMAT_PRETTY_AUTO|JSON_FORMAT_PRETTY)) == JSON_FORMAT_PRETTY_AUTO))
+                flags |= on_tty() ? JSON_FORMAT_PRETTY : JSON_FORMAT_NEWLINE;
+
         if (flags & JSON_FORMAT_SSE)
                 fputs("data: ", f);
         if (flags & JSON_FORMAT_SEQ)
diff --git a/src/shared/json.h b/src/shared/json.h
index a89381dc51..b9b300a1d2 100644
--- a/src/shared/json.h
+++ b/src/shared/json.h
@@ -161,14 +161,15 @@ struct json_variant_foreach_state {
 int json_variant_get_source(JsonVariant *v, const char **ret_source, unsigned *ret_line, unsigned *ret_column);
 
 typedef enum JsonFormatFlags {
-        JSON_FORMAT_NEWLINE    = 1 << 0, /* suffix with newline */
-        JSON_FORMAT_PRETTY     = 1 << 1, /* add internal whitespace to appeal to human readers */
-        JSON_FORMAT_COLOR      = 1 << 2, /* insert ANSI color sequences */
-        JSON_FORMAT_COLOR_AUTO = 1 << 3, /* insert ANSI color sequences if colors_enabled() says so */
-        JSON_FORMAT_SOURCE     = 1 << 4, /* prefix with source filename/line/column */
-        JSON_FORMAT_SSE        = 1 << 5, /* prefix/suffix with W3C server-sent events */
-        JSON_FORMAT_SEQ        = 1 << 6, /* prefix/suffix with RFC 7464 application/json-seq */
-        JSON_FORMAT_FLUSH      = 1 << 7, /* call fflush() after dumping JSON */
+        JSON_FORMAT_NEWLINE     = 1 << 0, /* suffix with newline */
+        JSON_FORMAT_PRETTY      = 1 << 1, /* add internal whitespace to appeal to human readers */
+        JSON_FORMAT_PRETTY_AUTO = 1 << 2, /* same, but only if connected to a tty (and JSON_FORMAT_NEWLINE otherwise) */
+        JSON_FORMAT_COLOR       = 1 << 3, /* insert ANSI color sequences */
+        JSON_FORMAT_COLOR_AUTO  = 1 << 4, /* insert ANSI color sequences if colors_enabled() says so */
+        JSON_FORMAT_SOURCE      = 1 << 5, /* prefix with source filename/line/column */
+        JSON_FORMAT_SSE         = 1 << 6, /* prefix/suffix with W3C server-sent events */
+        JSON_FORMAT_SEQ         = 1 << 7, /* prefix/suffix with RFC 7464 application/json-seq */
+        JSON_FORMAT_FLUSH       = 1 << 8, /* call fflush() after dumping JSON */
 } JsonFormatFlags;
 
 int json_variant_format(JsonVariant *v, JsonFormatFlags flags, char **ret);