terminal-util: add terminal_is_pty_fd() helper

The helper checks if an fd references a pty

src/basic/terminal-util.c

@@ -32,6 +32,7 @@
 #include "io-util.h"
 #include "log.h"
 #include "macro.h"
+#include "missing_magic.h"
 #include "namespace-util.h"
 #include "parse-util.h"
 #include "path-util.h"
@@ -2117,3 +2118,31 @@ int terminal_fix_size(int input_fd, int output_fd) {
         log_debug("Fixed terminal dimensions to %ux%u based on ANSI sequence information.", columns, rows);
         return 1;
 }
+
+int terminal_is_pty_fd(int fd) {
+        int r;
+
+        assert(fd >= 0);
+
+        /* Returns true if we are looking at a pty, i.e. if it's backed by the /dev/pts/ file system */
+
+        if (!isatty_safe(fd))
+                return false;
+
+        r = is_fs_type_at(fd, NULL, DEVPTS_SUPER_MAGIC);
+        if (r != 0)
+                return r;
+
+        /* The ptmx device is weird, it exists twice, once inside and once outside devpts. To detect the
+         * latter case, let's fire off an ioctl() that only works on ptmx devices. */
+
+        int v;
+        if (ioctl(fd, TIOCGPKT, &v) < 0) {
+                if (ERRNO_IS_NOT_SUPPORTED(errno))
+                        return false;
+
+                return -errno;
+        }
+
+        return true;
+}

src/basic/terminal-util.h

@@ -299,3 +299,5 @@ int get_default_background_color(double *ret_red, double *ret_green, double *ret
 int terminal_get_size_by_dsr(int input_fd, int output_fd, unsigned *ret_rows, unsigned *ret_columns);
 
 int terminal_fix_size(int input_fd, int output_fd);
+
+int terminal_is_pty_fd(int fd);