terminal-util: add terminal_is_pty_fd() helper

The helper checks if an fd references a pty

src/basic/terminal-util.c

@@ -32,6 +32,7 @@
 #include "io-util.h"
 #include "log.h"
 #include "macro.h"
+#include "missing_magic.h"
 #include "namespace-util.h"
 #include "parse-util.h"
 #include "path-util.h"
@@ -2117,3 +2118,31 @@ int terminal_fix_size(int input_fd, int output_fd) {
         log_debug("Fixed terminal dimensions to %ux%u based on ANSI sequence information.", columns, rows);
         return 1;
 }
+
+int terminal_is_pty_fd(int fd) {
+        int r;
+
+        assert(fd >= 0);
+
+        /* Returns true if we are looking at a pty, i.e. if it's backed by the /dev/pts/ file system */
+
+        if (!isatty_safe(fd))
+                return false;
+
+        r = is_fs_type_at(fd, NULL, DEVPTS_SUPER_MAGIC);
+        if (r != 0)
+                return r;
+
+        /* The ptmx device is weird, it exists twice, once inside and once outside devpts. To detect the
+         * latter case, let's fire off an ioctl() that only works on ptmx devices. */
+
+        int v;
+        if (ioctl(fd, TIOCGPKT, &v) < 0) {
+                if (ERRNO_IS_NOT_SUPPORTED(errno))
+                        return false;
+
+                return -errno;
+        }
+
+        return true;
+}

src/basic/terminal-util.h

@@ -299,3 +299,5 @@ int get_default_background_color(double *ret_red, double *ret_green, double *ret
 int terminal_get_size_by_dsr(int input_fd, int output_fd, unsigned *ret_rows, unsigned *ret_columns);
 
 int terminal_fix_size(int input_fd, int output_fd);
+
+int terminal_is_pty_fd(int fd);

src/test/test-terminal-util.c

@@ -209,6 +209,46 @@ TEST(terminal_fix_size) {
                 log_notice("Fixed terminal size.");
 }
 
+TEST(terminal_is_pty_fd) {
+        _cleanup_close_ int fd1 = -EBADF, fd2 = -EBADF;
+        _cleanup_free_ char *peer = NULL;
+        int r;
+
+        fd1 = openpt_allocate(O_RDWR, &peer);
+        assert_se(fd1 >= 0);
+        assert_se(terminal_is_pty_fd(fd1) > 0);
+
+        fd2 = open_terminal(peer, O_RDWR|O_CLOEXEC|O_NOCTTY);
+        assert_se(fd2 >= 0);
+        assert_se(terminal_is_pty_fd(fd2) > 0);
+
+        fd1 = safe_close(fd1);
+        fd2 = safe_close(fd2);
+
+        fd1 = open("/dev/null", O_RDONLY|O_CLOEXEC);
+        assert_se(fd1 >= 0);
+        assert_se(terminal_is_pty_fd(fd1) == 0);
+
+        /* In container managers real tty devices might be weird, avoid them. */
+        r = path_is_read_only_fs("/sys");
+        if (r != 0)
+                return;
+
+        FOREACH_STRING(p, "/dev/ttyS0", "/dev/tty1") {
+                _cleanup_close_ int tfd = -EBADF;
+
+                tfd = open_terminal(p, O_CLOEXEC|O_NOCTTY|O_RDONLY|O_NONBLOCK);
+                if (tfd == -ENOENT)
+                        continue;
+                if (tfd < 0)  {
+                        log_notice_errno(tfd, "Failed to open '%s', skipping: %m", p);
+                        continue;
+                }
+
+                assert_se(terminal_is_pty_fd(tfd) <= 0);
+        }
+}
+
 static void test_get_color_mode_with_env(const char *key, const char *val, ColorMode expected) {
         ASSERT_OK(setenv(key, val, true));
         reset_terminal_feature_caches();