tree-wide: several cleanups for reading/writing /proc/sys/fs/nr_open

- use unsigned for the return value of read_nr_open(), as it does not
  fail, and the kernel internally uses unsigned for the value,
- when bumping the value by PID1, let's start from the kernel's maximum
  value defined in fs/file.c. The maximum value should be mostly an API
  of the kernel, but may changed in a future, hence still try several
  times if we fail to bump the value.

Co-authored-by: Jared Baur <jaredbaur@fastmail.com>
Co-authored-by: John Rinehart <johnrichardrinehart@gmail.com>

src/basic/fd-util.c

@@ -990,7 +990,7 @@ int fd_verify_safe_flags_full(int fd, int extra_flags) {
         return flags & (O_ACCMODE_STRICT | extra_flags); /* return the flags variable, but remove the noise */
 }
 
-int read_nr_open(void) {
+unsigned read_nr_open(void) {
         _cleanup_free_ char *nr_open = NULL;
         int r;
 
@@ -1001,9 +1001,9 @@ int read_nr_open(void) {
         if (r < 0)
                 log_debug_errno(r, "Failed to read /proc/sys/fs/nr_open, ignoring: %m");
         else {
-                int v;
+                unsigned v;
 
-                r = safe_atoi(nr_open, &v);
+                r = safe_atou(nr_open, &v);
                 if (r < 0)
                         log_debug_errno(r, "Failed to parse /proc/sys/fs/nr_open value '%s', ignoring: %m", nr_open);
                 else
@@ -1011,7 +1011,7 @@ int read_nr_open(void) {
         }
 
         /* If we fail, fall back to the hard-coded kernel limit of 1024 * 1024. */
-        return 1024 * 1024;
+        return NR_OPEN_DEFAULT;
 }
 
 int fd_get_diskseq(int fd, uint64_t *ret) {

src/basic/fd-util.h

@@ -50,6 +50,13 @@
  * around the problems with musl's definition. */
 #define O_ACCMODE_STRICT (O_RDONLY|O_WRONLY|O_RDWR)
 
+/* The default, minimum, and maximum values of /proc/sys/fs/nr_open. See kernel's fs/file.c.
+ * These values have been unchanged since kernel-2.6.26:
+ * https://github.com/torvalds/linux/commit/eceea0b3df05ed262ae32e0c6340cc7a3626632d */
+#define NR_OPEN_DEFAULT ((unsigned) (1024 * 1024))
+#define NR_OPEN_MINIMUM ((unsigned) (sizeof(long) * 8))
+#define NR_OPEN_MAXIMUM ((unsigned) (CONST_MIN((size_t) INT_MAX, SIZE_MAX / __SIZEOF_POINTER__) & ~(sizeof(long) * 8 - 1)))
+
 int close_nointr(int fd);
 int safe_close(int fd);
 void safe_close_pair(int p[static 2]);
@@ -148,7 +155,7 @@ static inline int fd_verify_safe_flags(int fd) {
         return fd_verify_safe_flags_full(fd, 0);
 }
 
-int read_nr_open(void);
+unsigned read_nr_open(void);
 int fd_get_diskseq(int fd, uint64_t *ret);
 
 int path_is_root_at(int dir_fd, const char *path);