SymbX/systemd
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #2839 from keszybz/use-sendfile-smarter

Browse Source 
Use sendfile smarter
This commit is contained in:
Daniel Mack
2016-03-17 20:33:41 +01:00
parent da9a4daa08 2f368e4a11
commit 5fec580c84
7 changed files with 438 additions and 230 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/basic/copy.c
+51 -12
View File
@@ -40,17 +40,38 @@
#include "fs-util.h"
#include "io-util.h"
#include "macro.h"
#include "missing.h"
#include "string-util.h"
#include "strv.h"
#include "time-util.h"
#include "umask-util.h"
#include "xattr-util.h"
#define COPY_BUFFER_SIZE (16*1024)
#define COPY_BUFFER_SIZE (16*1024u)
static ssize_t try_copy_file_range(int fd_in, loff_t *off_in,
int fd_out, loff_t *off_out,
size_t len,
unsigned int flags) {
static int have = -1;
ssize_t r;
if (have == false)
return -ENOSYS;
r = copy_file_range(fd_in, off_in, fd_out, off_out, len, flags);
if (_unlikely_(have < 0))
have = r >= 0 || errno != ENOSYS;
if (r >= 0)
return r;
else
return -errno;
}
int copy_bytes(int fdf, int fdt, uint64_t max_bytes, bool try_reflink) {
bool try_sendfile = true, try_splice = true;
bool try_cfr = true, try_sendfile = true, try_splice = true;
int r;
size_t m = SSIZE_MAX; /* that the maximum that sendfile and c_f_r accept */
assert(fdf >= 0);
assert(fdt >= 0);
@@ -67,11 +88,9 @@ int copy_bytes(int fdf, int fdt, uint64_t max_bytes, bool try_reflink) {
}
for (;;) {
size_t m = COPY_BUFFER_SIZE;
ssize_t n;
if (max_bytes != (uint64_t) -1) {
if (max_bytes <= 0)
return 1; /* return > 0 if we hit the max_bytes limit */
@@ -79,44 +98,59 @@ int copy_bytes(int fdf, int fdt, uint64_t max_bytes, bool try_reflink) {
m = (size_t) max_bytes;
}
/* First try copy_file_range(), unless we already tried */
if (try_cfr) {
n = try_copy_file_range(fdf, NULL, fdt, NULL, m, 0u);
if (n < 0) {
if (!IN_SET(n, -EINVAL, -ENOSYS, -EXDEV))
return n;
try_cfr = false;
/* use fallback below */
} else if (n == 0) /* EOF */
break;
else
/* Success! */
goto next;
}
/* First try sendfile(), unless we already tried */
if (try_sendfile) {
n = sendfile(fdt, fdf, NULL, m);
if (n < 0) {
if (errno != EINVAL && errno != ENOSYS)
if (!IN_SET(errno, EINVAL, ENOSYS))
return -errno;
try_sendfile = false;
/* use fallback below */
} else if (n == 0) /* EOF */
break;
else if (n > 0)
else
/* Success! */
goto next;
}
/* The try splice, unless we already tried */
/* Then try splice, unless we already tried */
if (try_splice) {
n = splice(fdf, NULL, fdt, NULL, m, 0);
if (n < 0) {
if (errno != EINVAL && errno != ENOSYS)
if (!IN_SET(errno, EINVAL, ENOSYS))
return -errno;
try_splice = false;
/* use fallback below */
} else if (n == 0) /* EOF */
break;
else if (n > 0)
else
/* Success! */
goto next;
}
/* As a fallback just copy bits by hand */
{
uint8_t buf[m];
uint8_t buf[MIN(m, COPY_BUFFER_SIZE)];
n = read(fdf, buf, m);
n = read(fdf, buf, sizeof buf);
if (n < 0)
return -errno;
if (n == 0) /* EOF */
@@ -132,6 +166,11 @@ int copy_bytes(int fdf, int fdt, uint64_t max_bytes, bool try_reflink) {
assert(max_bytes >= (uint64_t) n);
max_bytes -= n;
}
/* sendfile accepts at most SSIZE_MAX-offset bytes to copy,
* so reduce our maximum by the amount we already copied,
* but don't go below our copy buffer size, unless we are
* close the the limit of bytes we are allowed to copy. */
m = MAX(MIN(COPY_BUFFER_SIZE, max_bytes), m - n);
}
return 0; /* return 0 if we hit EOF earlier than the size limit */
src/basic/missing.h
+2 -214
View File
@@ -135,84 +135,6 @@
#define SOL_SCTP 132
#endif
#if !HAVE_DECL_PIVOT_ROOT
static inline int pivot_root(const char *new_root, const char *put_old) {
return syscall(SYS_pivot_root, new_root, put_old);
}
#endif
#ifndef __NR_memfd_create
# if defined __x86_64__
# define __NR_memfd_create 319
# elif defined __arm__
# define __NR_memfd_create 385
# elif defined __aarch64__
# define __NR_memfd_create 279
# elif defined __s390__
# define __NR_memfd_create 350
# elif defined _MIPS_SIM
# if _MIPS_SIM == _MIPS_SIM_ABI32
# define __NR_memfd_create 4354
# endif
# if _MIPS_SIM == _MIPS_SIM_NABI32
# define __NR_memfd_create 6318
# endif
# if _MIPS_SIM == _MIPS_SIM_ABI64
# define __NR_memfd_create 5314
# endif
# elif defined __i386__
# define __NR_memfd_create 356
# else
# warning "__NR_memfd_create unknown for your architecture"
# define __NR_memfd_create 0xffffffff
# endif
#endif
#if !HAVE_DECL_MEMFD_CREATE
static inline int memfd_create(const char *name, unsigned int flags) {
return syscall(__NR_memfd_create, name, flags);
}
#endif
#ifndef __NR_getrandom
# if defined __x86_64__
# define __NR_getrandom 318
# elif defined(__i386__)
# define __NR_getrandom 355
# elif defined(__arm__)
# define __NR_getrandom 384
# elif defined(__aarch64__)
# define __NR_getrandom 278
# elif defined(__ia64__)
# define __NR_getrandom 1339
# elif defined(__m68k__)
# define __NR_getrandom 352
# elif defined(__s390x__)
# define __NR_getrandom 349
# elif defined(__powerpc__)
# define __NR_getrandom 359
# elif defined _MIPS_SIM
# if _MIPS_SIM == _MIPS_SIM_ABI32
# define __NR_getrandom 4353
# endif
# if _MIPS_SIM == _MIPS_SIM_NABI32
# define __NR_getrandom 6317
# endif
# if _MIPS_SIM == _MIPS_SIM_ABI64
# define __NR_getrandom 5313
# endif
# else
# warning "__NR_getrandom unknown for your architecture"
# define __NR_getrandom 0xffffffff
# endif
#endif
#if !HAVE_DECL_GETRANDOM
static inline int getrandom(void *buffer, size_t count, unsigned flags) {
return syscall(__NR_getrandom, buffer, count, flags);
}
#endif
#ifndef GRND_NONBLOCK
#define GRND_NONBLOCK 0x0001
#endif
@@ -527,12 +449,6 @@ struct btrfs_ioctl_quota_ctl_args {
#define MS_PRIVATE (1 << 18)
#endif
#if !HAVE_DECL_GETTID
static inline pid_t gettid(void) {
return (pid_t) syscall(SYS_gettid);
}
#endif
#ifndef SCM_SECURITY
#define SCM_SECURITY 0x03
#endif
@@ -561,32 +477,6 @@ static inline pid_t gettid(void) {
#define MAX_HANDLE_SZ 128
#endif
#ifndef __NR_name_to_handle_at
# if defined(__x86_64__)
# define __NR_name_to_handle_at 303
# elif defined(__i386__)
# define __NR_name_to_handle_at 341
# elif defined(__arm__)
# define __NR_name_to_handle_at 370
# elif defined(__powerpc__)
# define __NR_name_to_handle_at 345
# else
# error "__NR_name_to_handle_at is not defined"
# endif
#endif
#if !HAVE_DECL_NAME_TO_HANDLE_AT
struct file_handle {
unsigned int handle_bytes;
int handle_type;
unsigned char f_handle[0];
};
static inline int name_to_handle_at(int fd, const char *name, struct file_handle *handle, int *mnt_id, int flags) {
return syscall(__NR_name_to_handle_at, fd, name, handle, mnt_id, flags);
}
#endif
#ifndef HAVE_SECURE_GETENV
# ifdef HAVE___SECURE_GETENV
# define secure_getenv __secure_getenv
@@ -635,22 +525,6 @@ static inline int name_to_handle_at(int fd, const char *name, struct file_handle
#endif
#ifndef __NR_setns
# if defined(__x86_64__)
# define __NR_setns 308
# elif defined(__i386__)
# define __NR_setns 346
# else
# error "__NR_setns is not defined"
# endif
#endif
#if !HAVE_DECL_SETNS
static inline int setns(int fd, int nstype) {
return syscall(__NR_setns, fd, nstype);
}
#endif
#if !HAVE_DECL_LO_FLAGS_PARTSCAN
#define LO_FLAGS_PARTSCAN 8
#endif
@@ -1018,69 +892,10 @@ static inline int setns(int fd, int nstype) {
#define CAP_AUDIT_READ 37
#endif
static inline int raw_clone(unsigned long flags, void *child_stack) {
#if defined(__s390__) || defined(__CRIS__)
/* On s390 and cris the order of the first and second arguments
* of the raw clone() system call is reversed. */
return (int) syscall(__NR_clone, child_stack, flags);
#else
return (int) syscall(__NR_clone, flags, child_stack);
#endif
}
static inline pid_t raw_getpid(void) {
#if defined(__alpha__)
return (pid_t) syscall(__NR_getxpid);
#else
return (pid_t) syscall(__NR_getpid);
#endif
}
#if !HAVE_DECL_RENAMEAT2
#ifndef __NR_renameat2
# if defined __x86_64__
# define __NR_renameat2 316
# elif defined __arm__
# define __NR_renameat2 382
# elif defined _MIPS_SIM
# if _MIPS_SIM == _MIPS_SIM_ABI32
# define __NR_renameat2 4351
# endif
# if _MIPS_SIM == _MIPS_SIM_NABI32
# define __NR_renameat2 6315
# endif
# if _MIPS_SIM == _MIPS_SIM_ABI64
# define __NR_renameat2 5311
# endif
# elif defined __i386__
# define __NR_renameat2 353
# else
# warning "__NR_renameat2 unknown for your architecture"
# define __NR_renameat2 0xffffffff
# endif
#endif
static inline int renameat2(int oldfd, const char *oldname, int newfd, const char *newname, unsigned flags) {
return syscall(__NR_renameat2, oldfd, oldname, newfd, newname, flags);
}
#endif
#ifndef RENAME_NOREPLACE
#define RENAME_NOREPLACE (1 << 0)
#endif
#if !HAVE_DECL_KCMP
static inline int kcmp(pid_t pid1, pid_t pid2, int type, unsigned long idx1, unsigned long idx2) {
#if defined(__NR_kcmp)
return syscall(__NR_kcmp, pid1, pid2, type, idx1, idx2);
#else
errno = ENOSYS;
return -1;
#endif
}
#endif
#ifndef KCMP_FILE
#define KCMP_FILE 0
#endif
@@ -1097,35 +912,6 @@ static inline int kcmp(pid_t pid1, pid_t pid2, int type, unsigned long idx1, uns
typedef int32_t key_serial_t;
#endif
#if !HAVE_DECL_KEYCTL
static inline long keyctl(int cmd, unsigned long arg2, unsigned long arg3, unsigned long arg4,unsigned long arg5) {
#if defined(__NR_keyctl)
return syscall(__NR_keyctl, cmd, arg2, arg3, arg4, arg5);
#else
errno = ENOSYS;
return -1;
#endif
}
static inline key_serial_t add_key(const char *type, const char *description, const void *payload, size_t plen, key_serial_t ringid) {
#if defined (__NR_add_key)
return syscall(__NR_add_key, type, description, payload, plen, ringid);
#else
errno = ENOSYS;
return -1;
#endif
}
static inline key_serial_t request_key(const char *type, const char *description, const char * callout_info, key_serial_t destringid) {
#if defined (__NR_request_key)
return syscall(__NR_request_key, type, description, callout_info, destringid);
#else
errno = ENOSYS;
return -1;
#endif
}
#endif
#ifndef KEYCTL_READ
#define KEYCTL_READ 11
#endif
@@ -1177,3 +963,5 @@ static inline key_serial_t request_key(const char *type, const char *description
#endif
#endif
#include "missing_syscall.h"
src/basic/missing_syscall.h
+310
View File
@@ -0,0 +1,310 @@
#pragma once
/***
This file is part of systemd.
Copyright 2010 Lennart Poettering
Copyright 2016 Zbigniew Jędrzejewski-Szmek
systemd is free software; you can redistribute it and/or modify it
under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation; either version 2.1 of the License, or
(at your option) any later version.
systemd is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with systemd; If not, see <http://www.gnu.org/licenses/>.
***/
/* Missing glibc definitions to access certain kernel APIs */
#if !HAVE_DECL_PIVOT_ROOT
static inline int pivot_root(const char *new_root, const char *put_old) {
return syscall(SYS_pivot_root, new_root, put_old);
}
#endif
/* ======================================================================= */
#if !HAVE_DECL_MEMFD_CREATE
# ifndef __NR_memfd_create
# if defined __x86_64__
# define __NR_memfd_create 319
# elif defined __arm__
# define __NR_memfd_create 385
# elif defined __aarch64__
# define __NR_memfd_create 279
# elif defined __s390__
# define __NR_memfd_create 350
# elif defined _MIPS_SIM
# if _MIPS_SIM == _MIPS_SIM_ABI32
# define __NR_memfd_create 4354
# endif
# if _MIPS_SIM == _MIPS_SIM_NABI32
# define __NR_memfd_create 6318
# endif
# if _MIPS_SIM == _MIPS_SIM_ABI64
# define __NR_memfd_create 5314
# endif
# elif defined __i386__
# define __NR_memfd_create 356
# else
# warning "__NR_memfd_create unknown for your architecture"
# endif
# endif
static inline int memfd_create(const char *name, unsigned int flags) {
# ifdef __NR_memfd_create
return syscall(__NR_memfd_create, name, flags);
# else
errno = ENOSYS;
return -1;
# endif
}
#endif
/* ======================================================================= */
#if !HAVE_DECL_GETRANDOM
# ifndef __NR_getrandom
# if defined __x86_64__
# define __NR_getrandom 318
# elif defined(__i386__)
# define __NR_getrandom 355
# elif defined(__arm__)
# define __NR_getrandom 384
# elif defined(__aarch64__)
# define __NR_getrandom 278
# elif defined(__ia64__)
# define __NR_getrandom 1339
# elif defined(__m68k__)
# define __NR_getrandom 352
# elif defined(__s390x__)
# define __NR_getrandom 349
# elif defined(__powerpc__)
# define __NR_getrandom 359
# elif defined _MIPS_SIM
# if _MIPS_SIM == _MIPS_SIM_ABI32
# define __NR_getrandom 4353
# endif
# if _MIPS_SIM == _MIPS_SIM_NABI32
# define __NR_getrandom 6317
# endif
# if _MIPS_SIM == _MIPS_SIM_ABI64
# define __NR_getrandom 5313
# endif
# else
# warning "__NR_getrandom unknown for your architecture"
# endif
# endif
static inline int getrandom(void *buffer, size_t count, unsigned flags) {
# ifdef __NR_getrandom
return syscall(__NR_getrandom, buffer, count, flags);
# else
errno = ENOSYS;
return -1;
# endif
}
#endif
/* ======================================================================= */
#if !HAVE_DECL_GETTID
static inline pid_t gettid(void) {
return (pid_t) syscall(SYS_gettid);
}
#endif
/* ======================================================================= */
#if !HAVE_DECL_NAME_TO_HANDLE_AT
# ifndef __NR_name_to_handle_at
# if defined(__x86_64__)
# define __NR_name_to_handle_at 303
# elif defined(__i386__)
# define __NR_name_to_handle_at 341
# elif defined(__arm__)
# define __NR_name_to_handle_at 370
# elif defined(__powerpc__)
# define __NR_name_to_handle_at 345
# else
# error "__NR_name_to_handle_at is not defined"
# endif
# endif
struct file_handle {
unsigned int handle_bytes;
int handle_type;
unsigned char f_handle[0];
};
static inline int name_to_handle_at(int fd, const char *name, struct file_handle *handle, int *mnt_id, int flags) {
# ifdef __NR_name_to_handle_at
return syscall(__NR_name_to_handle_at, fd, name, handle, mnt_id, flags);
# else
errno = ENOSYS;
return -1;
# endif
}
#endif
/* ======================================================================= */
#if !HAVE_DECL_SETNS
# ifndef __NR_setns
# if defined(__x86_64__)
# define __NR_setns 308
# elif defined(__i386__)
# define __NR_setns 346
# else
# error "__NR_setns is not defined"
# endif
# endif
static inline int setns(int fd, int nstype) {
# ifdef __NR_setns
return syscall(__NR_setns, fd, nstype);
# else
errno = ENOSYS;
return -1;
# endif
}
#endif
/* ======================================================================= */
static inline int raw_clone(unsigned long flags, void *child_stack) {
#if defined(__s390__) || defined(__CRIS__)
/* On s390 and cris the order of the first and second arguments
* of the raw clone() system call is reversed. */
return (int) syscall(__NR_clone, child_stack, flags);
#else
return (int) syscall(__NR_clone, flags, child_stack);
#endif
}
/* ======================================================================= */
static inline pid_t raw_getpid(void) {
#if defined(__alpha__)
return (pid_t) syscall(__NR_getxpid);
#else
return (pid_t) syscall(__NR_getpid);
#endif
}
/* ======================================================================= */
#if !HAVE_DECL_RENAMEAT2
# ifndef __NR_renameat2
# if defined __x86_64__
# define __NR_renameat2 316
# elif defined __arm__
# define __NR_renameat2 382
# elif defined _MIPS_SIM
# if _MIPS_SIM == _MIPS_SIM_ABI32
# define __NR_renameat2 4351
# endif
# if _MIPS_SIM == _MIPS_SIM_NABI32
# define __NR_renameat2 6315
# endif
# if _MIPS_SIM == _MIPS_SIM_ABI64
# define __NR_renameat2 5311
# endif
# elif defined __i386__
# define __NR_renameat2 353
# else
# warning "__NR_renameat2 unknown for your architecture"
# endif
# endif
static inline int renameat2(int oldfd, const char *oldname, int newfd, const char *newname, unsigned flags) {
# ifdef __NR_renameat2
return syscall(__NR_renameat2, oldfd, oldname, newfd, newname, flags);
# else
errno = ENOSYS;
return -1;
# endif
}
#endif
/* ======================================================================= */
#if !HAVE_DECL_KCMP
static inline int kcmp(pid_t pid1, pid_t pid2, int type, unsigned long idx1, unsigned long idx2) {
# ifdef __NR_kcmp
return syscall(__NR_kcmp, pid1, pid2, type, idx1, idx2);
# else
errno = ENOSYS;
return -1;
# endif
}
#endif
/* ======================================================================= */
#if !HAVE_DECL_KEYCTL
static inline long keyctl(int cmd, unsigned long arg2, unsigned long arg3, unsigned long arg4,unsigned long arg5) {
# ifdef __NR_keyctl
return syscall(__NR_keyctl, cmd, arg2, arg3, arg4, arg5);
# else
errno = ENOSYS;
return -1;
# endif
}
static inline key_serial_t add_key(const char *type, const char *description, const void *payload, size_t plen, key_serial_t ringid) {
# ifdef __NR_add_key
return syscall(__NR_add_key, type, description, payload, plen, ringid);
# else
errno = ENOSYS;
return -1;
# endif
}
static inline key_serial_t request_key(const char *type, const char *description, const char * callout_info, key_serial_t destringid) {
# ifdef __NR_request_key
return syscall(__NR_request_key, type, description, callout_info, destringid);
# else
errno = ENOSYS;
return -1;
# endif
}
#endif
/* ======================================================================= */
#if !HAVE_DECL_COPY_FILE_RANGE
# ifndef __NR_copy_file_range
# if defined(__x86_64__)
# define __NR_copy_file_range 326
# elif defined(__i386__)
# define __NR_copy_file_range 377
# elif defined __s390__
# define __NR_copy_file_range 375
# elif defined __arm__
# define __NR_copy_file_range 391
# elif defined __aarch64__
# define __NR_copy_file_range 285
# else
# warning "__NR_copy_file_range not defined for your architecture"
# endif
# endif
static inline ssize_t copy_file_range(int fd_in, loff_t *off_in,
int fd_out, loff_t *off_out,
size_t len,
unsigned int flags) {
# ifdef __NR_copy_file_range
return syscall(__NR_copy_file_range, fd_in, off_in, fd_out, off_out, len, flags);
# else
errno = ENOSYS;
return -1;
# endif
}
#endif