mirror of
https://github.com/adulau/aha.git
synced 2024-12-27 19:26:25 +00:00
[PATCH] pipe: enable atomic copying of pipe data to/from user space
The pipe ->map() method uses kmap() to virtually map the pages, which is both slow and has known scalability issues on SMP. This patch enables atomic copying of pipe pages, by pre-faulting data and using kmap_atomic() instead. lmbench bw_pipe and lat_pipe measurements agree this is a Good Thing. Here are results from that on a UP machine with highmem (1.5GiB of RAM), running first a UP kernel, SMP kernel, and SMP kernel patched. Vanilla-UP: Pipe bandwidth: 1622.28 MB/sec Pipe bandwidth: 1610.59 MB/sec Pipe bandwidth: 1608.30 MB/sec Pipe latency: 7.3275 microseconds Pipe latency: 7.2995 microseconds Pipe latency: 7.3097 microseconds Vanilla-SMP: Pipe bandwidth: 1382.19 MB/sec Pipe bandwidth: 1317.27 MB/sec Pipe bandwidth: 1355.61 MB/sec Pipe latency: 9.6402 microseconds Pipe latency: 9.6696 microseconds Pipe latency: 9.6153 microseconds Patched-SMP: Pipe bandwidth: 1578.70 MB/sec Pipe bandwidth: 1579.95 MB/sec Pipe bandwidth: 1578.63 MB/sec Pipe latency: 9.1654 microseconds Pipe latency: 9.2266 microseconds Pipe latency: 9.1527 microseconds Signed-off-by: Jens Axboe <axboe@suse.de>
This commit is contained in:
Jens Axboe
parent
e27dedd84c
commit
f6762b7ad8
3 changed files with 126 additions and 30 deletions
141
fs/pipe.c
141
fs/pipe.c
|
|
@ -55,7 +55,8 @@ void pipe_wait(struct pipe_inode_info *pipe)
|
|||
}
|
||||
|
||||
static int
|
||||
pipe_iov_copy_from_user(void *to, struct iovec *iov, unsigned long len)
|
||||
pipe_iov_copy_from_user(void *to, struct iovec *iov, unsigned long len,
|
||||
int atomic)
|
||||
{
|
||||
unsigned long copy;
|
||||
|
||||
|
|
@ -64,8 +65,13 @@ pipe_iov_copy_from_user(void *to, struct iovec *iov, unsigned long len)
|
|||
iov++;
|
||||
copy = min_t(unsigned long, len, iov->iov_len);
|
||||
|
||||
if (copy_from_user(to, iov->iov_base, copy))
|
||||
return -EFAULT;
|
||||
if (atomic) {
|
||||
if (__copy_from_user_inatomic(to, iov->iov_base, copy))
|
||||
return -EFAULT;
|
||||
} else {
|
||||
if (copy_from_user(to, iov->iov_base, copy))
|
||||
return -EFAULT;
|
||||
}
|
||||
to += copy;
|
||||
len -= copy;
|
||||
iov->iov_base += copy;
|
||||
|
|
@ -75,7 +81,8 @@ pipe_iov_copy_from_user(void *to, struct iovec *iov, unsigned long len)
|
|||
}
|
||||
|
||||
static int
|
||||
pipe_iov_copy_to_user(struct iovec *iov, const void *from, unsigned long len)
|
||||
pipe_iov_copy_to_user(struct iovec *iov, const void *from, unsigned long len,
|
||||
int atomic)
|
||||
{
|
||||
unsigned long copy;
|
||||
|
||||
|
|
@ -84,8 +91,13 @@ pipe_iov_copy_to_user(struct iovec *iov, const void *from, unsigned long len)
|
|||
iov++;
|
||||
copy = min_t(unsigned long, len, iov->iov_len);
|
||||
|
||||
if (copy_to_user(iov->iov_base, from, copy))
|
||||
return -EFAULT;
|
||||
if (atomic) {
|
||||
if (__copy_to_user_inatomic(iov->iov_base, from, copy))
|
||||
return -EFAULT;
|
||||
} else {
|
||||
if (copy_to_user(iov->iov_base, from, copy))
|
||||
return -EFAULT;
|
||||
}
|
||||
from += copy;
|
||||
len -= copy;
|
||||
iov->iov_base += copy;
|
||||
|
|
@ -94,6 +106,47 @@ pipe_iov_copy_to_user(struct iovec *iov, const void *from, unsigned long len)
|
|||
return 0;
|
||||
}
|
||||
|
||||
/*
|
||||
* Attempt to pre-fault in the user memory, so we can use atomic copies.
|
||||
* Returns the number of bytes not faulted in.
|
||||
*/
|
||||
static int iov_fault_in_pages_write(struct iovec *iov, unsigned long len)
|
||||
{
|
||||
while (!iov->iov_len)
|
||||
iov++;
|
||||
|
||||
while (len > 0) {
|
||||
unsigned long this_len;
|
||||
|
||||
this_len = min_t(unsigned long, len, iov->iov_len);
|
||||
if (fault_in_pages_writeable(iov->iov_base, this_len))
|
||||
break;
|
||||
|
||||
len -= this_len;
|
||||
iov++;
|
||||
}
|
||||
|
||||
return len;
|
||||
}
|
||||
|
||||
/*
|
||||
* Pre-fault in the user memory, so we can use atomic copies.
|
||||
*/
|
||||
static void iov_fault_in_pages_read(struct iovec *iov, unsigned long len)
|
||||
{
|
||||
while (!iov->iov_len)
|
||||
iov++;
|
||||
|
||||
while (len > 0) {
|
||||
unsigned long this_len;
|
||||
|
||||
this_len = min_t(unsigned long, len, iov->iov_len);
|
||||
fault_in_pages_readable(iov->iov_base, this_len);
|
||||
len -= this_len;
|
||||
iov++;
|
||||
}
|
||||
}
|
||||
|
||||
static void anon_pipe_buf_release(struct pipe_inode_info *pipe,
|
||||
struct pipe_buffer *buf)
|
||||
{
|
||||
|
|
@ -111,15 +164,24 @@ static void anon_pipe_buf_release(struct pipe_inode_info *pipe,
|
|||
}
|
||||
|
||||
void *generic_pipe_buf_map(struct pipe_inode_info *pipe,
|
||||
struct pipe_buffer *buf)
|
||||
struct pipe_buffer *buf, int atomic)
|
||||
{
|
||||
if (atomic) {
|
||||
buf->flags |= PIPE_BUF_FLAG_ATOMIC;
|
||||
return kmap_atomic(buf->page, KM_USER0);
|
||||
}
|
||||
|
||||
return kmap(buf->page);
|
||||
}
|
||||
|
||||
void generic_pipe_buf_unmap(struct pipe_inode_info *pipe,
|
||||
struct pipe_buffer *buf)
|
||||
struct pipe_buffer *buf, void *map_data)
|
||||
{
|
||||
kunmap(buf->page);
|
||||
if (buf->flags & PIPE_BUF_FLAG_ATOMIC) {
|
||||
buf->flags &= ~PIPE_BUF_FLAG_ATOMIC;
|
||||
kunmap_atomic(map_data, KM_USER0);
|
||||
} else
|
||||
kunmap(buf->page);
|
||||
}
|
||||
|
||||
static int anon_pipe_buf_steal(struct pipe_inode_info *pipe,
|
||||
|
|
@ -183,7 +245,7 @@ pipe_readv(struct file *filp, const struct iovec *_iov,
|
|||
struct pipe_buf_operations *ops = buf->ops;
|
||||
void *addr;
|
||||
size_t chars = buf->len;
|
||||
int error;
|
||||
int error, atomic;
|
||||
|
||||
if (chars > total_len)
|
||||
chars = total_len;
|
||||
|
|
@ -195,12 +257,21 @@ pipe_readv(struct file *filp, const struct iovec *_iov,
|
|||
break;
|
||||
}
|
||||
|
||||
addr = ops->map(pipe, buf);
|
||||
error = pipe_iov_copy_to_user(iov, addr + buf->offset, chars);
|
||||
ops->unmap(pipe, buf);
|
||||
atomic = !iov_fault_in_pages_write(iov, chars);
|
||||
redo:
|
||||
addr = ops->map(pipe, buf, atomic);
|
||||
error = pipe_iov_copy_to_user(iov, addr + buf->offset, chars, atomic);
|
||||
ops->unmap(pipe, buf, addr);
|
||||
if (unlikely(error)) {
|
||||
/*
|
||||
* Just retry with the slow path if we failed.
|
||||
*/
|
||||
if (atomic) {
|
||||
atomic = 0;
|
||||
goto redo;
|
||||
}
|
||||
if (!ret)
|
||||
ret = -EFAULT;
|
||||
ret = error;
|
||||
break;
|
||||
}
|
||||
ret += chars;
|
||||
|
|
@ -304,21 +375,28 @@ pipe_writev(struct file *filp, const struct iovec *_iov,
|
|||
int offset = buf->offset + buf->len;
|
||||
|
||||
if (ops->can_merge && offset + chars <= PAGE_SIZE) {
|
||||
int error, atomic = 1;
|
||||
void *addr;
|
||||
int error;
|
||||
|
||||
error = ops->pin(pipe, buf);
|
||||
if (error)
|
||||
goto out;
|
||||
|
||||
addr = ops->map(pipe, buf);
|
||||
iov_fault_in_pages_read(iov, chars);
|
||||
redo1:
|
||||
addr = ops->map(pipe, buf, atomic);
|
||||
error = pipe_iov_copy_from_user(offset + addr, iov,
|
||||
chars);
|
||||
ops->unmap(pipe, buf);
|
||||
chars, atomic);
|
||||
ops->unmap(pipe, buf, addr);
|
||||
ret = error;
|
||||
do_wakeup = 1;
|
||||
if (error)
|
||||
if (error) {
|
||||
if (atomic) {
|
||||
atomic = 0;
|
||||
goto redo1;
|
||||
}
|
||||
goto out;
|
||||
}
|
||||
buf->len += chars;
|
||||
total_len -= chars;
|
||||
ret = chars;
|
||||
|
|
@ -341,7 +419,8 @@ pipe_writev(struct file *filp, const struct iovec *_iov,
|
|||
int newbuf = (pipe->curbuf + bufs) & (PIPE_BUFFERS-1);
|
||||
struct pipe_buffer *buf = pipe->bufs + newbuf;
|
||||
struct page *page = pipe->tmp_page;
|
||||
int error;
|
||||
char *src;
|
||||
int error, atomic = 1;
|
||||
|
||||
if (!page) {
|
||||
page = alloc_page(GFP_HIGHUSER);
|
||||
|
|
@ -361,11 +440,27 @@ pipe_writev(struct file *filp, const struct iovec *_iov,
|
|||
if (chars > total_len)
|
||||
chars = total_len;
|
||||
|
||||
error = pipe_iov_copy_from_user(kmap(page), iov, chars);
|
||||
kunmap(page);
|
||||
iov_fault_in_pages_read(iov, chars);
|
||||
redo2:
|
||||
if (atomic)
|
||||
src = kmap_atomic(page, KM_USER0);
|
||||
else
|
||||
src = kmap(page);
|
||||
|
||||
error = pipe_iov_copy_from_user(src, iov, chars,
|
||||
atomic);
|
||||
if (atomic)
|
||||
kunmap_atomic(src, KM_USER0);
|
||||
else
|
||||
kunmap(page);
|
||||
|
||||
if (unlikely(error)) {
|
||||
if (atomic) {
|
||||
atomic = 0;
|
||||
goto redo2;
|
||||
}
|
||||
if (!ret)
|
||||
ret = -EFAULT;
|
||||
ret = error;
|
||||
break;
|
||||
}
|
||||
ret += chars;
|
||||
|
|
|
|||
|
|
@ -640,13 +640,13 @@ find_page:
|
|||
/*
|
||||
* Careful, ->map() uses KM_USER0!
|
||||
*/
|
||||
char *src = buf->ops->map(info, buf);
|
||||
char *src = buf->ops->map(info, buf, 1);
|
||||
char *dst = kmap_atomic(page, KM_USER1);
|
||||
|
||||
memcpy(dst + offset, src + buf->offset, this_len);
|
||||
flush_dcache_page(page);
|
||||
kunmap_atomic(dst, KM_USER1);
|
||||
buf->ops->unmap(info, buf);
|
||||
buf->ops->unmap(info, buf, src);
|
||||
}
|
||||
|
||||
ret = mapping->a_ops->commit_write(file, page, offset, offset+this_len);
|
||||
|
|
|
|||
|
|
@ -5,7 +5,8 @@
|
|||
|
||||
#define PIPE_BUFFERS (16)
|
||||
|
||||
#define PIPE_BUF_FLAG_LRU 0x01
|
||||
#define PIPE_BUF_FLAG_LRU 0x01 /* page is on the LRU */
|
||||
#define PIPE_BUF_FLAG_ATOMIC 0x02 /* was atomically mapped */
|
||||
|
||||
struct pipe_buffer {
|
||||
struct page *page;
|
||||
|
|
@ -28,8 +29,8 @@ struct pipe_buffer {
|
|||
*/
|
||||
struct pipe_buf_operations {
|
||||
int can_merge;
|
||||
void * (*map)(struct pipe_inode_info *, struct pipe_buffer *);
|
||||
void (*unmap)(struct pipe_inode_info *, struct pipe_buffer *);
|
||||
void * (*map)(struct pipe_inode_info *, struct pipe_buffer *, int);
|
||||
void (*unmap)(struct pipe_inode_info *, struct pipe_buffer *, void *);
|
||||
int (*pin)(struct pipe_inode_info *, struct pipe_buffer *);
|
||||
void (*release)(struct pipe_inode_info *, struct pipe_buffer *);
|
||||
int (*steal)(struct pipe_inode_info *, struct pipe_buffer *);
|
||||
|
|
@ -64,8 +65,8 @@ void free_pipe_info(struct inode * inode);
|
|||
void __free_pipe_info(struct pipe_inode_info *);
|
||||
|
||||
/* Generic pipe buffer ops functions */
|
||||
void *generic_pipe_buf_map(struct pipe_inode_info *, struct pipe_buffer *);
|
||||
void generic_pipe_buf_unmap(struct pipe_inode_info *, struct pipe_buffer *);
|
||||
void *generic_pipe_buf_map(struct pipe_inode_info *, struct pipe_buffer *, int);
|
||||
void generic_pipe_buf_unmap(struct pipe_inode_info *, struct pipe_buffer *, void *);
|
||||
void generic_pipe_buf_get(struct pipe_inode_info *, struct pipe_buffer *);
|
||||
int generic_pipe_buf_pin(struct pipe_inode_info *, struct pipe_buffer *);
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue