From 53cd9ae886273d6c2b8ba4aa63d6cd6b1217b57f Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Sun, 2 Apr 2006 23:04:21 +0200 Subject: [PATCH 1/7] [PATCH] splice: fix shadow[] filling logic Clear the entire range, and don't increment pidx or we keep filling the same position again and again. Thanks to KAMEZAWA Hiroyuki. Signed-off-by: Jens Axboe --- fs/splice.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fs/splice.c b/fs/splice.c index 6081cf7d2d1..a555d0a83fe 100644 --- a/fs/splice.c +++ b/fs/splice.c @@ -237,9 +237,9 @@ static int __generic_file_splice_read(struct file *in, struct inode *pipe, * fill shadow[] with pages at the right locations, so we only * have to fill holes */ - memset(shadow, 0, i * sizeof(struct page *)); - for (j = 0, pidx = index; j < i; pidx++, j++) - shadow[pages[j]->index - pidx] = pages[j]; + memset(shadow, 0, nr_pages * sizeof(struct page *)); + for (j = 0; j < i; j++) + shadow[pages[j]->index - index] = pages[j]; /* * now fill in the holes From 4f6f0bd2ffa4e31c3524f5e65c84a29b6ab73307 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Sun, 2 Apr 2006 23:04:46 +0200 Subject: [PATCH 2/7] [PATCH] splice: improve writeback and clean up page stealing By cleaning up the writeback logic (killing write_one_page() and the manual set_page_dirty()), we can get rid of ->stolen inside the pipe_buffer and just keep it local in pipe_to_file(). This also adds dirty page balancing logic and O_SYNC handling. Signed-off-by: Jens Axboe --- fs/pipe.c | 1 - fs/splice.c | 64 +++++++++++++++++++++++++++++---------- include/linux/pipe_fs_i.h | 1 - 3 files changed, 48 insertions(+), 18 deletions(-) diff --git a/fs/pipe.c b/fs/pipe.c index 109a102c150..5093408746b 100644 --- a/fs/pipe.c +++ b/fs/pipe.c @@ -124,7 +124,6 @@ static void anon_pipe_buf_unmap(struct pipe_inode_info *info, struct pipe_buffer static int anon_pipe_buf_steal(struct pipe_inode_info *info, struct pipe_buffer *buf) { - buf->stolen = 1; return 0; } diff --git a/fs/splice.c b/fs/splice.c index a555d0a83fe..07f4d863c2d 100644 --- a/fs/splice.c +++ b/fs/splice.c @@ -22,7 +22,10 @@ #include #include #include +#include +#include #include +#include /* * Passed to the actors @@ -38,11 +41,15 @@ static int page_cache_pipe_buf_steal(struct pipe_inode_info *info, struct pipe_buffer *buf) { struct page *page = buf->page; + struct address_space *mapping = page_mapping(page); WARN_ON(!PageLocked(page)); WARN_ON(!PageUptodate(page)); - if (!remove_mapping(page_mapping(page), page)) + if (PagePrivate(page)) + try_to_release_page(page, mapping_gfp_mask(mapping)); + + if (!remove_mapping(mapping, page)) return 1; if (PageLRU(page)) { @@ -55,7 +62,6 @@ static int page_cache_pipe_buf_steal(struct pipe_inode_info *info, spin_unlock_irq(&zone->lru_lock); } - buf->stolen = 1; return 0; } @@ -64,7 +70,6 @@ static void page_cache_pipe_buf_release(struct pipe_inode_info *info, { page_cache_release(buf->page); buf->page = NULL; - buf->stolen = 0; } static void *page_cache_pipe_buf_map(struct file *file, @@ -91,8 +96,7 @@ static void *page_cache_pipe_buf_map(struct file *file, static void page_cache_pipe_buf_unmap(struct pipe_inode_info *info, struct pipe_buffer *buf) { - if (!buf->stolen) - unlock_page(buf->page); + unlock_page(buf->page); kunmap(buf->page); } @@ -319,7 +323,8 @@ ssize_t generic_file_splice_read(struct file *in, struct inode *pipe, } /* - * Send 'len' bytes to socket from 'file' at position 'pos' using sendpage(). + * Send 'sd->len' bytes to socket from 'sd->file' at position 'sd->pos' + * using sendpage(). */ static int pipe_to_sendpage(struct pipe_inode_info *info, struct pipe_buffer *buf, struct splice_desc *sd) @@ -379,7 +384,7 @@ static int pipe_to_file(struct pipe_inode_info *info, struct pipe_buffer *buf, struct page *page; pgoff_t index; char *src; - int ret; + int ret, stolen; /* * after this, page will be locked and unmapped @@ -390,6 +395,7 @@ static int pipe_to_file(struct pipe_inode_info *info, struct pipe_buffer *buf, index = sd->pos >> PAGE_CACHE_SHIFT; offset = sd->pos & ~PAGE_CACHE_MASK; + stolen = 0; /* * reuse buf page, if SPLICE_F_MOVE is set @@ -399,6 +405,7 @@ static int pipe_to_file(struct pipe_inode_info *info, struct pipe_buffer *buf, goto find_page; page = buf->page; + stolen = 1; if (add_to_page_cache_lru(page, mapping, index, mapping_gfp_mask(mapping))) goto find_page; @@ -443,10 +450,13 @@ find_page: } ret = mapping->a_ops->prepare_write(file, page, 0, sd->len); - if (ret) + if (ret == AOP_TRUNCATED_PAGE) { + page_cache_release(page); + goto find_page; + } else if (ret) goto out; - if (!buf->stolen) { + if (!stolen) { char *dst = kmap_atomic(page, KM_USER0); memcpy(dst + offset, src + buf->offset, sd->len); @@ -455,16 +465,18 @@ find_page: } ret = mapping->a_ops->commit_write(file, page, 0, sd->len); - if (ret < 0) + if (ret == AOP_TRUNCATED_PAGE) { + page_cache_release(page); + goto find_page; + } else if (ret) goto out; - set_page_dirty(page); - ret = write_one_page(page, 0); + balance_dirty_pages_ratelimited(mapping); out: - if (ret < 0) - unlock_page(page); - if (!buf->stolen) + if (!stolen) { page_cache_release(page); + unlock_page(page); + } buf->ops->unmap(info, buf); return ret; } @@ -576,7 +588,27 @@ static ssize_t move_from_pipe(struct inode *inode, struct file *out, ssize_t generic_file_splice_write(struct inode *inode, struct file *out, size_t len, unsigned int flags) { - return move_from_pipe(inode, out, len, flags, pipe_to_file); + struct address_space *mapping = out->f_mapping; + ssize_t ret = move_from_pipe(inode, out, len, flags, pipe_to_file); + + /* + * if file or inode is SYNC and we actually wrote some data, sync it + */ + if (unlikely((out->f_flags & O_SYNC) || IS_SYNC(mapping->host)) + && ret > 0) { + struct inode *inode = mapping->host; + int err; + + mutex_lock(&inode->i_mutex); + err = generic_osync_inode(mapping->host, mapping, + OSYNC_METADATA|OSYNC_DATA); + mutex_unlock(&inode->i_mutex); + + if (err) + ret = err; + } + + return ret; } ssize_t generic_splice_sendpage(struct inode *inode, struct file *out, diff --git a/include/linux/pipe_fs_i.h b/include/linux/pipe_fs_i.h index d218fc72931..32865004751 100644 --- a/include/linux/pipe_fs_i.h +++ b/include/linux/pipe_fs_i.h @@ -9,7 +9,6 @@ struct pipe_buffer { struct page *page; unsigned int offset, len; struct pipe_buf_operations *ops; - unsigned int stolen; }; struct pipe_buf_operations { From 83f9135bddffded9f1716519b6c147bcf046c87e Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Sun, 2 Apr 2006 23:05:09 +0200 Subject: [PATCH 3/7] [PATCH] splice: add comments documenting more of the code Hopefully this will make Andrew a little more happy. Signed-off-by: Jens Axboe --- fs/splice.c | 70 ++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 66 insertions(+), 4 deletions(-) diff --git a/fs/splice.c b/fs/splice.c index 07f4d863c2d..34591924c78 100644 --- a/fs/splice.c +++ b/fs/splice.c @@ -37,6 +37,12 @@ struct splice_desc { loff_t pos; /* file position */ }; +/* + * Attempt to steal a page from a pipe buffer. This should perhaps go into + * a vm helper function, it's already simplified quite a bit by the + * addition of remove_mapping(). If success is returned, the caller may + * attempt to reuse this page for another destination. + */ static int page_cache_pipe_buf_steal(struct pipe_inode_info *info, struct pipe_buffer *buf) { @@ -108,6 +114,10 @@ static struct pipe_buf_operations page_cache_pipe_buf_ops = { .steal = page_cache_pipe_buf_steal, }; +/* + * Pipe output worker. This sets up our pipe format with the page cache + * pipe buffer operations. Otherwise very similar to the regular pipe_writev(). + */ static ssize_t move_to_pipe(struct inode *inode, struct page **pages, int nr_pages, unsigned long offset, unsigned long len, unsigned int flags) @@ -292,6 +302,16 @@ splice_them: return move_to_pipe(pipe, pages, i, offset, len, flags); } +/** + * generic_file_splice_read - splice data from file to a pipe + * @in: file to splice from + * @pipe: pipe to splice to + * @len: number of bytes to splice + * @flags: splice modifier flags + * + * Will read pages from given file and fill them into a pipe. + * + */ ssize_t generic_file_splice_read(struct file *in, struct inode *pipe, size_t len, unsigned int flags) { @@ -370,10 +390,12 @@ static int pipe_to_sendpage(struct pipe_inode_info *info, * - Destination page does not exist, we can add the pipe page to * the page cache and avoid the copy. * - * For now we just do the slower thing and always copy pages over, it's - * easier than migrating pages from the pipe to the target file. For the - * case of doing file | file splicing, the migrate approach had some LRU - * nastiness... + * If asked to move pages to the output file (SPLICE_F_MOVE is set in + * sd->flags), we attempt to migrate pages from the pipe to the output + * file address space page cache. This is possible if no one else has + * the pipe page referenced outside of the pipe and page cache. If + * SPLICE_F_MOVE isn't set, or we cannot move the page, we simply create + * a new page in the output file page cache and fill/dirty that. */ static int pipe_to_file(struct pipe_inode_info *info, struct pipe_buffer *buf, struct splice_desc *sd) @@ -401,6 +423,10 @@ static int pipe_to_file(struct pipe_inode_info *info, struct pipe_buffer *buf, * reuse buf page, if SPLICE_F_MOVE is set */ if (sd->flags & SPLICE_F_MOVE) { + /* + * If steal succeeds, buf->page is now pruned from the vm + * side (LRU and page cache) and we can reuse it. + */ if (buf->ops->steal(info, buf)) goto find_page; @@ -484,6 +510,11 @@ out: typedef int (splice_actor)(struct pipe_inode_info *, struct pipe_buffer *, struct splice_desc *); +/* + * Pipe input worker. Most of this logic works like a regular pipe, the + * key here is the 'actor' worker passed in that actually moves the data + * to the wanted destination. See pipe_to_file/pipe_to_sendpage above. + */ static ssize_t move_from_pipe(struct inode *inode, struct file *out, size_t len, unsigned int flags, splice_actor *actor) @@ -585,6 +616,17 @@ static ssize_t move_from_pipe(struct inode *inode, struct file *out, } +/** + * generic_file_splice_write - splice data from a pipe to a file + * @inode: pipe inode + * @out: file to write to + * @len: number of bytes to splice + * @flags: splice modifier flags + * + * Will either move or copy pages (determined by @flags options) from + * the given pipe inode to the given file. + * + */ ssize_t generic_file_splice_write(struct inode *inode, struct file *out, size_t len, unsigned int flags) { @@ -611,6 +653,17 @@ ssize_t generic_file_splice_write(struct inode *inode, struct file *out, return ret; } +/** + * generic_splice_sendpage - splice data from a pipe to a socket + * @inode: pipe inode + * @out: socket to write to + * @len: number of bytes to splice + * @flags: splice modifier flags + * + * Will send @len bytes from the pipe to a network socket. No data copying + * is involved. + * + */ ssize_t generic_splice_sendpage(struct inode *inode, struct file *out, size_t len, unsigned int flags) { @@ -620,6 +673,9 @@ ssize_t generic_splice_sendpage(struct inode *inode, struct file *out, EXPORT_SYMBOL(generic_file_splice_write); EXPORT_SYMBOL(generic_file_splice_read); +/* + * Attempt to initiate a splice from pipe to file. + */ static long do_splice_from(struct inode *pipe, struct file *out, size_t len, unsigned int flags) { @@ -640,6 +696,9 @@ static long do_splice_from(struct inode *pipe, struct file *out, size_t len, return out->f_op->splice_write(pipe, out, len, flags); } +/* + * Attempt to initiate a splice from a file to a pipe. + */ static long do_splice_to(struct file *in, struct inode *pipe, size_t len, unsigned int flags) { @@ -668,6 +727,9 @@ static long do_splice_to(struct file *in, struct inode *pipe, size_t len, return in->f_op->splice_read(in, pipe, len, flags); } +/* + * Determine where to splice to/from. + */ static long do_splice(struct file *in, struct file *out, size_t len, unsigned int flags) { From b2b39fa478db6db89b7ccafb0649973845b0eb75 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Sun, 2 Apr 2006 23:05:41 +0200 Subject: [PATCH 4/7] [PATCH] splice: add a SPLICE_F_MORE flag This lets userspace indicate whether more data will be coming in a subsequent splice call. Signed-off-by: Jens Axboe --- fs/splice.c | 5 +++-- include/linux/pipe_fs_i.h | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/fs/splice.c b/fs/splice.c index 34591924c78..e8454455894 100644 --- a/fs/splice.c +++ b/fs/splice.c @@ -354,6 +354,7 @@ static int pipe_to_sendpage(struct pipe_inode_info *info, unsigned int offset; ssize_t ret; void *ptr; + int more; /* * sub-optimal, but we are limited by the pipe ->map. we don't @@ -366,9 +367,9 @@ static int pipe_to_sendpage(struct pipe_inode_info *info, return PTR_ERR(ptr); offset = pos & ~PAGE_CACHE_MASK; + more = (sd->flags & SPLICE_F_MORE) || sd->len < sd->total_len; - ret = file->f_op->sendpage(file, buf->page, offset, sd->len, &pos, - sd->len < sd->total_len); + ret = file->f_op->sendpage(file, buf->page, offset, sd->len, &pos,more); buf->ops->unmap(info, buf); if (ret == sd->len) diff --git a/include/linux/pipe_fs_i.h b/include/linux/pipe_fs_i.h index 32865004751..70ae9332ec1 100644 --- a/include/linux/pipe_fs_i.h +++ b/include/linux/pipe_fs_i.h @@ -62,5 +62,6 @@ void free_pipe_info(struct inode* inode); #define SPLICE_F_NONBLOCK (0x02) /* don't block on the pipe splicing (but */ /* we may still block on the fd we splice */ /* from/to, of course */ +#define SPLICE_F_MORE (0x04) /* expect more data */ #endif From 059a8f3734a66cb2c41731083b0cbf836721485b Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Sun, 2 Apr 2006 23:06:05 +0200 Subject: [PATCH 5/7] [PATCH] splice: export generic_splice_sendpage Forgot that one, thanks Jeff. Also move the other EXPORT_SYMBOL to right below the functions. Signed-off-by: Jens Axboe --- fs/splice.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/fs/splice.c b/fs/splice.c index e8454455894..7c2abd4504d 100644 --- a/fs/splice.c +++ b/fs/splice.c @@ -342,6 +342,8 @@ ssize_t generic_file_splice_read(struct file *in, struct inode *pipe, return ret; } +EXPORT_SYMBOL(generic_file_splice_read); + /* * Send 'sd->len' bytes to socket from 'sd->file' at position 'sd->pos' * using sendpage(). @@ -654,6 +656,8 @@ ssize_t generic_file_splice_write(struct inode *inode, struct file *out, return ret; } +EXPORT_SYMBOL(generic_file_splice_write); + /** * generic_splice_sendpage - splice data from a pipe to a socket * @inode: pipe inode @@ -671,8 +675,7 @@ ssize_t generic_splice_sendpage(struct inode *inode, struct file *out, return move_from_pipe(inode, out, len, flags, pipe_to_sendpage); } -EXPORT_SYMBOL(generic_file_splice_write); -EXPORT_SYMBOL(generic_file_splice_read); +EXPORT_SYMBOL(generic_splice_sendpage); /* * Attempt to initiate a splice from pipe to file. From ad8d6f0a783ffa2ff9b0cf09910b889715772201 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Sun, 2 Apr 2006 23:10:32 +0200 Subject: [PATCH 6/7] [PATCH] splice: page stealing needs to wait_on_page_writeback() Thanks to Andrew for the good explanation of why this is so. akpm writes: If a page is under writeback and we remove it from pagecache, it's still going to get written to disk. But the VFS no longer knows about that page, nor that this page is about to modify disk blocks. So there might be scenarios in which those blocks-which-are-about-to-be-written-to get reused for something else. When writeback completes, it'll scribble on those blocks. This won't happen in ext2/ext3-style filesystems in normal mode because the page has buffers and try_to_release_page() will fail. But ext2 in nobh mode doesn't attach buffers at all - it just sticks the page in a BIO, finds some new blocks, points the BIO at those blocks and lets it rip. While that write IO's in flight, someone could truncate the file. Truncate won't block on the writeout because the page isn't in pagecache any more. So truncate will the free the blocks from the file under the page's feet. Then something else can reallocate those blocks. Then write data to them. Now, the original write completes, corrupting the filesystem. Signed-off-by: Jens Axboe --- fs/splice.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/fs/splice.c b/fs/splice.c index 7c2abd4504d..b5fb2f3e3ac 100644 --- a/fs/splice.c +++ b/fs/splice.c @@ -52,6 +52,15 @@ static int page_cache_pipe_buf_steal(struct pipe_inode_info *info, WARN_ON(!PageLocked(page)); WARN_ON(!PageUptodate(page)); + /* + * At least for ext2 with nobh option, we need to wait on writeback + * completing on this page, since we'll remove it from the pagecache. + * Otherwise truncate wont wait on the page, allowing the disk + * blocks to be reused by someone else before we actually wrote our + * data to them. fs corruption ensues. + */ + wait_on_page_writeback(page); + if (PagePrivate(page)) try_to_release_page(page, mapping_gfp_mask(mapping)); From 3e7ee3e7b36fa4e2d88d8fb0a2577be95fc4636d Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Sun, 2 Apr 2006 23:11:04 +0200 Subject: [PATCH 7/7] [PATCH] splice: fix page stealing LRU handling. Originally from Nick Piggin, just adapted to the newer branch. You can't check PageLRU without holding zone->lru_lock. The page release code can get away with it only because the page refcount is 0 at that point. Also, you can't reliably remove pages from the LRU unless the refcount is 0. Ever. Signed-off-by: Nick Piggin Signed-off-by: Jens Axboe --- fs/pipe.c | 3 +++ fs/splice.c | 30 +++++++++++------------------- include/linux/pipe_fs_i.h | 4 ++++ 3 files changed, 18 insertions(+), 19 deletions(-) diff --git a/fs/pipe.c b/fs/pipe.c index 5093408746b..795df987cd3 100644 --- a/fs/pipe.c +++ b/fs/pipe.c @@ -95,6 +95,8 @@ static void anon_pipe_buf_release(struct pipe_inode_info *info, struct pipe_buff { struct page *page = buf->page; + buf->flags &= ~PIPE_BUF_FLAG_STOLEN; + /* * If nobody else uses this page, and we don't already have a * temporary page, let's keep track of it as a one-deep @@ -124,6 +126,7 @@ static void anon_pipe_buf_unmap(struct pipe_inode_info *info, struct pipe_buffer static int anon_pipe_buf_steal(struct pipe_inode_info *info, struct pipe_buffer *buf) { + buf->flags |= PIPE_BUF_FLAG_STOLEN; return 0; } diff --git a/fs/splice.c b/fs/splice.c index b5fb2f3e3ac..bfa42a277bb 100644 --- a/fs/splice.c +++ b/fs/splice.c @@ -67,16 +67,7 @@ static int page_cache_pipe_buf_steal(struct pipe_inode_info *info, if (!remove_mapping(mapping, page)) return 1; - if (PageLRU(page)) { - struct zone *zone = page_zone(page); - - spin_lock_irq(&zone->lru_lock); - BUG_ON(!PageLRU(page)); - __ClearPageLRU(page); - del_page_from_lru(zone, page); - spin_unlock_irq(&zone->lru_lock); - } - + buf->flags |= PIPE_BUF_FLAG_STOLEN | PIPE_BUF_FLAG_LRU; return 0; } @@ -85,6 +76,7 @@ static void page_cache_pipe_buf_release(struct pipe_inode_info *info, { page_cache_release(buf->page); buf->page = NULL; + buf->flags &= ~(PIPE_BUF_FLAG_STOLEN | PIPE_BUF_FLAG_LRU); } static void *page_cache_pipe_buf_map(struct file *file, @@ -414,11 +406,12 @@ static int pipe_to_file(struct pipe_inode_info *info, struct pipe_buffer *buf, { struct file *file = sd->file; struct address_space *mapping = file->f_mapping; + gfp_t gfp_mask = mapping_gfp_mask(mapping); unsigned int offset; struct page *page; pgoff_t index; char *src; - int ret, stolen; + int ret; /* * after this, page will be locked and unmapped @@ -429,7 +422,6 @@ static int pipe_to_file(struct pipe_inode_info *info, struct pipe_buffer *buf, index = sd->pos >> PAGE_CACHE_SHIFT; offset = sd->pos & ~PAGE_CACHE_MASK; - stolen = 0; /* * reuse buf page, if SPLICE_F_MOVE is set @@ -443,15 +435,15 @@ static int pipe_to_file(struct pipe_inode_info *info, struct pipe_buffer *buf, goto find_page; page = buf->page; - stolen = 1; - if (add_to_page_cache_lru(page, mapping, index, - mapping_gfp_mask(mapping))) + if (add_to_page_cache(page, mapping, index, gfp_mask)) goto find_page; + + if (!(buf->flags & PIPE_BUF_FLAG_LRU)) + lru_cache_add(page); } else { find_page: ret = -ENOMEM; - page = find_or_create_page(mapping, index, - mapping_gfp_mask(mapping)); + page = find_or_create_page(mapping, index, gfp_mask); if (!page) goto out; @@ -494,7 +486,7 @@ find_page: } else if (ret) goto out; - if (!stolen) { + if (!(buf->flags & PIPE_BUF_FLAG_STOLEN)) { char *dst = kmap_atomic(page, KM_USER0); memcpy(dst + offset, src + buf->offset, sd->len); @@ -511,7 +503,7 @@ find_page: balance_dirty_pages_ratelimited(mapping); out: - if (!stolen) { + if (!(buf->flags & PIPE_BUF_FLAG_STOLEN)) { page_cache_release(page); unlock_page(page); } diff --git a/include/linux/pipe_fs_i.h b/include/linux/pipe_fs_i.h index 70ae9332ec1..ec384958d50 100644 --- a/include/linux/pipe_fs_i.h +++ b/include/linux/pipe_fs_i.h @@ -5,10 +5,14 @@ #define PIPE_BUFFERS (16) +#define PIPE_BUF_FLAG_STOLEN 0x01 +#define PIPE_BUF_FLAG_LRU 0x02 + struct pipe_buffer { struct page *page; unsigned int offset, len; struct pipe_buf_operations *ops; + unsigned int flags; }; struct pipe_buf_operations {