read.c 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * linux/fs/nfs/read.c
  4. *
  5. * Block I/O for NFS
  6. *
  7. * Partial copy of Linus' read cache modifications to fs/nfs/file.c
  8. * modified for async RPC by [email protected]
  9. */
  10. #include <linux/time.h>
  11. #include <linux/kernel.h>
  12. #include <linux/errno.h>
  13. #include <linux/fcntl.h>
  14. #include <linux/stat.h>
  15. #include <linux/mm.h>
  16. #include <linux/slab.h>
  17. #include <linux/task_io_accounting_ops.h>
  18. #include <linux/pagemap.h>
  19. #include <linux/sunrpc/clnt.h>
  20. #include <linux/nfs_fs.h>
  21. #include <linux/nfs_page.h>
  22. #include <linux/module.h>
  23. #include "nfs4_fs.h"
  24. #include "internal.h"
  25. #include "iostat.h"
  26. #include "fscache.h"
  27. #include "pnfs.h"
  28. #include "nfstrace.h"
  29. #define NFSDBG_FACILITY NFSDBG_PAGECACHE
  30. static const struct nfs_pgio_completion_ops nfs_async_read_completion_ops;
  31. static const struct nfs_rw_ops nfs_rw_read_ops;
  32. static struct kmem_cache *nfs_rdata_cachep;
  33. static struct nfs_pgio_header *nfs_readhdr_alloc(void)
  34. {
  35. struct nfs_pgio_header *p = kmem_cache_zalloc(nfs_rdata_cachep, GFP_KERNEL);
  36. if (p)
  37. p->rw_mode = FMODE_READ;
  38. return p;
  39. }
  40. static void nfs_readhdr_free(struct nfs_pgio_header *rhdr)
  41. {
  42. if (rhdr->res.scratch != NULL)
  43. kfree(rhdr->res.scratch);
  44. kmem_cache_free(nfs_rdata_cachep, rhdr);
  45. }
  46. static
  47. int nfs_return_empty_page(struct page *page)
  48. {
  49. zero_user(page, 0, PAGE_SIZE);
  50. SetPageUptodate(page);
  51. unlock_page(page);
  52. return 0;
  53. }
  54. void nfs_pageio_init_read(struct nfs_pageio_descriptor *pgio,
  55. struct inode *inode, bool force_mds,
  56. const struct nfs_pgio_completion_ops *compl_ops)
  57. {
  58. struct nfs_server *server = NFS_SERVER(inode);
  59. const struct nfs_pageio_ops *pg_ops = &nfs_pgio_rw_ops;
  60. #ifdef CONFIG_NFS_V4_1
  61. if (server->pnfs_curr_ld && !force_mds)
  62. pg_ops = server->pnfs_curr_ld->pg_read_ops;
  63. #endif
  64. nfs_pageio_init(pgio, inode, pg_ops, compl_ops, &nfs_rw_read_ops,
  65. server->rsize, 0);
  66. }
  67. EXPORT_SYMBOL_GPL(nfs_pageio_init_read);
  68. static void nfs_pageio_complete_read(struct nfs_pageio_descriptor *pgio)
  69. {
  70. struct nfs_pgio_mirror *pgm;
  71. unsigned long npages;
  72. nfs_pageio_complete(pgio);
  73. /* It doesn't make sense to do mirrored reads! */
  74. WARN_ON_ONCE(pgio->pg_mirror_count != 1);
  75. pgm = &pgio->pg_mirrors[0];
  76. NFS_I(pgio->pg_inode)->read_io += pgm->pg_bytes_written;
  77. npages = (pgm->pg_bytes_written + PAGE_SIZE - 1) >> PAGE_SHIFT;
  78. nfs_add_stats(pgio->pg_inode, NFSIOS_READPAGES, npages);
  79. }
  80. void nfs_pageio_reset_read_mds(struct nfs_pageio_descriptor *pgio)
  81. {
  82. struct nfs_pgio_mirror *mirror;
  83. if (pgio->pg_ops && pgio->pg_ops->pg_cleanup)
  84. pgio->pg_ops->pg_cleanup(pgio);
  85. pgio->pg_ops = &nfs_pgio_rw_ops;
  86. /* read path should never have more than one mirror */
  87. WARN_ON_ONCE(pgio->pg_mirror_count != 1);
  88. mirror = &pgio->pg_mirrors[0];
  89. mirror->pg_bsize = NFS_SERVER(pgio->pg_inode)->rsize;
  90. }
  91. EXPORT_SYMBOL_GPL(nfs_pageio_reset_read_mds);
  92. bool nfs_read_alloc_scratch(struct nfs_pgio_header *hdr, size_t size)
  93. {
  94. WARN_ON(hdr->res.scratch != NULL);
  95. hdr->res.scratch = kmalloc(size, GFP_KERNEL);
  96. return hdr->res.scratch != NULL;
  97. }
  98. EXPORT_SYMBOL_GPL(nfs_read_alloc_scratch);
  99. static void nfs_readpage_release(struct nfs_page *req, int error)
  100. {
  101. struct inode *inode = d_inode(nfs_req_openctx(req)->dentry);
  102. struct page *page = req->wb_page;
  103. dprintk("NFS: read done (%s/%llu %d@%lld)\n", inode->i_sb->s_id,
  104. (unsigned long long)NFS_FILEID(inode), req->wb_bytes,
  105. (long long)req_offset(req));
  106. if (nfs_error_is_fatal_on_server(error) && error != -ETIMEDOUT)
  107. SetPageError(page);
  108. if (nfs_page_group_sync_on_bit(req, PG_UNLOCKPAGE)) {
  109. if (PageUptodate(page))
  110. nfs_fscache_write_page(inode, page);
  111. unlock_page(page);
  112. }
  113. nfs_release_request(req);
  114. }
  115. struct nfs_readdesc {
  116. struct nfs_pageio_descriptor pgio;
  117. struct nfs_open_context *ctx;
  118. };
  119. static void nfs_page_group_set_uptodate(struct nfs_page *req)
  120. {
  121. if (nfs_page_group_sync_on_bit(req, PG_UPTODATE))
  122. SetPageUptodate(req->wb_page);
  123. }
  124. static void nfs_read_completion(struct nfs_pgio_header *hdr)
  125. {
  126. unsigned long bytes = 0;
  127. int error;
  128. if (test_bit(NFS_IOHDR_REDO, &hdr->flags))
  129. goto out;
  130. while (!list_empty(&hdr->pages)) {
  131. struct nfs_page *req = nfs_list_entry(hdr->pages.next);
  132. struct page *page = req->wb_page;
  133. unsigned long start = req->wb_pgbase;
  134. unsigned long end = req->wb_pgbase + req->wb_bytes;
  135. if (test_bit(NFS_IOHDR_EOF, &hdr->flags)) {
  136. /* note: regions of the page not covered by a
  137. * request are zeroed in readpage_async_filler */
  138. if (bytes > hdr->good_bytes) {
  139. /* nothing in this request was good, so zero
  140. * the full extent of the request */
  141. zero_user_segment(page, start, end);
  142. } else if (hdr->good_bytes - bytes < req->wb_bytes) {
  143. /* part of this request has good bytes, but
  144. * not all. zero the bad bytes */
  145. start += hdr->good_bytes - bytes;
  146. WARN_ON(start < req->wb_pgbase);
  147. zero_user_segment(page, start, end);
  148. }
  149. }
  150. error = 0;
  151. bytes += req->wb_bytes;
  152. if (test_bit(NFS_IOHDR_ERROR, &hdr->flags)) {
  153. if (bytes <= hdr->good_bytes)
  154. nfs_page_group_set_uptodate(req);
  155. else {
  156. error = hdr->error;
  157. xchg(&nfs_req_openctx(req)->error, error);
  158. }
  159. } else
  160. nfs_page_group_set_uptodate(req);
  161. nfs_list_remove_request(req);
  162. nfs_readpage_release(req, error);
  163. }
  164. out:
  165. hdr->release(hdr);
  166. }
  167. static void nfs_initiate_read(struct nfs_pgio_header *hdr,
  168. struct rpc_message *msg,
  169. const struct nfs_rpc_ops *rpc_ops,
  170. struct rpc_task_setup *task_setup_data, int how)
  171. {
  172. rpc_ops->read_setup(hdr, msg);
  173. trace_nfs_initiate_read(hdr);
  174. }
  175. static void
  176. nfs_async_read_error(struct list_head *head, int error)
  177. {
  178. struct nfs_page *req;
  179. while (!list_empty(head)) {
  180. req = nfs_list_entry(head->next);
  181. nfs_list_remove_request(req);
  182. nfs_readpage_release(req, error);
  183. }
  184. }
  185. static const struct nfs_pgio_completion_ops nfs_async_read_completion_ops = {
  186. .error_cleanup = nfs_async_read_error,
  187. .completion = nfs_read_completion,
  188. };
  189. /*
  190. * This is the callback from RPC telling us whether a reply was
  191. * received or some error occurred (timeout or socket shutdown).
  192. */
  193. static int nfs_readpage_done(struct rpc_task *task,
  194. struct nfs_pgio_header *hdr,
  195. struct inode *inode)
  196. {
  197. int status = NFS_PROTO(inode)->read_done(task, hdr);
  198. if (status != 0)
  199. return status;
  200. nfs_add_stats(inode, NFSIOS_SERVERREADBYTES, hdr->res.count);
  201. trace_nfs_readpage_done(task, hdr);
  202. if (task->tk_status == -ESTALE) {
  203. nfs_set_inode_stale(inode);
  204. nfs_mark_for_revalidate(inode);
  205. }
  206. return 0;
  207. }
  208. static void nfs_readpage_retry(struct rpc_task *task,
  209. struct nfs_pgio_header *hdr)
  210. {
  211. struct nfs_pgio_args *argp = &hdr->args;
  212. struct nfs_pgio_res *resp = &hdr->res;
  213. /* This is a short read! */
  214. nfs_inc_stats(hdr->inode, NFSIOS_SHORTREAD);
  215. trace_nfs_readpage_short(task, hdr);
  216. /* Has the server at least made some progress? */
  217. if (resp->count == 0) {
  218. nfs_set_pgio_error(hdr, -EIO, argp->offset);
  219. return;
  220. }
  221. /* For non rpc-based layout drivers, retry-through-MDS */
  222. if (!task->tk_ops) {
  223. hdr->pnfs_error = -EAGAIN;
  224. return;
  225. }
  226. /* Yes, so retry the read at the end of the hdr */
  227. hdr->mds_offset += resp->count;
  228. argp->offset += resp->count;
  229. argp->pgbase += resp->count;
  230. argp->count -= resp->count;
  231. resp->count = 0;
  232. resp->eof = 0;
  233. rpc_restart_call_prepare(task);
  234. }
  235. static void nfs_readpage_result(struct rpc_task *task,
  236. struct nfs_pgio_header *hdr)
  237. {
  238. if (hdr->res.eof) {
  239. loff_t pos = hdr->args.offset + hdr->res.count;
  240. unsigned int new = pos - hdr->io_start;
  241. if (hdr->good_bytes > new) {
  242. hdr->good_bytes = new;
  243. set_bit(NFS_IOHDR_EOF, &hdr->flags);
  244. clear_bit(NFS_IOHDR_ERROR, &hdr->flags);
  245. }
  246. } else if (hdr->res.count < hdr->args.count)
  247. nfs_readpage_retry(task, hdr);
  248. }
  249. static int
  250. readpage_async_filler(struct nfs_readdesc *desc, struct page *page)
  251. {
  252. struct inode *inode = page_file_mapping(page)->host;
  253. unsigned int rsize = NFS_SERVER(inode)->rsize;
  254. struct nfs_page *new;
  255. unsigned int len, aligned_len;
  256. int error;
  257. len = nfs_page_length(page);
  258. if (len == 0)
  259. return nfs_return_empty_page(page);
  260. aligned_len = min_t(unsigned int, ALIGN(len, rsize), PAGE_SIZE);
  261. if (!IS_SYNC(page->mapping->host)) {
  262. error = nfs_fscache_read_page(page->mapping->host, page);
  263. if (error == 0)
  264. goto out_unlock;
  265. }
  266. new = nfs_create_request(desc->ctx, page, 0, aligned_len);
  267. if (IS_ERR(new))
  268. goto out_error;
  269. if (len < PAGE_SIZE)
  270. zero_user_segment(page, len, PAGE_SIZE);
  271. if (!nfs_pageio_add_request(&desc->pgio, new)) {
  272. nfs_list_remove_request(new);
  273. error = desc->pgio.pg_error;
  274. nfs_readpage_release(new, error);
  275. goto out;
  276. }
  277. return 0;
  278. out_error:
  279. error = PTR_ERR(new);
  280. out_unlock:
  281. unlock_page(page);
  282. out:
  283. return error;
  284. }
  285. /*
  286. * Read a page over NFS.
  287. * We read the page synchronously in the following case:
  288. * - The error flag is set for this page. This happens only when a
  289. * previous async read operation failed.
  290. */
  291. int nfs_read_folio(struct file *file, struct folio *folio)
  292. {
  293. struct page *page = &folio->page;
  294. struct nfs_readdesc desc;
  295. struct inode *inode = page_file_mapping(page)->host;
  296. int ret;
  297. trace_nfs_aop_readpage(inode, page);
  298. nfs_inc_stats(inode, NFSIOS_VFSREADPAGE);
  299. task_io_account_read(folio_size(folio));
  300. /*
  301. * Try to flush any pending writes to the file..
  302. *
  303. * NOTE! Because we own the page lock, there cannot
  304. * be any new pending writes generated at this point
  305. * for this page (other pages can be written to).
  306. */
  307. ret = nfs_wb_page(inode, page);
  308. if (ret)
  309. goto out_unlock;
  310. if (PageUptodate(page))
  311. goto out_unlock;
  312. ret = -ESTALE;
  313. if (NFS_STALE(inode))
  314. goto out_unlock;
  315. if (file == NULL) {
  316. ret = -EBADF;
  317. desc.ctx = nfs_find_open_context(inode, NULL, FMODE_READ);
  318. if (desc.ctx == NULL)
  319. goto out_unlock;
  320. } else
  321. desc.ctx = get_nfs_open_context(nfs_file_open_context(file));
  322. xchg(&desc.ctx->error, 0);
  323. nfs_pageio_init_read(&desc.pgio, inode, false,
  324. &nfs_async_read_completion_ops);
  325. ret = readpage_async_filler(&desc, page);
  326. if (ret)
  327. goto out;
  328. nfs_pageio_complete_read(&desc.pgio);
  329. ret = desc.pgio.pg_error < 0 ? desc.pgio.pg_error : 0;
  330. if (!ret) {
  331. ret = wait_on_page_locked_killable(page);
  332. if (!PageUptodate(page) && !ret)
  333. ret = xchg(&desc.ctx->error, 0);
  334. }
  335. out:
  336. put_nfs_open_context(desc.ctx);
  337. trace_nfs_aop_readpage_done(inode, page, ret);
  338. return ret;
  339. out_unlock:
  340. unlock_page(page);
  341. trace_nfs_aop_readpage_done(inode, page, ret);
  342. return ret;
  343. }
  344. void nfs_readahead(struct readahead_control *ractl)
  345. {
  346. unsigned int nr_pages = readahead_count(ractl);
  347. struct file *file = ractl->file;
  348. struct nfs_readdesc desc;
  349. struct inode *inode = ractl->mapping->host;
  350. struct page *page;
  351. int ret;
  352. trace_nfs_aop_readahead(inode, readahead_pos(ractl), nr_pages);
  353. nfs_inc_stats(inode, NFSIOS_VFSREADPAGES);
  354. task_io_account_read(readahead_length(ractl));
  355. ret = -ESTALE;
  356. if (NFS_STALE(inode))
  357. goto out;
  358. if (file == NULL) {
  359. ret = -EBADF;
  360. desc.ctx = nfs_find_open_context(inode, NULL, FMODE_READ);
  361. if (desc.ctx == NULL)
  362. goto out;
  363. } else
  364. desc.ctx = get_nfs_open_context(nfs_file_open_context(file));
  365. nfs_pageio_init_read(&desc.pgio, inode, false,
  366. &nfs_async_read_completion_ops);
  367. while ((page = readahead_page(ractl)) != NULL) {
  368. ret = readpage_async_filler(&desc, page);
  369. put_page(page);
  370. if (ret)
  371. break;
  372. }
  373. nfs_pageio_complete_read(&desc.pgio);
  374. put_nfs_open_context(desc.ctx);
  375. out:
  376. trace_nfs_aop_readahead_done(inode, nr_pages, ret);
  377. }
  378. int __init nfs_init_readpagecache(void)
  379. {
  380. nfs_rdata_cachep = kmem_cache_create("nfs_read_data",
  381. sizeof(struct nfs_pgio_header),
  382. 0, SLAB_HWCACHE_ALIGN,
  383. NULL);
  384. if (nfs_rdata_cachep == NULL)
  385. return -ENOMEM;
  386. return 0;
  387. }
  388. void nfs_destroy_readpagecache(void)
  389. {
  390. kmem_cache_destroy(nfs_rdata_cachep);
  391. }
  392. static const struct nfs_rw_ops nfs_rw_read_ops = {
  393. .rw_alloc_header = nfs_readhdr_alloc,
  394. .rw_free_header = nfs_readhdr_free,
  395. .rw_done = nfs_readpage_done,
  396. .rw_result = nfs_readpage_result,
  397. .rw_initiate = nfs_initiate_read,
  398. };