lib/iov_iter.c

   1 #include <linux/export.h>
   2 #include <linux/bvec.h>
   3 #include <linux/uio.h>
   4 #include <linux/pagemap.h>
   5 #include <linux/slab.h>
   6 #include <linux/vmalloc.h>
   7 #include <linux/splice.h>
   8 #include <net/checksum.h>
   9
  10 #define PIPE_PARANOIA /* for now */
  11
  12 #define iterate_iovec(i, n, __v, __p, skip, STEP) {     \
  13         size_t left;                                    \
  14         size_t wanted = n;                              \
  15         __p = i->iov;                                   \
  16         __v.iov_len = min(n, __p->iov_len - skip);      \
  17         if (likely(__v.iov_len)) {                      \
  18                 __v.iov_base = __p->iov_base + skip;    \
  19                 left = (STEP);                          \
  20                 __v.iov_len -= left;                    \
  21                 skip += __v.iov_len;                    \
  22                 n -= __v.iov_len;                       \
  23         } else {                                        \
  24                 left = 0;                               \
  25         }                                               \
  26         while (unlikely(!left && n)) {                  \
  27                 __p++;                                  \
  28                 __v.iov_len = min(n, __p->iov_len);     \
  29                 if (unlikely(!__v.iov_len))             \
  30                         continue;                       \
  31                 __v.iov_base = __p->iov_base;           \
  32                 left = (STEP);                          \
  33                 __v.iov_len -= left;                    \
  34                 skip = __v.iov_len;                     \
  35                 n -= __v.iov_len;                       \
  36         }                                               \
  37         n = wanted - n;                                 \
  38 }
  39
  40 #define iterate_kvec(i, n, __v, __p, skip, STEP) {      \
  41         size_t wanted = n;                              \
  42         __p = i->kvec;                                  \
  43         __v.iov_len = min(n, __p->iov_len - skip);      \
  44         if (likely(__v.iov_len)) {                      \
  45                 __v.iov_base = __p->iov_base + skip;    \
  46                 (void)(STEP);                           \
  47                 skip += __v.iov_len;                    \
  48                 n -= __v.iov_len;                       \
  49         }                                               \
  50         while (unlikely(n)) {                           \
  51                 __p++;                                  \
  52                 __v.iov_len = min(n, __p->iov_len);     \
  53                 if (unlikely(!__v.iov_len))             \
  54                         continue;                       \
  55                 __v.iov_base = __p->iov_base;           \
  56                 (void)(STEP);                           \
  57                 skip = __v.iov_len;                     \
  58                 n -= __v.iov_len;                       \
  59         }                                               \
  60         n = wanted;                                     \
  61 }
  62
  63 #define iterate_bvec(i, n, __v, __bi, skip, STEP) {     \
  64         struct bvec_iter __start;                       \
  65         __start.bi_size = n;                            \
  66         __start.bi_bvec_done = skip;                    \
  67         __start.bi_idx = 0;                             \
  68         for_each_bvec(__v, i->bvec, __bi, __start) {    \
  69                 if (!__v.bv_len)                        \
  70                         continue;                       \
  71                 (void)(STEP);                           \
  72         }                                               \
  73 }
  74
  75 #define iterate_all_kinds(i, n, v, I, B, K) {                   \
  76         if (likely(n)) {                                        \
  77                 size_t skip = i->iov_offset;                    \
  78                 if (unlikely(i->type & ITER_BVEC)) {            \
  79                         struct bio_vec v;                       \
  80                         struct bvec_iter __bi;                  \
  81                         iterate_bvec(i, n, v, __bi, skip, (B))  \
  82                 } else if (unlikely(i->type & ITER_KVEC)) {     \
  83                         const struct kvec *kvec;                \
  84                         struct kvec v;                          \
  85                         iterate_kvec(i, n, v, kvec, skip, (K))  \
  86                 } else {                                        \
  87                         const struct iovec *iov;                \
  88                         struct iovec v;                         \
  89                         iterate_iovec(i, n, v, iov, skip, (I))  \
  90                 }                                               \
  91         }                                                       \
  92 }
  93
  94 #define iterate_and_advance(i, n, v, I, B, K) {                 \
  95         if (unlikely(i->count < n))                             \
  96                 n = i->count;                                   \
  97         if (i->count) {                                         \
  98                 size_t skip = i->iov_offset;                    \
  99                 if (unlikely(i->type & ITER_BVEC)) {            \
 100                         const struct bio_vec *bvec = i->bvec;   \
 101                         struct bio_vec v;                       \
 102                         struct bvec_iter __bi;                  \
 103                         iterate_bvec(i, n, v, __bi, skip, (B))  \
 104                         i->bvec = __bvec_iter_bvec(i->bvec, __bi);      \
 105                         i->nr_segs -= i->bvec - bvec;           \
 106                         skip = __bi.bi_bvec_done;               \
 107                 } else if (unlikely(i->type & ITER_KVEC)) {     \
 108                         const struct kvec *kvec;                \
 109                         struct kvec v;                          \
 110                         iterate_kvec(i, n, v, kvec, skip, (K))  \
 111                         if (skip == kvec->iov_len) {            \
 112                                 kvec++;                         \
 113                                 skip = 0;                       \
 114                         }                                       \
 115                         i->nr_segs -= kvec - i->kvec;           \
 116                         i->kvec = kvec;                         \
 117                 } else {                                        \
 118                         const struct iovec *iov;                \
 119                         struct iovec v;                         \
 120                         iterate_iovec(i, n, v, iov, skip, (I))  \
 121                         if (skip == iov->iov_len) {             \
 122                                 iov++;                          \
 123                                 skip = 0;                       \
 124                         }                                       \
 125                         i->nr_segs -= iov - i->iov;             \
 126                         i->iov = iov;                           \
 127                 }                                               \
 128                 i->count -= n;                                  \
 129                 i->iov_offset = skip;                           \
 130         }                                                       \
 131 }
 132
 133 static int copyout(void __user *to, const void *from, size_t n)
 134 {
 135         if (access_ok(VERIFY_WRITE, to, n)) {
 136                 kasan_check_read(from, n);
 137                 n = raw_copy_to_user(to, from, n);
 138         }
 139         return n;
 140 }
 141
 142 static int copyin(void *to, const void __user *from, size_t n)
 143 {
 144         if (access_ok(VERIFY_READ, from, n)) {
 145                 kasan_check_write(to, n);
 146                 n = raw_copy_from_user(to, from, n);
 147         }
 148         return n;
 149 }
 150
 151 static size_t copy_page_to_iter_iovec(struct page *page, size_t offset, size_t bytes,
 152                          struct iov_iter *i)
 153 {
 154         size_t skip, copy, left, wanted;
 155         const struct iovec *iov;
 156         char __user *buf;
 157         void *kaddr, *from;
 158
 159         if (unlikely(bytes > i->count))
 160                 bytes = i->count;
 161
 162         if (unlikely(!bytes))
 163                 return 0;
 164
 165         might_fault();
 166         wanted = bytes;
 167         iov = i->iov;
 168         skip = i->iov_offset;
 169         buf = iov->iov_base + skip;
 170         copy = min(bytes, iov->iov_len - skip);
 171
 172         if (IS_ENABLED(CONFIG_HIGHMEM) && !fault_in_pages_writeable(buf, copy)) {
 173                 kaddr = kmap_atomic(page);
 174                 from = kaddr + offset;
 175
 176                 /* first chunk, usually the only one */
 177                 left = copyout(buf, from, copy);
 178                 copy -= left;
 179                 skip += copy;
 180                 from += copy;
 181                 bytes -= copy;
 182
 183                 while (unlikely(!left && bytes)) {
 184                         iov++;
 185                         buf = iov->iov_base;
 186                         copy = min(bytes, iov->iov_len);
 187                         left = copyout(buf, from, copy);
 188                         copy -= left;
 189                         skip = copy;
 190                         from += copy;
 191                         bytes -= copy;
 192                 }
 193                 if (likely(!bytes)) {
 194                         kunmap_atomic(kaddr);
 195                         goto done;
 196                 }
 197                 offset = from - kaddr;
 198                 buf += copy;
 199                 kunmap_atomic(kaddr);
 200                 copy = min(bytes, iov->iov_len - skip);
 201         }
 202         /* Too bad - revert to non-atomic kmap */
 203
 204         kaddr = kmap(page);
 205         from = kaddr + offset;
 206         left = copyout(buf, from, copy);
 207         copy -= left;
 208         skip += copy;
 209         from += copy;
 210         bytes -= copy;
 211         while (unlikely(!left && bytes)) {
 212                 iov++;
 213                 buf = iov->iov_base;
 214                 copy = min(bytes, iov->iov_len);
 215                 left = copyout(buf, from, copy);
 216                 copy -= left;
 217                 skip = copy;
 218                 from += copy;
 219                 bytes -= copy;
 220         }
 221         kunmap(page);
 222
 223 done:
 224         if (skip == iov->iov_len) {
 225                 iov++;
 226                 skip = 0;
 227         }
 228         i->count -= wanted - bytes;
 229         i->nr_segs -= iov - i->iov;
 230         i->iov = iov;
 231         i->iov_offset = skip;
 232         return wanted - bytes;
 233 }
 234
 235 static size_t copy_page_from_iter_iovec(struct page *page, size_t offset, size_t bytes,
 236                          struct iov_iter *i)
 237 {
 238         size_t skip, copy, left, wanted;
 239         const struct iovec *iov;
 240         char __user *buf;
 241         void *kaddr, *to;
 242
 243         if (unlikely(bytes > i->count))
 244                 bytes = i->count;
 245
 246         if (unlikely(!bytes))
 247                 return 0;
 248
 249         might_fault();
 250         wanted = bytes;
 251         iov = i->iov;
 252         skip = i->iov_offset;
 253         buf = iov->iov_base + skip;
 254         copy = min(bytes, iov->iov_len - skip);
 255
 256         if (IS_ENABLED(CONFIG_HIGHMEM) && !fault_in_pages_readable(buf, copy)) {
 257                 kaddr = kmap_atomic(page);
 258                 to = kaddr + offset;
 259
 260                 /* first chunk, usually the only one */
 261                 left = copyin(to, buf, copy);
 262                 copy -= left;
 263                 skip += copy;
 264                 to += copy;
 265                 bytes -= copy;
 266
 267                 while (unlikely(!left && bytes)) {
 268                         iov++;
 269                         buf = iov->iov_base;
 270                         copy = min(bytes, iov->iov_len);
 271                         left = copyin(to, buf, copy);
 272                         copy -= left;
 273                         skip = copy;
 274                         to += copy;
 275                         bytes -= copy;
 276                 }
 277                 if (likely(!bytes)) {
 278                         kunmap_atomic(kaddr);
 279                         goto done;
 280                 }
 281                 offset = to - kaddr;
 282                 buf += copy;
 283                 kunmap_atomic(kaddr);
 284                 copy = min(bytes, iov->iov_len - skip);
 285         }
 286         /* Too bad - revert to non-atomic kmap */
 287
 288         kaddr = kmap(page);
 289         to = kaddr + offset;
 290         left = copyin(to, buf, copy);
 291         copy -= left;
 292         skip += copy;
 293         to += copy;
 294         bytes -= copy;
 295         while (unlikely(!left && bytes)) {
 296                 iov++;
 297                 buf = iov->iov_base;
 298                 copy = min(bytes, iov->iov_len);
 299                 left = copyin(to, buf, copy);
 300                 copy -= left;
 301                 skip = copy;
 302                 to += copy;
 303                 bytes -= copy;
 304         }
 305         kunmap(page);
 306
 307 done:
 308         if (skip == iov->iov_len) {
 309                 iov++;
 310                 skip = 0;
 311         }
 312         i->count -= wanted - bytes;
 313         i->nr_segs -= iov - i->iov;
 314         i->iov = iov;
 315         i->iov_offset = skip;
 316         return wanted - bytes;
 317 }
 318
 319 #ifdef PIPE_PARANOIA
 320 static bool sanity(const struct iov_iter *i)
 321 {
 322         struct pipe_inode_info *pipe = i->pipe;
 323         int idx = i->idx;
 324         int next = pipe->curbuf + pipe->nrbufs;
 325         if (i->iov_offset) {
 326                 struct pipe_buffer *p;
 327                 if (unlikely(!pipe->nrbufs))
 328                         goto Bad;       // pipe must be non-empty
 329                 if (unlikely(idx != ((next - 1) & (pipe->buffers - 1))))
 330                         goto Bad;       // must be at the last buffer...
 331
 332                 p = &pipe->bufs[idx];
 333                 if (unlikely(p->offset + p->len != i->iov_offset))
 334                         goto Bad;       // ... at the end of segment
 335         } else {
 336                 if (idx != (next & (pipe->buffers - 1)))
 337                         goto Bad;       // must be right after the last buffer
 338         }
 339         return true;
 340 Bad:
 341         printk(KERN_ERR "idx = %d, offset = %zd\n", i->idx, i->iov_offset);
 342         printk(KERN_ERR "curbuf = %d, nrbufs = %d, buffers = %d\n",
 343                         pipe->curbuf, pipe->nrbufs, pipe->buffers);
 344         for (idx = 0; idx < pipe->buffers; idx++)
 345                 printk(KERN_ERR "[%p %p %d %d]\n",
 346                         pipe->bufs[idx].ops,
 347                         pipe->bufs[idx].page,
 348                         pipe->bufs[idx].offset,
 349                         pipe->bufs[idx].len);
 350         WARN_ON(1);
 351         return false;
 352 }
 353 #else
 354 #define sanity(i) true
 355 #endif
 356
 357 static inline int next_idx(int idx, struct pipe_inode_info *pipe)
 358 {
 359         return (idx + 1) & (pipe->buffers - 1);
 360 }
 361
 362 static size_t copy_page_to_iter_pipe(struct page *page, size_t offset, size_t bytes,
 363                          struct iov_iter *i)
 364 {
 365         struct pipe_inode_info *pipe = i->pipe;
 366         struct pipe_buffer *buf;
 367         size_t off;
 368         int idx;
 369
 370         if (unlikely(bytes > i->count))
 371                 bytes = i->count;
 372
 373         if (unlikely(!bytes))
 374                 return 0;
 375
 376         if (!sanity(i))
 377                 return 0;
 378
 379         off = i->iov_offset;
 380         idx = i->idx;
 381         buf = &pipe->bufs[idx];
 382         if (off) {
 383                 if (offset == off && buf->page == page) {
 384                         /* merge with the last one */
 385                         buf->len += bytes;
 386                         i->iov_offset += bytes;
 387                         goto out;
 388                 }
 389                 idx = next_idx(idx, pipe);
 390                 buf = &pipe->bufs[idx];
 391         }
 392         if (idx == pipe->curbuf && pipe->nrbufs)
 393                 return 0;
 394         pipe->nrbufs++;
 395         buf->ops = &page_cache_pipe_buf_ops;
 396         buf->flags = 0;
 397         get_page(buf->page = page);
 398         buf->offset = offset;
 399         buf->len = bytes;
 400         i->iov_offset = offset + bytes;
 401         i->idx = idx;
 402 out:
 403         i->count -= bytes;
 404         return bytes;
 405 }
 406
 407 /*
 408  * Fault in one or more iovecs of the given iov_iter, to a maximum length of
 409  * bytes.  For each iovec, fault in each page that constitutes the iovec.
 410  *
 411  * Return 0 on success, or non-zero if the memory could not be accessed (i.e.
 412  * because it is an invalid address).
 413  */
 414 int iov_iter_fault_in_readable(struct iov_iter *i, size_t bytes)
 415 {
 416         size_t skip = i->iov_offset;
 417         const struct iovec *iov;
 418         int err;
 419         struct iovec v;
 420
 421         if (iter_is_iovec(i)) {
 422                 iterate_iovec(i, bytes, v, iov, skip, ({
 423                         err = fault_in_pages_readable(v.iov_base, v.iov_len);
 424                         if (unlikely(err))
 425                         return err;
 426                 0;}))
 427         }
 428         return 0;
 429 }
 430 EXPORT_SYMBOL(iov_iter_fault_in_readable);
 431
 432 void iov_iter_init(struct iov_iter *i, int direction,
 433                         const struct iovec *iov, unsigned long nr_segs,
 434                         size_t count)
 435 {
 436         /* It will get better.  Eventually... */
 437         if (uaccess_kernel()) {
 438                 direction |= ITER_KVEC;
 439                 i->type = direction;
 440                 i->kvec = (struct kvec *)iov;
 441         } else {
 442                 i->type = direction;
 443                 i->iov = iov;
 444         }
 445         i->nr_segs = nr_segs;
 446         i->iov_offset = 0;
 447         i->count = count;
 448 }
 449 EXPORT_SYMBOL(iov_iter_init);
 450
 451 static void memcpy_from_page(char *to, struct page *page, size_t offset, size_t len)
 452 {
 453         char *from = kmap_atomic(page);
 454         memcpy(to, from + offset, len);
 455         kunmap_atomic(from);
 456 }
 457
 458 static void memcpy_to_page(struct page *page, size_t offset, const char *from, size_t len)
 459 {
 460         char *to = kmap_atomic(page);
 461         memcpy(to + offset, from, len);
 462         kunmap_atomic(to);
 463 }
 464
 465 static void memzero_page(struct page *page, size_t offset, size_t len)
 466 {
 467         char *addr = kmap_atomic(page);
 468         memset(addr + offset, 0, len);
 469         kunmap_atomic(addr);
 470 }
 471
 472 static inline bool allocated(struct pipe_buffer *buf)
 473 {
 474         return buf->ops == &default_pipe_buf_ops;
 475 }
 476
 477 static inline void data_start(const struct iov_iter *i, int *idxp, size_t *offp)
 478 {
 479         size_t off = i->iov_offset;
 480         int idx = i->idx;
 481         if (off && (!allocated(&i->pipe->bufs[idx]) || off == PAGE_SIZE)) {
 482                 idx = next_idx(idx, i->pipe);
 483                 off = 0;
 484         }
 485         *idxp = idx;
 486         *offp = off;
 487 }
 488
 489 static size_t push_pipe(struct iov_iter *i, size_t size,
 490                         int *idxp, size_t *offp)
 491 {
 492         struct pipe_inode_info *pipe = i->pipe;
 493         size_t off;
 494         int idx;
 495         ssize_t left;
 496
 497         if (unlikely(size > i->count))
 498                 size = i->count;
 499         if (unlikely(!size))
 500                 return 0;
 501
 502         left = size;
 503         data_start(i, &idx, &off);
 504         *idxp = idx;
 505         *offp = off;
 506         if (off) {
 507                 left -= PAGE_SIZE - off;
 508                 if (left <= 0) {
 509                         pipe->bufs[idx].len += size;
 510                         return size;
 511                 }
 512                 pipe->bufs[idx].len = PAGE_SIZE;
 513                 idx = next_idx(idx, pipe);
 514         }
 515         while (idx != pipe->curbuf || !pipe->nrbufs) {
 516                 struct page *page = alloc_page(GFP_USER);
 517                 if (!page)
 518                         break;
 519                 pipe->nrbufs++;
 520                 pipe->bufs[idx].ops = &default_pipe_buf_ops;
 521                 pipe->bufs[idx].flags = 0;
 522                 pipe->bufs[idx].page = page;
 523                 pipe->bufs[idx].offset = 0;
 524                 if (left <= PAGE_SIZE) {
 525                         pipe->bufs[idx].len = left;
 526                         return size;
 527                 }
 528                 pipe->bufs[idx].len = PAGE_SIZE;
 529                 left -= PAGE_SIZE;
 530                 idx = next_idx(idx, pipe);
 531         }
 532         return size - left;
 533 }
 534
 535 static size_t copy_pipe_to_iter(const void *addr, size_t bytes,
 536                                 struct iov_iter *i)
 537 {
 538         struct pipe_inode_info *pipe = i->pipe;
 539         size_t n, off;
 540         int idx;
 541
 542         if (!sanity(i))
 543                 return 0;
 544
 545         bytes = n = push_pipe(i, bytes, &idx, &off);
 546         if (unlikely(!n))
 547                 return 0;
 548         for ( ; n; idx = next_idx(idx, pipe), off = 0) {
 549                 size_t chunk = min_t(size_t, n, PAGE_SIZE - off);
 550                 memcpy_to_page(pipe->bufs[idx].page, off, addr, chunk);
 551                 i->idx = idx;
 552                 i->iov_offset = off + chunk;
 553                 n -= chunk;
 554                 addr += chunk;
 555         }
 556         i->count -= bytes;
 557         return bytes;
 558 }
 559
 560 size_t _copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i)
 561 {
 562         const char *from = addr;
 563         if (unlikely(i->type & ITER_PIPE))
 564                 return copy_pipe_to_iter(addr, bytes, i);
 565         if (iter_is_iovec(i))
 566                 might_fault();
 567         iterate_and_advance(i, bytes, v,
 568                 copyout(v.iov_base, (from += v.iov_len) - v.iov_len, v.iov_len),
 569                 memcpy_to_page(v.bv_page, v.bv_offset,
 570                                (from += v.bv_len) - v.bv_len, v.bv_len),
 571                 memcpy(v.iov_base, (from += v.iov_len) - v.iov_len, v.iov_len)
 572         )
 573
 574         return bytes;
 575 }
 576 EXPORT_SYMBOL(_copy_to_iter);
 577
 578 size_t _copy_from_iter(void *addr, size_t bytes, struct iov_iter *i)
 579 {
 580         char *to = addr;
 581         if (unlikely(i->type & ITER_PIPE)) {
 582                 WARN_ON(1);
 583                 return 0;
 584         }
 585         if (iter_is_iovec(i))
 586                 might_fault();
 587         iterate_and_advance(i, bytes, v,
 588                 copyin((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len),
 589                 memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page,
 590                                  v.bv_offset, v.bv_len),
 591                 memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len)
 592         )
 593
 594         return bytes;
 595 }
 596 EXPORT_SYMBOL(_copy_from_iter);
 597
 598 bool _copy_from_iter_full(void *addr, size_t bytes, struct iov_iter *i)
 599 {
 600         char *to = addr;
 601         if (unlikely(i->type & ITER_PIPE)) {
 602                 WARN_ON(1);
 603                 return false;
 604         }
 605         if (unlikely(i->count < bytes))
 606                 return false;
 607
 608         if (iter_is_iovec(i))
 609                 might_fault();
 610         iterate_all_kinds(i, bytes, v, ({
 611                 if (copyin((to += v.iov_len) - v.iov_len,
 612                                       v.iov_base, v.iov_len))
 613                         return false;
 614                 0;}),
 615                 memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page,
 616                                  v.bv_offset, v.bv_len),
 617                 memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len)
 618         )
 619
 620         iov_iter_advance(i, bytes);
 621         return true;
 622 }
 623 EXPORT_SYMBOL(_copy_from_iter_full);
 624
 625 size_t _copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i)
 626 {
 627         char *to = addr;
 628         if (unlikely(i->type & ITER_PIPE)) {
 629                 WARN_ON(1);
 630                 return 0;
 631         }
 632         iterate_and_advance(i, bytes, v,
 633                 __copy_from_user_inatomic_nocache((to += v.iov_len) - v.iov_len,
 634                                          v.iov_base, v.iov_len),
 635                 memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page,
 636                                  v.bv_offset, v.bv_len),
 637                 memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len)
 638         )
 639
 640         return bytes;
 641 }
 642 EXPORT_SYMBOL(_copy_from_iter_nocache);
 643
 644 #ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE
 645 size_t _copy_from_iter_flushcache(void *addr, size_t bytes, struct iov_iter *i)
 646 {
 647         char *to = addr;
 648         if (unlikely(i->type & ITER_PIPE)) {
 649                 WARN_ON(1);
 650                 return 0;
 651         }
 652         iterate_and_advance(i, bytes, v,
 653                 __copy_from_user_flushcache((to += v.iov_len) - v.iov_len,
 654                                          v.iov_base, v.iov_len),
 655                 memcpy_page_flushcache((to += v.bv_len) - v.bv_len, v.bv_page,
 656                                  v.bv_offset, v.bv_len),
 657                 memcpy_flushcache((to += v.iov_len) - v.iov_len, v.iov_base,
 658                         v.iov_len)
 659         )
 660
 661         return bytes;
 662 }
 663 EXPORT_SYMBOL_GPL(_copy_from_iter_flushcache);
 664 #endif
 665
 666 bool _copy_from_iter_full_nocache(void *addr, size_t bytes, struct iov_iter *i)
 667 {
 668         char *to = addr;
 669         if (unlikely(i->type & ITER_PIPE)) {
 670                 WARN_ON(1);
 671                 return false;
 672         }
 673         if (unlikely(i->count < bytes))
 674                 return false;
 675         iterate_all_kinds(i, bytes, v, ({
 676                 if (__copy_from_user_inatomic_nocache((to += v.iov_len) - v.iov_len,
 677                                              v.iov_base, v.iov_len))
 678                         return false;
 679                 0;}),
 680                 memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page,
 681                                  v.bv_offset, v.bv_len),
 682                 memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len)
 683         )
 684
 685         iov_iter_advance(i, bytes);
 686         return true;
 687 }
 688 EXPORT_SYMBOL(_copy_from_iter_full_nocache);
 689
 690 static inline bool page_copy_sane(struct page *page, size_t offset, size_t n)
 691 {
 692         struct page *head;
 693         size_t v = n + offset;
 694
 695         /*
 696          * The general case needs to access the page order in order
 697          * to compute the page size.
 698          * However, we mostly deal with order-0 pages and thus can
 699          * avoid a possible cache line miss for requests that fit all
 700          * page orders.
 701          */
 702         if (n <= v && v <= PAGE_SIZE)
 703                 return true;
 704
 705         head = compound_head(page);
 706         v += (page - head) << PAGE_SHIFT;
 707
 708         if (likely(n <= v && v <= (PAGE_SIZE << compound_order(head))))
 709                 return true;
 710         WARN_ON(1);
 711         return false;
 712 }
 713
 714 size_t copy_page_to_iter(struct page *page, size_t offset, size_t bytes,
 715                          struct iov_iter *i)
 716 {
 717         if (unlikely(!page_copy_sane(page, offset, bytes)))
 718                 return 0;
 719         if (i->type & (ITER_BVEC|ITER_KVEC)) {
 720                 void *kaddr = kmap_atomic(page);
 721                 size_t wanted = copy_to_iter(kaddr + offset, bytes, i);
 722                 kunmap_atomic(kaddr);
 723                 return wanted;
 724         } else if (likely(!(i->type & ITER_PIPE)))
 725                 return copy_page_to_iter_iovec(page, offset, bytes, i);
 726         else
 727                 return copy_page_to_iter_pipe(page, offset, bytes, i);
 728 }
 729 EXPORT_SYMBOL(copy_page_to_iter);
 730
 731 size_t copy_page_from_iter(struct page *page, size_t offset, size_t bytes,
 732                          struct iov_iter *i)
 733 {
 734         if (unlikely(!page_copy_sane(page, offset, bytes)))
 735                 return 0;
 736         if (unlikely(i->type & ITER_PIPE)) {
 737                 WARN_ON(1);
 738                 return 0;
 739         }
 740         if (i->type & (ITER_BVEC|ITER_KVEC)) {
 741                 void *kaddr = kmap_atomic(page);
 742                 size_t wanted = _copy_from_iter(kaddr + offset, bytes, i);
 743                 kunmap_atomic(kaddr);
 744                 return wanted;
 745         } else
 746                 return copy_page_from_iter_iovec(page, offset, bytes, i);
 747 }
 748 EXPORT_SYMBOL(copy_page_from_iter);
 749
 750 static size_t pipe_zero(size_t bytes, struct iov_iter *i)
 751 {
 752         struct pipe_inode_info *pipe = i->pipe;
 753         size_t n, off;
 754         int idx;
 755
 756         if (!sanity(i))
 757                 return 0;
 758
 759         bytes = n = push_pipe(i, bytes, &idx, &off);
 760         if (unlikely(!n))
 761                 return 0;
 762
 763         for ( ; n; idx = next_idx(idx, pipe), off = 0) {
 764                 size_t chunk = min_t(size_t, n, PAGE_SIZE - off);
 765                 memzero_page(pipe->bufs[idx].page, off, chunk);
 766                 i->idx = idx;
 767                 i->iov_offset = off + chunk;
 768                 n -= chunk;
 769         }
 770         i->count -= bytes;
 771         return bytes;
 772 }
 773
 774 size_t iov_iter_zero(size_t bytes, struct iov_iter *i)
 775 {
 776         if (unlikely(i->type & ITER_PIPE))
 777                 return pipe_zero(bytes, i);
 778         iterate_and_advance(i, bytes, v,
 779                 clear_user(v.iov_base, v.iov_len),
 780                 memzero_page(v.bv_page, v.bv_offset, v.bv_len),
 781                 memset(v.iov_base, 0, v.iov_len)
 782         )
 783
 784         return bytes;
 785 }
 786 EXPORT_SYMBOL(iov_iter_zero);
 787
 788 size_t iov_iter_copy_from_user_atomic(struct page *page,
 789                 struct iov_iter *i, unsigned long offset, size_t bytes)
 790 {
 791         char *kaddr = kmap_atomic(page), *p = kaddr + offset;
 792         if (unlikely(!page_copy_sane(page, offset, bytes))) {
 793                 kunmap_atomic(kaddr);
 794                 return 0;
 795         }
 796         if (unlikely(i->type & ITER_PIPE)) {
 797                 kunmap_atomic(kaddr);
 798                 WARN_ON(1);
 799                 return 0;
 800         }
 801         iterate_all_kinds(i, bytes, v,
 802                 copyin((p += v.iov_len) - v.iov_len, v.iov_base, v.iov_len),
 803                 memcpy_from_page((p += v.bv_len) - v.bv_len, v.bv_page,
 804                                  v.bv_offset, v.bv_len),
 805                 memcpy((p += v.iov_len) - v.iov_len, v.iov_base, v.iov_len)
 806         )
 807         kunmap_atomic(kaddr);
 808         return bytes;
 809 }
 810 EXPORT_SYMBOL(iov_iter_copy_from_user_atomic);
 811
 812 static inline void pipe_truncate(struct iov_iter *i)
 813 {
 814         struct pipe_inode_info *pipe = i->pipe;
 815         if (pipe->nrbufs) {
 816                 size_t off = i->iov_offset;
 817                 int idx = i->idx;
 818                 int nrbufs = (idx - pipe->curbuf) & (pipe->buffers - 1);
 819                 if (off) {
 820                         pipe->bufs[idx].len = off - pipe->bufs[idx].offset;
 821                         idx = next_idx(idx, pipe);
 822                         nrbufs++;
 823                 }
 824                 while (pipe->nrbufs > nrbufs) {
 825                         pipe_buf_release(pipe, &pipe->bufs[idx]);
 826                         idx = next_idx(idx, pipe);
 827                         pipe->nrbufs--;
 828                 }
 829         }
 830 }
 831
 832 static void pipe_advance(struct iov_iter *i, size_t size)
 833 {
 834         struct pipe_inode_info *pipe = i->pipe;
 835         if (unlikely(i->count < size))
 836                 size = i->count;
 837         if (size) {
 838                 struct pipe_buffer *buf;
 839                 size_t off = i->iov_offset, left = size;
 840                 int idx = i->idx;
 841                 if (off) /* make it relative to the beginning of buffer */
 842                         left += off - pipe->bufs[idx].offset;
 843                 while (1) {
 844                         buf = &pipe->bufs[idx];
 845                         if (left <= buf->len)
 846                                 break;
 847                         left -= buf->len;
 848                         idx = next_idx(idx, pipe);
 849                 }
 850                 i->idx = idx;
 851                 i->iov_offset = buf->offset + left;
 852         }
 853         i->count -= size;
 854         /* ... and discard everything past that point */
 855         pipe_truncate(i);
 856 }
 857
 858 void iov_iter_advance(struct iov_iter *i, size_t size)
 859 {
 860         if (unlikely(i->type & ITER_PIPE)) {
 861                 pipe_advance(i, size);
 862                 return;
 863         }
 864         iterate_and_advance(i, size, v, 0, 0, 0)
 865 }
 866 EXPORT_SYMBOL(iov_iter_advance);
 867
 868 void iov_iter_revert(struct iov_iter *i, size_t unroll)
 869 {
 870         if (!unroll)
 871                 return;
 872         if (WARN_ON(unroll > MAX_RW_COUNT))
 873                 return;
 874         i->count += unroll;
 875         if (unlikely(i->type & ITER_PIPE)) {
 876                 struct pipe_inode_info *pipe = i->pipe;
 877                 int idx = i->idx;
 878                 size_t off = i->iov_offset;
 879                 while (1) {
 880                         size_t n = off - pipe->bufs[idx].offset;
 881                         if (unroll < n) {
 882                                 off -= unroll;
 883                                 break;
 884                         }
 885                         unroll -= n;
 886                         if (!unroll && idx == i->start_idx) {
 887                                 off = 0;
 888                                 break;
 889                         }
 890                         if (!idx--)
 891                                 idx = pipe->buffers - 1;
 892                         off = pipe->bufs[idx].offset + pipe->bufs[idx].len;
 893                 }
 894                 i->iov_offset = off;
 895                 i->idx = idx;
 896                 pipe_truncate(i);
 897                 return;
 898         }
 899         if (unroll <= i->iov_offset) {
 900                 i->iov_offset -= unroll;
 901                 return;
 902         }
 903         unroll -= i->iov_offset;
 904         if (i->type & ITER_BVEC) {
 905                 const struct bio_vec *bvec = i->bvec;
 906                 while (1) {
 907                         size_t n = (--bvec)->bv_len;
 908                         i->nr_segs++;
 909                         if (unroll <= n) {
 910                                 i->bvec = bvec;
 911                                 i->iov_offset = n - unroll;
 912                                 return;
 913                         }
 914                         unroll -= n;
 915                 }
 916         } else { /* same logics for iovec and kvec */
 917                 const struct iovec *iov = i->iov;
 918                 while (1) {
 919                         size_t n = (--iov)->iov_len;
 920                         i->nr_segs++;
 921                         if (unroll <= n) {
 922                                 i->iov = iov;
 923                                 i->iov_offset = n - unroll;
 924                                 return;
 925                         }
 926                         unroll -= n;
 927                 }
 928         }
 929 }
 930 EXPORT_SYMBOL(iov_iter_revert);
 931
 932 /*
 933  * Return the count of just the current iov_iter segment.
 934  */
 935 size_t iov_iter_single_seg_count(const struct iov_iter *i)
 936 {
 937         if (unlikely(i->type & ITER_PIPE))
 938                 return i->count;        // it is a silly place, anyway
 939         if (i->nr_segs == 1)
 940                 return i->count;
 941         else if (i->type & ITER_BVEC)
 942                 return min(i->count, i->bvec->bv_len - i->iov_offset);
 943         else
 944                 return min(i->count, i->iov->iov_len - i->iov_offset);
 945 }
 946 EXPORT_SYMBOL(iov_iter_single_seg_count);
 947
 948 void iov_iter_kvec(struct iov_iter *i, int direction,
 949                         const struct kvec *kvec, unsigned long nr_segs,
 950                         size_t count)
 951 {
 952         BUG_ON(!(direction & ITER_KVEC));
 953         i->type = direction;
 954         i->kvec = kvec;
 955         i->nr_segs = nr_segs;
 956         i->iov_offset = 0;
 957         i->count = count;
 958 }
 959 EXPORT_SYMBOL(iov_iter_kvec);
 960
 961 void iov_iter_bvec(struct iov_iter *i, int direction,
 962                         const struct bio_vec *bvec, unsigned long nr_segs,
 963                         size_t count)
 964 {
 965         BUG_ON(!(direction & ITER_BVEC));
 966         i->type = direction;
 967         i->bvec = bvec;
 968         i->nr_segs = nr_segs;
 969         i->iov_offset = 0;
 970         i->count = count;
 971 }
 972 EXPORT_SYMBOL(iov_iter_bvec);
 973
 974 void iov_iter_pipe(struct iov_iter *i, int direction,
 975                         struct pipe_inode_info *pipe,
 976                         size_t count)
 977 {
 978         BUG_ON(direction != ITER_PIPE);
 979         WARN_ON(pipe->nrbufs == pipe->buffers);
 980         i->type = direction;
 981         i->pipe = pipe;
 982         i->idx = (pipe->curbuf + pipe->nrbufs) & (pipe->buffers - 1);
 983         i->iov_offset = 0;
 984         i->count = count;
 985         i->start_idx = i->idx;
 986 }
 987 EXPORT_SYMBOL(iov_iter_pipe);
 988
 989 unsigned long iov_iter_alignment(const struct iov_iter *i)
 990 {
 991         unsigned long res = 0;
 992         size_t size = i->count;
 993
 994         if (unlikely(i->type & ITER_PIPE)) {
 995                 if (size && i->iov_offset && allocated(&i->pipe->bufs[i->idx]))
 996                         return size | i->iov_offset;
 997                 return size;
 998         }
 999         iterate_all_kinds(i, size, v,
1000                 (res |= (unsigned long)v.iov_base | v.iov_len, 0),
1001                 res |= v.bv_offset | v.bv_len,
1002                 res |= (unsigned long)v.iov_base | v.iov_len
1003         )
1004         return res;
1005 }
1006 EXPORT_SYMBOL(iov_iter_alignment);
1007
1008 unsigned long iov_iter_gap_alignment(const struct iov_iter *i)
1009 {
1010         unsigned long res = 0;
1011         size_t size = i->count;
1012
1013         if (unlikely(i->type & ITER_PIPE)) {
1014                 WARN_ON(1);
1015                 return ~0U;
1016         }
1017
1018         iterate_all_kinds(i, size, v,
1019                 (res |= (!res ? 0 : (unsigned long)v.iov_base) |
1020                         (size != v.iov_len ? size : 0), 0),
1021                 (res |= (!res ? 0 : (unsigned long)v.bv_offset) |
1022                         (size != v.bv_len ? size : 0)),
1023                 (res |= (!res ? 0 : (unsigned long)v.iov_base) |
1024                         (size != v.iov_len ? size : 0))
1025                 );
1026         return res;
1027 }
1028 EXPORT_SYMBOL(iov_iter_gap_alignment);
1029
1030 static inline size_t __pipe_get_pages(struct iov_iter *i,
1031                                 size_t maxsize,
1032                                 struct page **pages,
1033                                 int idx,
1034                                 size_t *start)
1035 {
1036         struct pipe_inode_info *pipe = i->pipe;
1037         ssize_t n = push_pipe(i, maxsize, &idx, start);
1038         if (!n)
1039                 return -EFAULT;
1040
1041         maxsize = n;
1042         n += *start;
1043         while (n > 0) {
1044                 get_page(*pages++ = pipe->bufs[idx].page);
1045                 idx = next_idx(idx, pipe);
1046                 n -= PAGE_SIZE;
1047         }
1048
1049         return maxsize;
1050 }
1051
1052 static ssize_t pipe_get_pages(struct iov_iter *i,
1053                    struct page **pages, size_t maxsize, unsigned maxpages,
1054                    size_t *start)
1055 {
1056         unsigned npages;
1057         size_t capacity;
1058         int idx;
1059
1060         if (!maxsize)
1061                 return 0;
1062
1063         if (!sanity(i))
1064                 return -EFAULT;
1065
1066         data_start(i, &idx, start);
1067         /* some of this one + all after this one */
1068         npages = ((i->pipe->curbuf - idx - 1) & (i->pipe->buffers - 1)) + 1;
1069         capacity = min(npages,maxpages) * PAGE_SIZE - *start;
1070
1071         return __pipe_get_pages(i, min(maxsize, capacity), pages, idx, start);
1072 }
1073
1074 ssize_t iov_iter_get_pages(struct iov_iter *i,
1075                    struct page **pages, size_t maxsize, unsigned maxpages,
1076                    size_t *start)
1077 {
1078         if (maxsize > i->count)
1079                 maxsize = i->count;
1080
1081         if (unlikely(i->type & ITER_PIPE))
1082                 return pipe_get_pages(i, pages, maxsize, maxpages, start);
1083         iterate_all_kinds(i, maxsize, v, ({
1084                 unsigned long addr = (unsigned long)v.iov_base;
1085                 size_t len = v.iov_len + (*start = addr & (PAGE_SIZE - 1));
1086                 int n;
1087                 int res;
1088
1089                 if (len > maxpages * PAGE_SIZE)
1090                         len = maxpages * PAGE_SIZE;
1091                 addr &= ~(PAGE_SIZE - 1);
1092                 n = DIV_ROUND_UP(len, PAGE_SIZE);
1093                 res = get_user_pages_fast(addr, n, (i->type & WRITE) != WRITE, pages);
1094                 if (unlikely(res < 0))
1095                         return res;
1096                 return (res == n ? len : res * PAGE_SIZE) - *start;
1097         0;}),({
1098                 /* can't be more than PAGE_SIZE */
1099                 *start = v.bv_offset;
1100                 get_page(*pages = v.bv_page);
1101                 return v.bv_len;
1102         }),({
1103                 return -EFAULT;
1104         })
1105         )
1106         return 0;
1107 }
1108 EXPORT_SYMBOL(iov_iter_get_pages);
1109
1110 static struct page **get_pages_array(size_t n)
1111 {
1112         return kvmalloc_array(n, sizeof(struct page *), GFP_KERNEL);
1113 }
1114
1115 static ssize_t pipe_get_pages_alloc(struct iov_iter *i,
1116                    struct page ***pages, size_t maxsize,
1117                    size_t *start)
1118 {
1119         struct page **p;
1120         size_t n;
1121         int idx;
1122         int npages;
1123
1124         if (!maxsize)
1125                 return 0;
1126
1127         if (!sanity(i))
1128                 return -EFAULT;
1129
1130         data_start(i, &idx, start);
1131         /* some of this one + all after this one */
1132         npages = ((i->pipe->curbuf - idx - 1) & (i->pipe->buffers - 1)) + 1;
1133         n = npages * PAGE_SIZE - *start;
1134         if (maxsize > n)
1135                 maxsize = n;
1136         else
1137                 npages = DIV_ROUND_UP(maxsize + *start, PAGE_SIZE);
1138         p = get_pages_array(npages);
1139         if (!p)
1140                 return -ENOMEM;
1141         n = __pipe_get_pages(i, maxsize, p, idx, start);
1142         if (n > 0)
1143                 *pages = p;
1144         else
1145                 kvfree(p);
1146         return n;
1147 }
1148
1149 ssize_t iov_iter_get_pages_alloc(struct iov_iter *i,
1150                    struct page ***pages, size_t maxsize,
1151                    size_t *start)
1152 {
1153         struct page **p;
1154
1155         if (maxsize > i->count)
1156                 maxsize = i->count;
1157
1158         if (unlikely(i->type & ITER_PIPE))
1159                 return pipe_get_pages_alloc(i, pages, maxsize, start);
1160         iterate_all_kinds(i, maxsize, v, ({
1161                 unsigned long addr = (unsigned long)v.iov_base;
1162                 size_t len = v.iov_len + (*start = addr & (PAGE_SIZE - 1));
1163                 int n;
1164                 int res;
1165
1166                 addr &= ~(PAGE_SIZE - 1);
1167                 n = DIV_ROUND_UP(len, PAGE_SIZE);
1168                 p = get_pages_array(n);
1169                 if (!p)
1170                         return -ENOMEM;
1171                 res = get_user_pages_fast(addr, n, (i->type & WRITE) != WRITE, p);
1172                 if (unlikely(res < 0)) {
1173                         kvfree(p);
1174                         return res;
1175                 }
1176                 *pages = p;
1177                 return (res == n ? len : res * PAGE_SIZE) - *start;
1178         0;}),({
1179                 /* can't be more than PAGE_SIZE */
1180                 *start = v.bv_offset;
1181                 *pages = p = get_pages_array(1);
1182                 if (!p)
1183                         return -ENOMEM;
1184                 get_page(*p = v.bv_page);
1185                 return v.bv_len;
1186         }),({
1187                 return -EFAULT;
1188         })
1189         )
1190         return 0;
1191 }
1192 EXPORT_SYMBOL(iov_iter_get_pages_alloc);
1193
1194 size_t csum_and_copy_from_iter(void *addr, size_t bytes, __wsum *csum,
1195                                struct iov_iter *i)
1196 {
1197         char *to = addr;
1198         __wsum sum, next;
1199         size_t off = 0;
1200         sum = *csum;
1201         if (unlikely(i->type & ITER_PIPE)) {
1202                 WARN_ON(1);
1203                 return 0;
1204         }
1205         iterate_and_advance(i, bytes, v, ({
1206                 int err = 0;
1207                 next = csum_and_copy_from_user(v.iov_base,
1208                                                (to += v.iov_len) - v.iov_len,
1209                                                v.iov_len, 0, &err);
1210                 if (!err) {
1211                         sum = csum_block_add(sum, next, off);
1212                         off += v.iov_len;
1213                 }
1214                 err ? v.iov_len : 0;
1215         }), ({
1216                 char *p = kmap_atomic(v.bv_page);
1217                 next = csum_partial_copy_nocheck(p + v.bv_offset,
1218                                                  (to += v.bv_len) - v.bv_len,
1219                                                  v.bv_len, 0);
1220                 kunmap_atomic(p);
1221                 sum = csum_block_add(sum, next, off);
1222                 off += v.bv_len;
1223         }),({
1224                 next = csum_partial_copy_nocheck(v.iov_base,
1225                                                  (to += v.iov_len) - v.iov_len,
1226                                                  v.iov_len, 0);
1227                 sum = csum_block_add(sum, next, off);
1228                 off += v.iov_len;
1229         })
1230         )
1231         *csum = sum;
1232         return bytes;
1233 }
1234 EXPORT_SYMBOL(csum_and_copy_from_iter);
1235
1236 bool csum_and_copy_from_iter_full(void *addr, size_t bytes, __wsum *csum,
1237                                struct iov_iter *i)
1238 {
1239         char *to = addr;
1240         __wsum sum, next;
1241         size_t off = 0;
1242         sum = *csum;
1243         if (unlikely(i->type & ITER_PIPE)) {
1244                 WARN_ON(1);
1245                 return false;
1246         }
1247         if (unlikely(i->count < bytes))
1248                 return false;
1249         iterate_all_kinds(i, bytes, v, ({
1250                 int err = 0;
1251                 next = csum_and_copy_from_user(v.iov_base,
1252                                                (to += v.iov_len) - v.iov_len,
1253                                                v.iov_len, 0, &err);
1254                 if (err)
1255                         return false;
1256                 sum = csum_block_add(sum, next, off);
1257                 off += v.iov_len;
1258                 0;
1259         }), ({
1260                 char *p = kmap_atomic(v.bv_page);
1261                 next = csum_partial_copy_nocheck(p + v.bv_offset,
1262                                                  (to += v.bv_len) - v.bv_len,
1263                                                  v.bv_len, 0);
1264                 kunmap_atomic(p);
1265                 sum = csum_block_add(sum, next, off);
1266                 off += v.bv_len;
1267         }),({
1268                 next = csum_partial_copy_nocheck(v.iov_base,
1269                                                  (to += v.iov_len) - v.iov_len,
1270                                                  v.iov_len, 0);
1271                 sum = csum_block_add(sum, next, off);
1272                 off += v.iov_len;
1273         })
1274         )
1275         *csum = sum;
1276         iov_iter_advance(i, bytes);
1277         return true;
1278 }
1279 EXPORT_SYMBOL(csum_and_copy_from_iter_full);
1280
1281 size_t csum_and_copy_to_iter(const void *addr, size_t bytes, __wsum *csum,
1282                              struct iov_iter *i)
1283 {
1284         const char *from = addr;
1285         __wsum sum, next;
1286         size_t off = 0;
1287         sum = *csum;
1288         if (unlikely(i->type & ITER_PIPE)) {
1289                 WARN_ON(1);     /* for now */
1290                 return 0;
1291         }
1292         iterate_and_advance(i, bytes, v, ({
1293                 int err = 0;
1294                 next = csum_and_copy_to_user((from += v.iov_len) - v.iov_len,
1295                                              v.iov_base,
1296                                              v.iov_len, 0, &err);
1297                 if (!err) {
1298                         sum = csum_block_add(sum, next, off);
1299                         off += v.iov_len;
1300                 }
1301                 err ? v.iov_len : 0;
1302         }), ({
1303                 char *p = kmap_atomic(v.bv_page);
1304                 next = csum_partial_copy_nocheck((from += v.bv_len) - v.bv_len,
1305                                                  p + v.bv_offset,
1306                                                  v.bv_len, 0);
1307                 kunmap_atomic(p);
1308                 sum = csum_block_add(sum, next, off);
1309                 off += v.bv_len;
1310         }),({
1311                 next = csum_partial_copy_nocheck((from += v.iov_len) - v.iov_len,
1312                                                  v.iov_base,
1313                                                  v.iov_len, 0);
1314                 sum = csum_block_add(sum, next, off);
1315                 off += v.iov_len;
1316         })
1317         )
1318         *csum = sum;
1319         return bytes;
1320 }
1321 EXPORT_SYMBOL(csum_and_copy_to_iter);
1322
1323 int iov_iter_npages(const struct iov_iter *i, int maxpages)
1324 {
1325         size_t size = i->count;
1326         int npages = 0;
1327
1328         if (!size)
1329                 return 0;
1330
1331         if (unlikely(i->type & ITER_PIPE)) {
1332                 struct pipe_inode_info *pipe = i->pipe;
1333                 size_t off;
1334                 int idx;
1335
1336                 if (!sanity(i))
1337                         return 0;
1338
1339                 data_start(i, &idx, &off);
1340                 /* some of this one + all after this one */
1341                 npages = ((pipe->curbuf - idx - 1) & (pipe->buffers - 1)) + 1;
1342                 if (npages >= maxpages)
1343                         return maxpages;
1344         } else iterate_all_kinds(i, size, v, ({
1345                 unsigned long p = (unsigned long)v.iov_base;
1346                 npages += DIV_ROUND_UP(p + v.iov_len, PAGE_SIZE)
1347                         - p / PAGE_SIZE;
1348                 if (npages >= maxpages)
1349                         return maxpages;
1350         0;}),({
1351                 npages++;
1352                 if (npages >= maxpages)
1353                         return maxpages;
1354         }),({
1355                 unsigned long p = (unsigned long)v.iov_base;
1356                 npages += DIV_ROUND_UP(p + v.iov_len, PAGE_SIZE)
1357                         - p / PAGE_SIZE;
1358                 if (npages >= maxpages)
1359                         return maxpages;
1360         })
1361         )
1362         return npages;
1363 }
1364 EXPORT_SYMBOL(iov_iter_npages);
1365
1366 const void *dup_iter(struct iov_iter *new, struct iov_iter *old, gfp_t flags)
1367 {
1368         *new = *old;
1369         if (unlikely(new->type & ITER_PIPE)) {
1370                 WARN_ON(1);
1371                 return NULL;
1372         }
1373         if (new->type & ITER_BVEC)
1374                 return new->bvec = kmemdup(new->bvec,
1375                                     new->nr_segs * sizeof(struct bio_vec),
1376                                     flags);
1377         else
1378                 /* iovec and kvec have identical layout */
1379                 return new->iov = kmemdup(new->iov,
1380                                    new->nr_segs * sizeof(struct iovec),
1381                                    flags);
1382 }
1383 EXPORT_SYMBOL(dup_iter);
1384
1385 /**
1386  * import_iovec() - Copy an array of &struct iovec from userspace
1387  *     into the kernel, check that it is valid, and initialize a new
1388  *     &struct iov_iter iterator to access it.
1389  *
1390  * @type: One of %READ or %WRITE.
1391  * @uvector: Pointer to the userspace array.
1392  * @nr_segs: Number of elements in userspace array.
1393  * @fast_segs: Number of elements in @iov.
1394  * @iov: (input and output parameter) Pointer to pointer to (usually small
1395  *     on-stack) kernel array.
1396  * @i: Pointer to iterator that will be initialized on success.
1397  *
1398  * If the array pointed to by *@iov is large enough to hold all @nr_segs,
1399  * then this function places %NULL in *@iov on return. Otherwise, a new
1400  * array will be allocated and the result placed in *@iov. This means that
1401  * the caller may call kfree() on *@iov regardless of whether the small
1402  * on-stack array was used or not (and regardless of whether this function
1403  * returns an error or not).
1404  *
1405  * Return: 0 on success or negative error code on error.
1406  */
1407 int import_iovec(int type, const struct iovec __user * uvector,
1408                  unsigned nr_segs, unsigned fast_segs,
1409                  struct iovec **iov, struct iov_iter *i)
1410 {
1411         ssize_t n;
1412         struct iovec *p;
1413         n = rw_copy_check_uvector(type, uvector, nr_segs, fast_segs,
1414                                   *iov, &p);
1415         if (n < 0) {
1416                 if (p != *iov)
1417                         kfree(p);
1418                 *iov = NULL;
1419                 return n;
1420         }
1421         iov_iter_init(i, type, p, nr_segs, n);
1422         *iov = p == *iov ? NULL : p;
1423         return 0;
1424 }
1425 EXPORT_SYMBOL(import_iovec);
1426
1427 #ifdef CONFIG_COMPAT
1428 #include <linux/compat.h>
1429
1430 int compat_import_iovec(int type, const struct compat_iovec __user * uvector,
1431                  unsigned nr_segs, unsigned fast_segs,
1432                  struct iovec **iov, struct iov_iter *i)
1433 {
1434         ssize_t n;
1435         struct iovec *p;
1436         n = compat_rw_copy_check_uvector(type, uvector, nr_segs, fast_segs,
1437                                   *iov, &p);
1438         if (n < 0) {
1439                 if (p != *iov)
1440                         kfree(p);
1441                 *iov = NULL;
1442                 return n;
1443         }
1444         iov_iter_init(i, type, p, nr_segs, n);
1445         *iov = p == *iov ? NULL : p;
1446         return 0;
1447 }
1448 #endif
1449
1450 int import_single_range(int rw, void __user *buf, size_t len,
1451                  struct iovec *iov, struct iov_iter *i)
1452 {
1453         if (len > MAX_RW_COUNT)
1454                 len = MAX_RW_COUNT;
1455         if (unlikely(!access_ok(!rw, buf, len)))
1456                 return -EFAULT;
1457
1458         iov->iov_base = buf;
1459         iov->iov_len = len;
1460         iov_iter_init(i, rw, iov, 1, len);
1461         return 0;
1462 }
1463 EXPORT_SYMBOL(import_single_range);