2019-05-19 15:08:55 +03:00
|
|
|
// SPDX-License-Identifier: GPL-2.0-only
|
2020-06-12 09:57:37 +03:00
|
|
|
#include <crypto/hash.h>
|
2014-02-06 04:11:33 +04:00
|
|
|
#include <linux/export.h>
|
2016-11-01 16:40:13 +03:00
|
|
|
#include <linux/bvec.h>
|
2020-10-16 06:13:50 +03:00
|
|
|
#include <linux/fault-inject-usercopy.h>
|
2014-02-06 04:11:33 +04:00
|
|
|
#include <linux/uio.h>
|
|
|
|
#include <linux/pagemap.h>
|
2021-05-05 04:40:03 +03:00
|
|
|
#include <linux/highmem.h>
|
2014-03-21 12:58:33 +04:00
|
|
|
#include <linux/slab.h>
|
|
|
|
#include <linux/vmalloc.h>
|
2016-09-22 23:33:12 +03:00
|
|
|
#include <linux/splice.h>
|
2020-09-25 07:51:40 +03:00
|
|
|
#include <linux/compat.h>
|
2014-11-24 09:08:00 +03:00
|
|
|
#include <net/checksum.h>
|
2018-12-04 04:52:09 +03:00
|
|
|
#include <linux/scatterlist.h>
|
2020-01-21 19:05:11 +03:00
|
|
|
#include <linux/instrumented.h>
|
2014-02-06 04:11:33 +04:00
|
|
|
|
2022-05-22 21:59:25 +03:00
|
|
|
/* covers ubuf and kbuf alike */
|
|
|
|
#define iterate_buf(i, n, base, len, off, __p, STEP) { \
|
|
|
|
size_t __maybe_unused off = 0; \
|
|
|
|
len = n; \
|
|
|
|
base = __p + i->iov_offset; \
|
|
|
|
len -= (STEP); \
|
|
|
|
i->iov_offset += len; \
|
|
|
|
n = len; \
|
|
|
|
}
|
|
|
|
|
2021-04-26 06:57:42 +03:00
|
|
|
/* covers iovec and kvec alike */
|
2021-05-02 20:03:41 +03:00
|
|
|
#define iterate_iovec(i, n, base, len, off, __p, STEP) { \
|
2021-05-02 18:35:03 +03:00
|
|
|
size_t off = 0; \
|
2021-05-02 20:03:41 +03:00
|
|
|
size_t skip = i->iov_offset; \
|
2021-04-26 06:46:09 +03:00
|
|
|
do { \
|
2021-05-02 18:35:03 +03:00
|
|
|
len = min(n, __p->iov_len - skip); \
|
|
|
|
if (likely(len)) { \
|
|
|
|
base = __p->iov_base + skip; \
|
|
|
|
len -= (STEP); \
|
|
|
|
off += len; \
|
|
|
|
skip += len; \
|
|
|
|
n -= len; \
|
2021-04-26 06:46:09 +03:00
|
|
|
if (skip < __p->iov_len) \
|
|
|
|
break; \
|
|
|
|
} \
|
|
|
|
__p++; \
|
|
|
|
skip = 0; \
|
|
|
|
} while (n); \
|
2021-05-02 20:03:41 +03:00
|
|
|
i->iov_offset = skip; \
|
2021-05-02 18:35:03 +03:00
|
|
|
n = off; \
|
iov_iter.c: macros for iterating over iov_iter
iterate_all_kinds(iter, size, ident, step_iovec, step_bvec)
iterates through the ranges covered by iter (up to size bytes total),
repeating step_iovec or step_bvec for each of those. ident is
declared in expansion of that thing, either as struct iovec or
struct bvec, and it contains the range we are currently looking
at. step_bvec should be a void expression, step_iovec - a size_t
one, with non-zero meaning "stop here, that many bytes from this
range left". In the end, the amount actually handled is stored
in size.
iov_iter_copy_from_user_atomic() and iov_iter_alignment() converted
to it.
Signed-off-by: Al Viro <viro@zeniv.linux.org.uk>
2014-11-27 21:51:41 +03:00
|
|
|
}
|
|
|
|
|
2021-05-02 20:03:41 +03:00
|
|
|
#define iterate_bvec(i, n, base, len, off, p, STEP) { \
|
2021-05-02 18:35:03 +03:00
|
|
|
size_t off = 0; \
|
2021-05-02 20:03:41 +03:00
|
|
|
unsigned skip = i->iov_offset; \
|
2021-04-27 03:19:14 +03:00
|
|
|
while (n) { \
|
|
|
|
unsigned offset = p->bv_offset + skip; \
|
2021-04-27 03:33:42 +03:00
|
|
|
unsigned left; \
|
2021-04-27 03:50:05 +03:00
|
|
|
void *kaddr = kmap_local_page(p->bv_page + \
|
|
|
|
offset / PAGE_SIZE); \
|
2021-05-02 18:35:03 +03:00
|
|
|
base = kaddr + offset % PAGE_SIZE; \
|
2021-05-02 20:03:41 +03:00
|
|
|
len = min(min(n, (size_t)(p->bv_len - skip)), \
|
2021-04-27 03:19:14 +03:00
|
|
|
(size_t)(PAGE_SIZE - offset % PAGE_SIZE)); \
|
2021-04-27 03:33:42 +03:00
|
|
|
left = (STEP); \
|
2021-04-27 03:50:05 +03:00
|
|
|
kunmap_local(kaddr); \
|
2021-05-02 18:35:03 +03:00
|
|
|
len -= left; \
|
|
|
|
off += len; \
|
|
|
|
skip += len; \
|
2021-04-27 03:19:14 +03:00
|
|
|
if (skip == p->bv_len) { \
|
|
|
|
skip = 0; \
|
|
|
|
p++; \
|
|
|
|
} \
|
2021-05-02 18:35:03 +03:00
|
|
|
n -= len; \
|
2021-04-27 03:33:42 +03:00
|
|
|
if (left) \
|
|
|
|
break; \
|
2021-04-27 03:19:14 +03:00
|
|
|
} \
|
2021-05-02 20:03:41 +03:00
|
|
|
i->iov_offset = skip; \
|
2021-05-02 18:35:03 +03:00
|
|
|
n = off; \
|
iov_iter.c: macros for iterating over iov_iter
iterate_all_kinds(iter, size, ident, step_iovec, step_bvec)
iterates through the ranges covered by iter (up to size bytes total),
repeating step_iovec or step_bvec for each of those. ident is
declared in expansion of that thing, either as struct iovec or
struct bvec, and it contains the range we are currently looking
at. step_bvec should be a void expression, step_iovec - a size_t
one, with non-zero meaning "stop here, that many bytes from this
range left". In the end, the amount actually handled is stored
in size.
iov_iter_copy_from_user_atomic() and iov_iter_alignment() converted
to it.
Signed-off-by: Al Viro <viro@zeniv.linux.org.uk>
2014-11-27 21:51:41 +03:00
|
|
|
}
|
|
|
|
|
2021-05-02 20:03:41 +03:00
|
|
|
#define iterate_xarray(i, n, base, len, __off, STEP) { \
|
2021-04-27 03:33:42 +03:00
|
|
|
__label__ __out; \
|
2021-05-02 18:13:09 +03:00
|
|
|
size_t __off = 0; \
|
2021-11-29 03:18:27 +03:00
|
|
|
struct folio *folio; \
|
2021-05-02 20:03:41 +03:00
|
|
|
loff_t start = i->xarray_start + i->iov_offset; \
|
2021-05-05 00:50:07 +03:00
|
|
|
pgoff_t index = start / PAGE_SIZE; \
|
2020-02-10 13:00:21 +03:00
|
|
|
XA_STATE(xas, i->xarray, index); \
|
|
|
|
\
|
2021-11-29 03:18:27 +03:00
|
|
|
len = PAGE_SIZE - offset_in_page(start); \
|
2021-05-02 18:35:03 +03:00
|
|
|
rcu_read_lock(); \
|
2021-11-29 03:18:27 +03:00
|
|
|
xas_for_each(&xas, folio, ULONG_MAX) { \
|
2021-05-02 18:35:03 +03:00
|
|
|
unsigned left; \
|
2021-11-29 03:18:27 +03:00
|
|
|
size_t offset; \
|
|
|
|
if (xas_retry(&xas, folio)) \
|
2021-05-02 18:35:03 +03:00
|
|
|
continue; \
|
2021-11-29 03:18:27 +03:00
|
|
|
if (WARN_ON(xa_is_value(folio))) \
|
2021-05-02 18:35:03 +03:00
|
|
|
break; \
|
2021-11-29 03:18:27 +03:00
|
|
|
if (WARN_ON(folio_test_hugetlb(folio))) \
|
2021-05-02 18:35:03 +03:00
|
|
|
break; \
|
2021-11-29 03:18:27 +03:00
|
|
|
offset = offset_in_folio(folio, start + __off); \
|
|
|
|
while (offset < folio_size(folio)) { \
|
|
|
|
base = kmap_local_folio(folio, offset); \
|
2021-05-02 18:35:03 +03:00
|
|
|
len = min(n, len); \
|
|
|
|
left = (STEP); \
|
2021-11-29 03:18:27 +03:00
|
|
|
kunmap_local(base); \
|
2021-05-02 18:35:03 +03:00
|
|
|
len -= left; \
|
|
|
|
__off += len; \
|
|
|
|
n -= len; \
|
|
|
|
if (left || n == 0) \
|
|
|
|
goto __out; \
|
2021-11-29 03:18:27 +03:00
|
|
|
offset += len; \
|
|
|
|
len = PAGE_SIZE; \
|
2021-05-02 18:35:03 +03:00
|
|
|
} \
|
2020-02-10 13:00:21 +03:00
|
|
|
} \
|
2021-04-27 03:33:42 +03:00
|
|
|
__out: \
|
2020-02-10 13:00:21 +03:00
|
|
|
rcu_read_unlock(); \
|
2021-11-29 03:18:27 +03:00
|
|
|
i->iov_offset += __off; \
|
2021-05-02 18:13:09 +03:00
|
|
|
n = __off; \
|
2020-02-10 13:00:21 +03:00
|
|
|
}
|
|
|
|
|
2021-05-02 18:35:03 +03:00
|
|
|
#define __iterate_and_advance(i, n, base, len, off, I, K) { \
|
2016-05-09 18:54:48 +03:00
|
|
|
if (unlikely(i->count < n)) \
|
|
|
|
n = i->count; \
|
2021-04-29 03:59:08 +03:00
|
|
|
if (likely(n)) { \
|
2022-05-22 21:59:25 +03:00
|
|
|
if (likely(iter_is_ubuf(i))) { \
|
|
|
|
void __user *base; \
|
|
|
|
size_t len; \
|
|
|
|
iterate_buf(i, n, base, len, off, \
|
|
|
|
i->ubuf, (I)) \
|
|
|
|
} else if (likely(iter_is_iovec(i))) { \
|
2023-03-29 17:52:15 +03:00
|
|
|
const struct iovec *iov = iter_iov(i); \
|
2021-05-02 18:35:03 +03:00
|
|
|
void __user *base; \
|
|
|
|
size_t len; \
|
|
|
|
iterate_iovec(i, n, base, len, off, \
|
2021-05-02 20:03:41 +03:00
|
|
|
iov, (I)) \
|
2023-03-29 17:52:15 +03:00
|
|
|
i->nr_segs -= iov - iter_iov(i); \
|
|
|
|
i->__iov = iov; \
|
2021-06-03 00:25:59 +03:00
|
|
|
} else if (iov_iter_is_bvec(i)) { \
|
2016-05-30 16:34:32 +03:00
|
|
|
const struct bio_vec *bvec = i->bvec; \
|
2021-05-02 18:35:03 +03:00
|
|
|
void *base; \
|
|
|
|
size_t len; \
|
|
|
|
iterate_bvec(i, n, base, len, off, \
|
2021-05-02 20:03:41 +03:00
|
|
|
bvec, (K)) \
|
2021-04-27 03:19:14 +03:00
|
|
|
i->nr_segs -= bvec - i->bvec; \
|
|
|
|
i->bvec = bvec; \
|
2021-06-03 00:25:59 +03:00
|
|
|
} else if (iov_iter_is_kvec(i)) { \
|
2021-04-26 06:57:42 +03:00
|
|
|
const struct kvec *kvec = i->kvec; \
|
2021-05-02 18:35:03 +03:00
|
|
|
void *base; \
|
|
|
|
size_t len; \
|
|
|
|
iterate_iovec(i, n, base, len, off, \
|
2021-05-02 20:03:41 +03:00
|
|
|
kvec, (K)) \
|
2016-05-09 18:54:48 +03:00
|
|
|
i->nr_segs -= kvec - i->kvec; \
|
|
|
|
i->kvec = kvec; \
|
2021-06-03 00:25:59 +03:00
|
|
|
} else if (iov_iter_is_xarray(i)) { \
|
2021-05-02 18:35:03 +03:00
|
|
|
void *base; \
|
|
|
|
size_t len; \
|
|
|
|
iterate_xarray(i, n, base, len, off, \
|
2021-05-02 20:03:41 +03:00
|
|
|
(K)) \
|
2014-11-27 21:59:45 +03:00
|
|
|
} \
|
2016-05-09 18:54:48 +03:00
|
|
|
i->count -= n; \
|
2014-11-27 21:59:45 +03:00
|
|
|
} \
|
|
|
|
}
|
2021-05-02 18:35:03 +03:00
|
|
|
#define iterate_and_advance(i, n, base, len, off, I, K) \
|
|
|
|
__iterate_and_advance(i, n, base, len, off, I, ((void)(K),0))
|
2014-11-27 21:59:45 +03:00
|
|
|
|
2017-06-30 05:25:14 +03:00
|
|
|
static int copyout(void __user *to, const void *from, size_t n)
|
|
|
|
{
|
2020-10-16 06:13:50 +03:00
|
|
|
if (should_fail_usercopy())
|
|
|
|
return n;
|
Remove 'type' argument from access_ok() function
Nobody has actually used the type (VERIFY_READ vs VERIFY_WRITE) argument
of the user address range verification function since we got rid of the
old racy i386-only code to walk page tables by hand.
It existed because the original 80386 would not honor the write protect
bit when in kernel mode, so you had to do COW by hand before doing any
user access. But we haven't supported that in a long time, and these
days the 'type' argument is a purely historical artifact.
A discussion about extending 'user_access_begin()' to do the range
checking resulted this patch, because there is no way we're going to
move the old VERIFY_xyz interface to that model. And it's best done at
the end of the merge window when I've done most of my merges, so let's
just get this done once and for all.
This patch was mostly done with a sed-script, with manual fix-ups for
the cases that weren't of the trivial 'access_ok(VERIFY_xyz' form.
There were a couple of notable cases:
- csky still had the old "verify_area()" name as an alias.
- the iter_iov code had magical hardcoded knowledge of the actual
values of VERIFY_{READ,WRITE} (not that they mattered, since nothing
really used it)
- microblaze used the type argument for a debug printout
but other than those oddities this should be a total no-op patch.
I tried to fix up all architectures, did fairly extensive grepping for
access_ok() uses, and the changes are trivial, but I may have missed
something. Any missed conversion should be trivially fixable, though.
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2019-01-04 05:57:57 +03:00
|
|
|
if (access_ok(to, n)) {
|
2020-01-21 19:05:11 +03:00
|
|
|
instrument_copy_to_user(to, from, n);
|
2017-06-30 05:25:14 +03:00
|
|
|
n = raw_copy_to_user(to, from, n);
|
|
|
|
}
|
|
|
|
return n;
|
|
|
|
}
|
|
|
|
|
2023-03-22 21:57:03 +03:00
|
|
|
static int copyout_nofault(void __user *to, const void *from, size_t n)
|
|
|
|
{
|
|
|
|
long res;
|
|
|
|
|
|
|
|
if (should_fail_usercopy())
|
|
|
|
return n;
|
|
|
|
|
|
|
|
res = copy_to_user_nofault(to, from, n);
|
|
|
|
|
|
|
|
return res < 0 ? n : res;
|
|
|
|
}
|
|
|
|
|
2017-06-30 05:25:14 +03:00
|
|
|
static int copyin(void *to, const void __user *from, size_t n)
|
|
|
|
{
|
2022-09-15 18:03:37 +03:00
|
|
|
size_t res = n;
|
|
|
|
|
2020-10-16 06:13:50 +03:00
|
|
|
if (should_fail_usercopy())
|
|
|
|
return n;
|
Remove 'type' argument from access_ok() function
Nobody has actually used the type (VERIFY_READ vs VERIFY_WRITE) argument
of the user address range verification function since we got rid of the
old racy i386-only code to walk page tables by hand.
It existed because the original 80386 would not honor the write protect
bit when in kernel mode, so you had to do COW by hand before doing any
user access. But we haven't supported that in a long time, and these
days the 'type' argument is a purely historical artifact.
A discussion about extending 'user_access_begin()' to do the range
checking resulted this patch, because there is no way we're going to
move the old VERIFY_xyz interface to that model. And it's best done at
the end of the merge window when I've done most of my merges, so let's
just get this done once and for all.
This patch was mostly done with a sed-script, with manual fix-ups for
the cases that weren't of the trivial 'access_ok(VERIFY_xyz' form.
There were a couple of notable cases:
- csky still had the old "verify_area()" name as an alias.
- the iter_iov code had magical hardcoded knowledge of the actual
values of VERIFY_{READ,WRITE} (not that they mattered, since nothing
really used it)
- microblaze used the type argument for a debug printout
but other than those oddities this should be a total no-op patch.
I tried to fix up all architectures, did fairly extensive grepping for
access_ok() uses, and the changes are trivial, but I may have missed
something. Any missed conversion should be trivially fixable, though.
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2019-01-04 05:57:57 +03:00
|
|
|
if (access_ok(from, n)) {
|
2022-09-15 18:03:37 +03:00
|
|
|
instrument_copy_from_user_before(to, from, n);
|
|
|
|
res = raw_copy_from_user(to, from, n);
|
|
|
|
instrument_copy_from_user_after(to, from, n, res);
|
2017-06-30 05:25:14 +03:00
|
|
|
}
|
2022-09-15 18:03:37 +03:00
|
|
|
return res;
|
2017-06-30 05:25:14 +03:00
|
|
|
}
|
|
|
|
|
2015-03-11 17:43:31 +03:00
|
|
|
/*
|
2021-08-02 15:54:16 +03:00
|
|
|
* fault_in_iov_iter_readable - fault in iov iterator for reading
|
|
|
|
* @i: iterator
|
|
|
|
* @size: maximum length
|
|
|
|
*
|
2015-03-11 17:43:31 +03:00
|
|
|
* Fault in one or more iovecs of the given iov_iter, to a maximum length of
|
2021-08-02 15:54:16 +03:00
|
|
|
* @size. For each iovec, fault in each page that constitutes the iovec.
|
|
|
|
*
|
|
|
|
* Returns the number of bytes not faulted in (like copy_to_user() and
|
|
|
|
* copy_from_user()).
|
2015-03-11 17:43:31 +03:00
|
|
|
*
|
2021-08-02 15:54:16 +03:00
|
|
|
* Always returns 0 for non-userspace iterators.
|
2015-03-11 17:43:31 +03:00
|
|
|
*/
|
2021-08-02 15:54:16 +03:00
|
|
|
size_t fault_in_iov_iter_readable(const struct iov_iter *i, size_t size)
|
2015-03-11 17:43:31 +03:00
|
|
|
{
|
2022-05-22 21:59:25 +03:00
|
|
|
if (iter_is_ubuf(i)) {
|
|
|
|
size_t n = min(size, iov_iter_count(i));
|
|
|
|
n -= fault_in_readable(i->ubuf + i->iov_offset, n);
|
|
|
|
return size - n;
|
|
|
|
} else if (iter_is_iovec(i)) {
|
2021-08-02 15:54:16 +03:00
|
|
|
size_t count = min(size, iov_iter_count(i));
|
2021-05-02 18:57:37 +03:00
|
|
|
const struct iovec *p;
|
|
|
|
size_t skip;
|
|
|
|
|
2021-08-02 15:54:16 +03:00
|
|
|
size -= count;
|
2023-03-29 17:52:15 +03:00
|
|
|
for (p = iter_iov(i), skip = i->iov_offset; count; p++, skip = 0) {
|
2021-08-02 15:54:16 +03:00
|
|
|
size_t len = min(count, p->iov_len - skip);
|
|
|
|
size_t ret;
|
2021-05-02 18:57:37 +03:00
|
|
|
|
|
|
|
if (unlikely(!len))
|
|
|
|
continue;
|
2021-08-02 15:54:16 +03:00
|
|
|
ret = fault_in_readable(p->iov_base + skip, len);
|
|
|
|
count -= len - ret;
|
|
|
|
if (ret)
|
|
|
|
break;
|
2021-05-02 18:57:37 +03:00
|
|
|
}
|
2021-08-02 15:54:16 +03:00
|
|
|
return count + size;
|
2015-03-11 17:43:31 +03:00
|
|
|
}
|
|
|
|
return 0;
|
|
|
|
}
|
2021-08-02 15:54:16 +03:00
|
|
|
EXPORT_SYMBOL(fault_in_iov_iter_readable);
|
2015-03-11 17:43:31 +03:00
|
|
|
|
2021-07-05 18:26:28 +03:00
|
|
|
/*
|
|
|
|
* fault_in_iov_iter_writeable - fault in iov iterator for writing
|
|
|
|
* @i: iterator
|
|
|
|
* @size: maximum length
|
|
|
|
*
|
|
|
|
* Faults in the iterator using get_user_pages(), i.e., without triggering
|
|
|
|
* hardware page faults. This is primarily useful when we already know that
|
|
|
|
* some or all of the pages in @i aren't in memory.
|
|
|
|
*
|
|
|
|
* Returns the number of bytes not faulted in, like copy_to_user() and
|
|
|
|
* copy_from_user().
|
|
|
|
*
|
|
|
|
* Always returns 0 for non-user-space iterators.
|
|
|
|
*/
|
|
|
|
size_t fault_in_iov_iter_writeable(const struct iov_iter *i, size_t size)
|
|
|
|
{
|
2022-05-22 21:59:25 +03:00
|
|
|
if (iter_is_ubuf(i)) {
|
|
|
|
size_t n = min(size, iov_iter_count(i));
|
|
|
|
n -= fault_in_safe_writeable(i->ubuf + i->iov_offset, n);
|
|
|
|
return size - n;
|
|
|
|
} else if (iter_is_iovec(i)) {
|
2021-07-05 18:26:28 +03:00
|
|
|
size_t count = min(size, iov_iter_count(i));
|
|
|
|
const struct iovec *p;
|
|
|
|
size_t skip;
|
|
|
|
|
|
|
|
size -= count;
|
2023-03-29 17:52:15 +03:00
|
|
|
for (p = iter_iov(i), skip = i->iov_offset; count; p++, skip = 0) {
|
2021-07-05 18:26:28 +03:00
|
|
|
size_t len = min(count, p->iov_len - skip);
|
|
|
|
size_t ret;
|
|
|
|
|
|
|
|
if (unlikely(!len))
|
|
|
|
continue;
|
|
|
|
ret = fault_in_safe_writeable(p->iov_base + skip, len);
|
|
|
|
count -= len - ret;
|
|
|
|
if (ret)
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
return count + size;
|
|
|
|
}
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(fault_in_iov_iter_writeable);
|
|
|
|
|
2018-10-20 02:57:56 +03:00
|
|
|
void iov_iter_init(struct iov_iter *i, unsigned int direction,
|
2014-03-06 04:28:09 +04:00
|
|
|
const struct iovec *iov, unsigned long nr_segs,
|
|
|
|
size_t count)
|
|
|
|
{
|
2018-10-20 02:57:56 +03:00
|
|
|
WARN_ON(direction & ~(READ | WRITE));
|
2021-04-22 21:50:39 +03:00
|
|
|
*i = (struct iov_iter) {
|
|
|
|
.iter_type = ITER_IOVEC,
|
mm: hwpoison: coredump: support recovery from dump_user_range()
dump_user_range() is used to copy the user page to a coredump file, but if
a hardware memory error occurred during copy, which called from
__kernel_write_iter() in dump_user_range(), it crashes,
CPU: 112 PID: 7014 Comm: mca-recover Not tainted 6.3.0-rc2 #425
pc : __memcpy+0x110/0x260
lr : _copy_from_iter+0x3bc/0x4c8
...
Call trace:
__memcpy+0x110/0x260
copy_page_from_iter+0xcc/0x130
pipe_write+0x164/0x6d8
__kernel_write_iter+0x9c/0x210
dump_user_range+0xc8/0x1d8
elf_core_dump+0x308/0x368
do_coredump+0x2e8/0xa40
get_signal+0x59c/0x788
do_signal+0x118/0x1f8
do_notify_resume+0xf0/0x280
el0_da+0x130/0x138
el0t_64_sync_handler+0x68/0xc0
el0t_64_sync+0x188/0x190
Generally, the '->write_iter' of file ops will use copy_page_from_iter()
and copy_page_from_iter_atomic(), change memcpy() to copy_mc_to_kernel()
in both of them to handle #MC during source read, which stop coredump
processing and kill the task instead of kernel panic, but the source
address may not always a user address, so introduce a new copy_mc flag in
struct iov_iter{} to indicate that the iter could do a safe memory copy,
also introduce the helpers to set/cleck the flag, for now, it's only used
in coredump's dump_user_range(), but it could expand to any other
scenarios to fix the similar issue.
Link: https://lkml.kernel.org/r/20230417045323.11054-1-wangkefeng.wang@huawei.com
Signed-off-by: Kefeng Wang <wangkefeng.wang@huawei.com>
Cc: Alexander Viro <viro@zeniv.linux.org.uk>
Cc: Christian Brauner <brauner@kernel.org>
Cc: Miaohe Lin <linmiaohe@huawei.com>
Cc: Naoya Horiguchi <naoya.horiguchi@nec.com>
Cc: Tong Tiangen <tongtiangen@huawei.com>
Cc: Jens Axboe <axboe@kernel.dk>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
2023-04-17 07:53:23 +03:00
|
|
|
.copy_mc = false,
|
2021-07-12 13:06:14 +03:00
|
|
|
.nofault = false,
|
2022-05-22 21:59:25 +03:00
|
|
|
.user_backed = true,
|
2021-04-22 21:50:39 +03:00
|
|
|
.data_source = direction,
|
2023-03-29 17:52:15 +03:00
|
|
|
.__iov = iov,
|
2021-04-22 21:50:39 +03:00
|
|
|
.nr_segs = nr_segs,
|
|
|
|
.iov_offset = 0,
|
|
|
|
.count = count
|
|
|
|
};
|
2014-03-06 04:28:09 +04:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(iov_iter_init);
|
2014-03-15 12:05:57 +04:00
|
|
|
|
2018-11-28 06:32:59 +03:00
|
|
|
static __wsum csum_and_memcpy(void *to, const void *from, size_t len,
|
|
|
|
__wsum sum, size_t off)
|
|
|
|
{
|
2020-07-11 07:12:07 +03:00
|
|
|
__wsum next = csum_partial_copy_nocheck(from, to, len);
|
2018-11-28 06:32:59 +03:00
|
|
|
return csum_block_add(sum, next, off);
|
|
|
|
}
|
|
|
|
|
2017-06-30 04:45:10 +03:00
|
|
|
size_t _copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i)
|
2014-04-05 07:12:29 +04:00
|
|
|
{
|
2022-09-16 03:11:15 +03:00
|
|
|
if (WARN_ON_ONCE(i->data_source))
|
|
|
|
return 0;
|
2022-05-22 21:59:25 +03:00
|
|
|
if (user_backed_iter(i))
|
2017-06-30 05:25:14 +03:00
|
|
|
might_fault();
|
2021-05-02 18:35:03 +03:00
|
|
|
iterate_and_advance(i, bytes, base, len, off,
|
|
|
|
copyout(base, addr + off, len),
|
|
|
|
memcpy(base, addr + off, len)
|
2014-11-27 22:28:06 +03:00
|
|
|
)
|
2014-04-05 07:12:29 +04:00
|
|
|
|
2014-11-27 22:28:06 +03:00
|
|
|
return bytes;
|
2014-08-01 17:27:22 +04:00
|
|
|
}
|
2017-06-30 04:45:10 +03:00
|
|
|
EXPORT_SYMBOL(_copy_to_iter);
|
2014-08-01 17:27:22 +04:00
|
|
|
|
x86, powerpc: Rename memcpy_mcsafe() to copy_mc_to_{user, kernel}()
In reaction to a proposal to introduce a memcpy_mcsafe_fast()
implementation Linus points out that memcpy_mcsafe() is poorly named
relative to communicating the scope of the interface. Specifically what
addresses are valid to pass as source, destination, and what faults /
exceptions are handled.
Of particular concern is that even though x86 might be able to handle
the semantics of copy_mc_to_user() with its common copy_user_generic()
implementation other archs likely need / want an explicit path for this
case:
On Fri, May 1, 2020 at 11:28 AM Linus Torvalds <torvalds@linux-foundation.org> wrote:
>
> On Thu, Apr 30, 2020 at 6:21 PM Dan Williams <dan.j.williams@intel.com> wrote:
> >
> > However now I see that copy_user_generic() works for the wrong reason.
> > It works because the exception on the source address due to poison
> > looks no different than a write fault on the user address to the
> > caller, it's still just a short copy. So it makes copy_to_user() work
> > for the wrong reason relative to the name.
>
> Right.
>
> And it won't work that way on other architectures. On x86, we have a
> generic function that can take faults on either side, and we use it
> for both cases (and for the "in_user" case too), but that's an
> artifact of the architecture oddity.
>
> In fact, it's probably wrong even on x86 - because it can hide bugs -
> but writing those things is painful enough that everybody prefers
> having just one function.
Replace a single top-level memcpy_mcsafe() with either
copy_mc_to_user(), or copy_mc_to_kernel().
Introduce an x86 copy_mc_fragile() name as the rename for the
low-level x86 implementation formerly named memcpy_mcsafe(). It is used
as the slow / careful backend that is supplanted by a fast
copy_mc_generic() in a follow-on patch.
One side-effect of this reorganization is that separating copy_mc_64.S
to its own file means that perf no longer needs to track dependencies
for its memcpy_64.S benchmarks.
[ bp: Massage a bit. ]
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
Signed-off-by: Borislav Petkov <bp@suse.de>
Reviewed-by: Tony Luck <tony.luck@intel.com>
Acked-by: Michael Ellerman <mpe@ellerman.id.au>
Cc: <stable@vger.kernel.org>
Link: http://lore.kernel.org/r/CAHk-=wjSqtXAqfUJxFtWNwmguFASTgB0dz1dT3V-78Quiezqbg@mail.gmail.com
Link: https://lkml.kernel.org/r/160195561680.2163339.11574962055305783722.stgit@dwillia2-desk3.amr.corp.intel.com
2020-10-06 06:40:16 +03:00
|
|
|
#ifdef CONFIG_ARCH_HAS_COPY_MC
|
|
|
|
static int copyout_mc(void __user *to, const void *from, size_t n)
|
2018-05-04 03:06:31 +03:00
|
|
|
{
|
Remove 'type' argument from access_ok() function
Nobody has actually used the type (VERIFY_READ vs VERIFY_WRITE) argument
of the user address range verification function since we got rid of the
old racy i386-only code to walk page tables by hand.
It existed because the original 80386 would not honor the write protect
bit when in kernel mode, so you had to do COW by hand before doing any
user access. But we haven't supported that in a long time, and these
days the 'type' argument is a purely historical artifact.
A discussion about extending 'user_access_begin()' to do the range
checking resulted this patch, because there is no way we're going to
move the old VERIFY_xyz interface to that model. And it's best done at
the end of the merge window when I've done most of my merges, so let's
just get this done once and for all.
This patch was mostly done with a sed-script, with manual fix-ups for
the cases that weren't of the trivial 'access_ok(VERIFY_xyz' form.
There were a couple of notable cases:
- csky still had the old "verify_area()" name as an alias.
- the iter_iov code had magical hardcoded knowledge of the actual
values of VERIFY_{READ,WRITE} (not that they mattered, since nothing
really used it)
- microblaze used the type argument for a debug printout
but other than those oddities this should be a total no-op patch.
I tried to fix up all architectures, did fairly extensive grepping for
access_ok() uses, and the changes are trivial, but I may have missed
something. Any missed conversion should be trivially fixable, though.
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2019-01-04 05:57:57 +03:00
|
|
|
if (access_ok(to, n)) {
|
2020-01-21 19:05:11 +03:00
|
|
|
instrument_copy_to_user(to, from, n);
|
x86, powerpc: Rename memcpy_mcsafe() to copy_mc_to_{user, kernel}()
In reaction to a proposal to introduce a memcpy_mcsafe_fast()
implementation Linus points out that memcpy_mcsafe() is poorly named
relative to communicating the scope of the interface. Specifically what
addresses are valid to pass as source, destination, and what faults /
exceptions are handled.
Of particular concern is that even though x86 might be able to handle
the semantics of copy_mc_to_user() with its common copy_user_generic()
implementation other archs likely need / want an explicit path for this
case:
On Fri, May 1, 2020 at 11:28 AM Linus Torvalds <torvalds@linux-foundation.org> wrote:
>
> On Thu, Apr 30, 2020 at 6:21 PM Dan Williams <dan.j.williams@intel.com> wrote:
> >
> > However now I see that copy_user_generic() works for the wrong reason.
> > It works because the exception on the source address due to poison
> > looks no different than a write fault on the user address to the
> > caller, it's still just a short copy. So it makes copy_to_user() work
> > for the wrong reason relative to the name.
>
> Right.
>
> And it won't work that way on other architectures. On x86, we have a
> generic function that can take faults on either side, and we use it
> for both cases (and for the "in_user" case too), but that's an
> artifact of the architecture oddity.
>
> In fact, it's probably wrong even on x86 - because it can hide bugs -
> but writing those things is painful enough that everybody prefers
> having just one function.
Replace a single top-level memcpy_mcsafe() with either
copy_mc_to_user(), or copy_mc_to_kernel().
Introduce an x86 copy_mc_fragile() name as the rename for the
low-level x86 implementation formerly named memcpy_mcsafe(). It is used
as the slow / careful backend that is supplanted by a fast
copy_mc_generic() in a follow-on patch.
One side-effect of this reorganization is that separating copy_mc_64.S
to its own file means that perf no longer needs to track dependencies
for its memcpy_64.S benchmarks.
[ bp: Massage a bit. ]
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
Signed-off-by: Borislav Petkov <bp@suse.de>
Reviewed-by: Tony Luck <tony.luck@intel.com>
Acked-by: Michael Ellerman <mpe@ellerman.id.au>
Cc: <stable@vger.kernel.org>
Link: http://lore.kernel.org/r/CAHk-=wjSqtXAqfUJxFtWNwmguFASTgB0dz1dT3V-78Quiezqbg@mail.gmail.com
Link: https://lkml.kernel.org/r/160195561680.2163339.11574962055305783722.stgit@dwillia2-desk3.amr.corp.intel.com
2020-10-06 06:40:16 +03:00
|
|
|
n = copy_mc_to_user((__force void *) to, from, n);
|
2018-05-04 03:06:31 +03:00
|
|
|
}
|
|
|
|
return n;
|
|
|
|
}
|
|
|
|
|
2018-07-08 23:46:02 +03:00
|
|
|
/**
|
x86, powerpc: Rename memcpy_mcsafe() to copy_mc_to_{user, kernel}()
In reaction to a proposal to introduce a memcpy_mcsafe_fast()
implementation Linus points out that memcpy_mcsafe() is poorly named
relative to communicating the scope of the interface. Specifically what
addresses are valid to pass as source, destination, and what faults /
exceptions are handled.
Of particular concern is that even though x86 might be able to handle
the semantics of copy_mc_to_user() with its common copy_user_generic()
implementation other archs likely need / want an explicit path for this
case:
On Fri, May 1, 2020 at 11:28 AM Linus Torvalds <torvalds@linux-foundation.org> wrote:
>
> On Thu, Apr 30, 2020 at 6:21 PM Dan Williams <dan.j.williams@intel.com> wrote:
> >
> > However now I see that copy_user_generic() works for the wrong reason.
> > It works because the exception on the source address due to poison
> > looks no different than a write fault on the user address to the
> > caller, it's still just a short copy. So it makes copy_to_user() work
> > for the wrong reason relative to the name.
>
> Right.
>
> And it won't work that way on other architectures. On x86, we have a
> generic function that can take faults on either side, and we use it
> for both cases (and for the "in_user" case too), but that's an
> artifact of the architecture oddity.
>
> In fact, it's probably wrong even on x86 - because it can hide bugs -
> but writing those things is painful enough that everybody prefers
> having just one function.
Replace a single top-level memcpy_mcsafe() with either
copy_mc_to_user(), or copy_mc_to_kernel().
Introduce an x86 copy_mc_fragile() name as the rename for the
low-level x86 implementation formerly named memcpy_mcsafe(). It is used
as the slow / careful backend that is supplanted by a fast
copy_mc_generic() in a follow-on patch.
One side-effect of this reorganization is that separating copy_mc_64.S
to its own file means that perf no longer needs to track dependencies
for its memcpy_64.S benchmarks.
[ bp: Massage a bit. ]
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
Signed-off-by: Borislav Petkov <bp@suse.de>
Reviewed-by: Tony Luck <tony.luck@intel.com>
Acked-by: Michael Ellerman <mpe@ellerman.id.au>
Cc: <stable@vger.kernel.org>
Link: http://lore.kernel.org/r/CAHk-=wjSqtXAqfUJxFtWNwmguFASTgB0dz1dT3V-78Quiezqbg@mail.gmail.com
Link: https://lkml.kernel.org/r/160195561680.2163339.11574962055305783722.stgit@dwillia2-desk3.amr.corp.intel.com
2020-10-06 06:40:16 +03:00
|
|
|
* _copy_mc_to_iter - copy to iter with source memory error exception handling
|
2018-07-08 23:46:02 +03:00
|
|
|
* @addr: source kernel address
|
|
|
|
* @bytes: total transfer length
|
2021-09-08 05:58:54 +03:00
|
|
|
* @i: destination iterator
|
2018-07-08 23:46:02 +03:00
|
|
|
*
|
x86, powerpc: Rename memcpy_mcsafe() to copy_mc_to_{user, kernel}()
In reaction to a proposal to introduce a memcpy_mcsafe_fast()
implementation Linus points out that memcpy_mcsafe() is poorly named
relative to communicating the scope of the interface. Specifically what
addresses are valid to pass as source, destination, and what faults /
exceptions are handled.
Of particular concern is that even though x86 might be able to handle
the semantics of copy_mc_to_user() with its common copy_user_generic()
implementation other archs likely need / want an explicit path for this
case:
On Fri, May 1, 2020 at 11:28 AM Linus Torvalds <torvalds@linux-foundation.org> wrote:
>
> On Thu, Apr 30, 2020 at 6:21 PM Dan Williams <dan.j.williams@intel.com> wrote:
> >
> > However now I see that copy_user_generic() works for the wrong reason.
> > It works because the exception on the source address due to poison
> > looks no different than a write fault on the user address to the
> > caller, it's still just a short copy. So it makes copy_to_user() work
> > for the wrong reason relative to the name.
>
> Right.
>
> And it won't work that way on other architectures. On x86, we have a
> generic function that can take faults on either side, and we use it
> for both cases (and for the "in_user" case too), but that's an
> artifact of the architecture oddity.
>
> In fact, it's probably wrong even on x86 - because it can hide bugs -
> but writing those things is painful enough that everybody prefers
> having just one function.
Replace a single top-level memcpy_mcsafe() with either
copy_mc_to_user(), or copy_mc_to_kernel().
Introduce an x86 copy_mc_fragile() name as the rename for the
low-level x86 implementation formerly named memcpy_mcsafe(). It is used
as the slow / careful backend that is supplanted by a fast
copy_mc_generic() in a follow-on patch.
One side-effect of this reorganization is that separating copy_mc_64.S
to its own file means that perf no longer needs to track dependencies
for its memcpy_64.S benchmarks.
[ bp: Massage a bit. ]
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
Signed-off-by: Borislav Petkov <bp@suse.de>
Reviewed-by: Tony Luck <tony.luck@intel.com>
Acked-by: Michael Ellerman <mpe@ellerman.id.au>
Cc: <stable@vger.kernel.org>
Link: http://lore.kernel.org/r/CAHk-=wjSqtXAqfUJxFtWNwmguFASTgB0dz1dT3V-78Quiezqbg@mail.gmail.com
Link: https://lkml.kernel.org/r/160195561680.2163339.11574962055305783722.stgit@dwillia2-desk3.amr.corp.intel.com
2020-10-06 06:40:16 +03:00
|
|
|
* The pmem driver deploys this for the dax operation
|
|
|
|
* (dax_copy_to_iter()) for dax reads (bypass page-cache and the
|
|
|
|
* block-layer). Upon #MC read(2) aborts and returns EIO or the bytes
|
|
|
|
* successfully copied.
|
2018-07-08 23:46:02 +03:00
|
|
|
*
|
x86, powerpc: Rename memcpy_mcsafe() to copy_mc_to_{user, kernel}()
In reaction to a proposal to introduce a memcpy_mcsafe_fast()
implementation Linus points out that memcpy_mcsafe() is poorly named
relative to communicating the scope of the interface. Specifically what
addresses are valid to pass as source, destination, and what faults /
exceptions are handled.
Of particular concern is that even though x86 might be able to handle
the semantics of copy_mc_to_user() with its common copy_user_generic()
implementation other archs likely need / want an explicit path for this
case:
On Fri, May 1, 2020 at 11:28 AM Linus Torvalds <torvalds@linux-foundation.org> wrote:
>
> On Thu, Apr 30, 2020 at 6:21 PM Dan Williams <dan.j.williams@intel.com> wrote:
> >
> > However now I see that copy_user_generic() works for the wrong reason.
> > It works because the exception on the source address due to poison
> > looks no different than a write fault on the user address to the
> > caller, it's still just a short copy. So it makes copy_to_user() work
> > for the wrong reason relative to the name.
>
> Right.
>
> And it won't work that way on other architectures. On x86, we have a
> generic function that can take faults on either side, and we use it
> for both cases (and for the "in_user" case too), but that's an
> artifact of the architecture oddity.
>
> In fact, it's probably wrong even on x86 - because it can hide bugs -
> but writing those things is painful enough that everybody prefers
> having just one function.
Replace a single top-level memcpy_mcsafe() with either
copy_mc_to_user(), or copy_mc_to_kernel().
Introduce an x86 copy_mc_fragile() name as the rename for the
low-level x86 implementation formerly named memcpy_mcsafe(). It is used
as the slow / careful backend that is supplanted by a fast
copy_mc_generic() in a follow-on patch.
One side-effect of this reorganization is that separating copy_mc_64.S
to its own file means that perf no longer needs to track dependencies
for its memcpy_64.S benchmarks.
[ bp: Massage a bit. ]
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
Signed-off-by: Borislav Petkov <bp@suse.de>
Reviewed-by: Tony Luck <tony.luck@intel.com>
Acked-by: Michael Ellerman <mpe@ellerman.id.au>
Cc: <stable@vger.kernel.org>
Link: http://lore.kernel.org/r/CAHk-=wjSqtXAqfUJxFtWNwmguFASTgB0dz1dT3V-78Quiezqbg@mail.gmail.com
Link: https://lkml.kernel.org/r/160195561680.2163339.11574962055305783722.stgit@dwillia2-desk3.amr.corp.intel.com
2020-10-06 06:40:16 +03:00
|
|
|
* The main differences between this and typical _copy_to_iter().
|
2018-07-08 23:46:02 +03:00
|
|
|
*
|
|
|
|
* * Typical tail/residue handling after a fault retries the copy
|
|
|
|
* byte-by-byte until the fault happens again. Re-triggering machine
|
|
|
|
* checks is potentially fatal so the implementation uses source
|
|
|
|
* alignment and poison alignment assumptions to avoid re-triggering
|
|
|
|
* hardware exceptions.
|
|
|
|
*
|
2023-05-22 16:50:17 +03:00
|
|
|
* * ITER_KVEC and ITER_BVEC can return short copies. Compare to
|
|
|
|
* copy_to_iter() where only ITER_IOVEC attempts might return a short copy.
|
2021-09-08 05:58:54 +03:00
|
|
|
*
|
|
|
|
* Return: number of bytes copied (may be %0)
|
2018-07-08 23:46:02 +03:00
|
|
|
*/
|
x86, powerpc: Rename memcpy_mcsafe() to copy_mc_to_{user, kernel}()
In reaction to a proposal to introduce a memcpy_mcsafe_fast()
implementation Linus points out that memcpy_mcsafe() is poorly named
relative to communicating the scope of the interface. Specifically what
addresses are valid to pass as source, destination, and what faults /
exceptions are handled.
Of particular concern is that even though x86 might be able to handle
the semantics of copy_mc_to_user() with its common copy_user_generic()
implementation other archs likely need / want an explicit path for this
case:
On Fri, May 1, 2020 at 11:28 AM Linus Torvalds <torvalds@linux-foundation.org> wrote:
>
> On Thu, Apr 30, 2020 at 6:21 PM Dan Williams <dan.j.williams@intel.com> wrote:
> >
> > However now I see that copy_user_generic() works for the wrong reason.
> > It works because the exception on the source address due to poison
> > looks no different than a write fault on the user address to the
> > caller, it's still just a short copy. So it makes copy_to_user() work
> > for the wrong reason relative to the name.
>
> Right.
>
> And it won't work that way on other architectures. On x86, we have a
> generic function that can take faults on either side, and we use it
> for both cases (and for the "in_user" case too), but that's an
> artifact of the architecture oddity.
>
> In fact, it's probably wrong even on x86 - because it can hide bugs -
> but writing those things is painful enough that everybody prefers
> having just one function.
Replace a single top-level memcpy_mcsafe() with either
copy_mc_to_user(), or copy_mc_to_kernel().
Introduce an x86 copy_mc_fragile() name as the rename for the
low-level x86 implementation formerly named memcpy_mcsafe(). It is used
as the slow / careful backend that is supplanted by a fast
copy_mc_generic() in a follow-on patch.
One side-effect of this reorganization is that separating copy_mc_64.S
to its own file means that perf no longer needs to track dependencies
for its memcpy_64.S benchmarks.
[ bp: Massage a bit. ]
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
Signed-off-by: Borislav Petkov <bp@suse.de>
Reviewed-by: Tony Luck <tony.luck@intel.com>
Acked-by: Michael Ellerman <mpe@ellerman.id.au>
Cc: <stable@vger.kernel.org>
Link: http://lore.kernel.org/r/CAHk-=wjSqtXAqfUJxFtWNwmguFASTgB0dz1dT3V-78Quiezqbg@mail.gmail.com
Link: https://lkml.kernel.org/r/160195561680.2163339.11574962055305783722.stgit@dwillia2-desk3.amr.corp.intel.com
2020-10-06 06:40:16 +03:00
|
|
|
size_t _copy_mc_to_iter(const void *addr, size_t bytes, struct iov_iter *i)
|
2018-05-04 03:06:31 +03:00
|
|
|
{
|
2022-09-16 03:11:15 +03:00
|
|
|
if (WARN_ON_ONCE(i->data_source))
|
|
|
|
return 0;
|
2022-05-22 21:59:25 +03:00
|
|
|
if (user_backed_iter(i))
|
2018-05-04 03:06:31 +03:00
|
|
|
might_fault();
|
2021-05-02 18:35:03 +03:00
|
|
|
__iterate_and_advance(i, bytes, base, len, off,
|
|
|
|
copyout_mc(base, addr + off, len),
|
|
|
|
copy_mc_to_kernel(base, addr + off, len)
|
2018-05-04 03:06:31 +03:00
|
|
|
)
|
|
|
|
|
|
|
|
return bytes;
|
|
|
|
}
|
x86, powerpc: Rename memcpy_mcsafe() to copy_mc_to_{user, kernel}()
In reaction to a proposal to introduce a memcpy_mcsafe_fast()
implementation Linus points out that memcpy_mcsafe() is poorly named
relative to communicating the scope of the interface. Specifically what
addresses are valid to pass as source, destination, and what faults /
exceptions are handled.
Of particular concern is that even though x86 might be able to handle
the semantics of copy_mc_to_user() with its common copy_user_generic()
implementation other archs likely need / want an explicit path for this
case:
On Fri, May 1, 2020 at 11:28 AM Linus Torvalds <torvalds@linux-foundation.org> wrote:
>
> On Thu, Apr 30, 2020 at 6:21 PM Dan Williams <dan.j.williams@intel.com> wrote:
> >
> > However now I see that copy_user_generic() works for the wrong reason.
> > It works because the exception on the source address due to poison
> > looks no different than a write fault on the user address to the
> > caller, it's still just a short copy. So it makes copy_to_user() work
> > for the wrong reason relative to the name.
>
> Right.
>
> And it won't work that way on other architectures. On x86, we have a
> generic function that can take faults on either side, and we use it
> for both cases (and for the "in_user" case too), but that's an
> artifact of the architecture oddity.
>
> In fact, it's probably wrong even on x86 - because it can hide bugs -
> but writing those things is painful enough that everybody prefers
> having just one function.
Replace a single top-level memcpy_mcsafe() with either
copy_mc_to_user(), or copy_mc_to_kernel().
Introduce an x86 copy_mc_fragile() name as the rename for the
low-level x86 implementation formerly named memcpy_mcsafe(). It is used
as the slow / careful backend that is supplanted by a fast
copy_mc_generic() in a follow-on patch.
One side-effect of this reorganization is that separating copy_mc_64.S
to its own file means that perf no longer needs to track dependencies
for its memcpy_64.S benchmarks.
[ bp: Massage a bit. ]
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
Signed-off-by: Borislav Petkov <bp@suse.de>
Reviewed-by: Tony Luck <tony.luck@intel.com>
Acked-by: Michael Ellerman <mpe@ellerman.id.au>
Cc: <stable@vger.kernel.org>
Link: http://lore.kernel.org/r/CAHk-=wjSqtXAqfUJxFtWNwmguFASTgB0dz1dT3V-78Quiezqbg@mail.gmail.com
Link: https://lkml.kernel.org/r/160195561680.2163339.11574962055305783722.stgit@dwillia2-desk3.amr.corp.intel.com
2020-10-06 06:40:16 +03:00
|
|
|
EXPORT_SYMBOL_GPL(_copy_mc_to_iter);
|
|
|
|
#endif /* CONFIG_ARCH_HAS_COPY_MC */
|
2018-05-04 03:06:31 +03:00
|
|
|
|
mm: hwpoison: coredump: support recovery from dump_user_range()
dump_user_range() is used to copy the user page to a coredump file, but if
a hardware memory error occurred during copy, which called from
__kernel_write_iter() in dump_user_range(), it crashes,
CPU: 112 PID: 7014 Comm: mca-recover Not tainted 6.3.0-rc2 #425
pc : __memcpy+0x110/0x260
lr : _copy_from_iter+0x3bc/0x4c8
...
Call trace:
__memcpy+0x110/0x260
copy_page_from_iter+0xcc/0x130
pipe_write+0x164/0x6d8
__kernel_write_iter+0x9c/0x210
dump_user_range+0xc8/0x1d8
elf_core_dump+0x308/0x368
do_coredump+0x2e8/0xa40
get_signal+0x59c/0x788
do_signal+0x118/0x1f8
do_notify_resume+0xf0/0x280
el0_da+0x130/0x138
el0t_64_sync_handler+0x68/0xc0
el0t_64_sync+0x188/0x190
Generally, the '->write_iter' of file ops will use copy_page_from_iter()
and copy_page_from_iter_atomic(), change memcpy() to copy_mc_to_kernel()
in both of them to handle #MC during source read, which stop coredump
processing and kill the task instead of kernel panic, but the source
address may not always a user address, so introduce a new copy_mc flag in
struct iov_iter{} to indicate that the iter could do a safe memory copy,
also introduce the helpers to set/cleck the flag, for now, it's only used
in coredump's dump_user_range(), but it could expand to any other
scenarios to fix the similar issue.
Link: https://lkml.kernel.org/r/20230417045323.11054-1-wangkefeng.wang@huawei.com
Signed-off-by: Kefeng Wang <wangkefeng.wang@huawei.com>
Cc: Alexander Viro <viro@zeniv.linux.org.uk>
Cc: Christian Brauner <brauner@kernel.org>
Cc: Miaohe Lin <linmiaohe@huawei.com>
Cc: Naoya Horiguchi <naoya.horiguchi@nec.com>
Cc: Tong Tiangen <tongtiangen@huawei.com>
Cc: Jens Axboe <axboe@kernel.dk>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
2023-04-17 07:53:23 +03:00
|
|
|
static void *memcpy_from_iter(struct iov_iter *i, void *to, const void *from,
|
|
|
|
size_t size)
|
|
|
|
{
|
|
|
|
if (iov_iter_is_copy_mc(i))
|
|
|
|
return (void *)copy_mc_to_kernel(to, from, size);
|
|
|
|
return memcpy(to, from, size);
|
|
|
|
}
|
|
|
|
|
2017-06-30 04:45:10 +03:00
|
|
|
size_t _copy_from_iter(void *addr, size_t bytes, struct iov_iter *i)
|
2014-08-01 17:27:22 +04:00
|
|
|
{
|
2022-09-16 03:11:15 +03:00
|
|
|
if (WARN_ON_ONCE(!i->data_source))
|
2016-09-22 23:33:12 +03:00
|
|
|
return 0;
|
2022-09-16 03:11:15 +03:00
|
|
|
|
2022-05-22 21:59:25 +03:00
|
|
|
if (user_backed_iter(i))
|
2017-06-30 05:25:14 +03:00
|
|
|
might_fault();
|
2021-05-02 18:35:03 +03:00
|
|
|
iterate_and_advance(i, bytes, base, len, off,
|
|
|
|
copyin(addr + off, base, len),
|
mm: hwpoison: coredump: support recovery from dump_user_range()
dump_user_range() is used to copy the user page to a coredump file, but if
a hardware memory error occurred during copy, which called from
__kernel_write_iter() in dump_user_range(), it crashes,
CPU: 112 PID: 7014 Comm: mca-recover Not tainted 6.3.0-rc2 #425
pc : __memcpy+0x110/0x260
lr : _copy_from_iter+0x3bc/0x4c8
...
Call trace:
__memcpy+0x110/0x260
copy_page_from_iter+0xcc/0x130
pipe_write+0x164/0x6d8
__kernel_write_iter+0x9c/0x210
dump_user_range+0xc8/0x1d8
elf_core_dump+0x308/0x368
do_coredump+0x2e8/0xa40
get_signal+0x59c/0x788
do_signal+0x118/0x1f8
do_notify_resume+0xf0/0x280
el0_da+0x130/0x138
el0t_64_sync_handler+0x68/0xc0
el0t_64_sync+0x188/0x190
Generally, the '->write_iter' of file ops will use copy_page_from_iter()
and copy_page_from_iter_atomic(), change memcpy() to copy_mc_to_kernel()
in both of them to handle #MC during source read, which stop coredump
processing and kill the task instead of kernel panic, but the source
address may not always a user address, so introduce a new copy_mc flag in
struct iov_iter{} to indicate that the iter could do a safe memory copy,
also introduce the helpers to set/cleck the flag, for now, it's only used
in coredump's dump_user_range(), but it could expand to any other
scenarios to fix the similar issue.
Link: https://lkml.kernel.org/r/20230417045323.11054-1-wangkefeng.wang@huawei.com
Signed-off-by: Kefeng Wang <wangkefeng.wang@huawei.com>
Cc: Alexander Viro <viro@zeniv.linux.org.uk>
Cc: Christian Brauner <brauner@kernel.org>
Cc: Miaohe Lin <linmiaohe@huawei.com>
Cc: Naoya Horiguchi <naoya.horiguchi@nec.com>
Cc: Tong Tiangen <tongtiangen@huawei.com>
Cc: Jens Axboe <axboe@kernel.dk>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
2023-04-17 07:53:23 +03:00
|
|
|
memcpy_from_iter(i, addr + off, base, len)
|
2014-11-27 22:26:43 +03:00
|
|
|
)
|
|
|
|
|
|
|
|
return bytes;
|
2014-08-01 17:27:22 +04:00
|
|
|
}
|
2017-06-30 04:45:10 +03:00
|
|
|
EXPORT_SYMBOL(_copy_from_iter);
|
2014-08-01 17:27:22 +04:00
|
|
|
|
2017-06-30 04:45:10 +03:00
|
|
|
size_t _copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i)
|
2014-11-28 04:27:08 +03:00
|
|
|
{
|
2022-09-16 03:11:15 +03:00
|
|
|
if (WARN_ON_ONCE(!i->data_source))
|
2016-09-22 23:33:12 +03:00
|
|
|
return 0;
|
2022-09-16 03:11:15 +03:00
|
|
|
|
2021-05-02 18:35:03 +03:00
|
|
|
iterate_and_advance(i, bytes, base, len, off,
|
|
|
|
__copy_from_user_inatomic_nocache(addr + off, base, len),
|
|
|
|
memcpy(addr + off, base, len)
|
2014-11-28 04:27:08 +03:00
|
|
|
)
|
|
|
|
|
|
|
|
return bytes;
|
|
|
|
}
|
2017-06-30 04:45:10 +03:00
|
|
|
EXPORT_SYMBOL(_copy_from_iter_nocache);
|
2014-11-28 04:27:08 +03:00
|
|
|
|
2017-05-29 22:22:50 +03:00
|
|
|
#ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE
|
2018-07-08 23:46:07 +03:00
|
|
|
/**
|
|
|
|
* _copy_from_iter_flushcache - write destination through cpu cache
|
|
|
|
* @addr: destination kernel address
|
|
|
|
* @bytes: total transfer length
|
2021-09-08 05:58:54 +03:00
|
|
|
* @i: source iterator
|
2018-07-08 23:46:07 +03:00
|
|
|
*
|
|
|
|
* The pmem driver arranges for filesystem-dax to use this facility via
|
|
|
|
* dax_copy_from_iter() for ensuring that writes to persistent memory
|
|
|
|
* are flushed through the CPU cache. It is differentiated from
|
|
|
|
* _copy_from_iter_nocache() in that guarantees all data is flushed for
|
|
|
|
* all iterator types. The _copy_from_iter_nocache() only attempts to
|
|
|
|
* bypass the cache for the ITER_IOVEC case, and on some archs may use
|
|
|
|
* instructions that strand dirty-data in the cache.
|
2021-09-08 05:58:54 +03:00
|
|
|
*
|
|
|
|
* Return: number of bytes copied (may be %0)
|
2018-07-08 23:46:07 +03:00
|
|
|
*/
|
Merge branch 'uaccess-work.iov_iter' of git://git.kernel.org/pub/scm/linux/kernel/git/viro/vfs
Pull iov_iter hardening from Al Viro:
"This is the iov_iter/uaccess/hardening pile.
For one thing, it trims the inline part of copy_to_user/copy_from_user
to the minimum that *does* need to be inlined - object size checks,
basically. For another, it sanitizes the checks for iov_iter
primitives. There are 4 groups of checks: access_ok(), might_fault(),
object size and KASAN.
- access_ok() had been verified by whoever had set the iov_iter up.
However, that has happened in a function far away, so proving that
there's no path to actual copying bypassing those checks is hard
and proving that iov_iter has not been buggered in the meanwhile is
also not pleasant. So we want those redone in actual
copyin/copyout.
- might_fault() is better off consolidated - we know whether it needs
to be checked as soon as we enter iov_iter primitive and observe
the iov_iter flavour. No need to wait until the copyin/copyout. The
call chains are short enough to make sure we won't miss anything -
in fact, it's more robust that way, since there are cases where we
do e.g. forced fault-in before getting to copyin/copyout. It's not
quite what we need to check (in particular, combination of
iovec-backed and set_fs(KERNEL_DS) is almost certainly a bug, not a
cause to skip checks), but that's for later series. For now let's
keep might_fault().
- KASAN checks belong in copyin/copyout - at the same level where
other iov_iter flavours would've hit them in memcpy().
- object size checks should apply to *all* iov_iter flavours, not
just iovec-backed ones.
There are two groups of primitives - one gets the kernel object
described as pointer + size (copy_to_iter(), etc.) while another gets
it as page + offset + size (copy_page_to_iter(), etc.)
For the first group the checks are best done where we actually have a
chance to find the object size. In other words, those belong in inline
wrappers in uio.h, before calling into iov_iter.c. Same kind as we
have for inlined part of copy_to_user().
For the second group there is no object to look at - offset in page is
just a number, it bears no type information. So we do them in the
common helper called by iov_iter.c primitives of that kind. All it
currently does is checking that we are not trying to access outside of
the compound page; eventually we might want to add some sanity checks
on the page involved.
So the things we need in copyin/copyout part of iov_iter.c do not
quite match anything in uaccess.h (we want no zeroing, we *do* want
access_ok() and KASAN and we want no might_fault() or object size
checks done on that level). OTOH, these needs are simple enough to
provide a couple of helpers (static in iov_iter.c) doing just what we
need..."
* 'uaccess-work.iov_iter' of git://git.kernel.org/pub/scm/linux/kernel/git/viro/vfs:
iov_iter: saner checks on copyin/copyout
iov_iter: sanity checks for copy to/from page primitives
iov_iter/hardening: move object size checks to inlined part
copy_{to,from}_user(): consolidate object size checks
copy_{from,to}_user(): move kasan checks and might_fault() out-of-line
2017-07-08 06:39:20 +03:00
|
|
|
size_t _copy_from_iter_flushcache(void *addr, size_t bytes, struct iov_iter *i)
|
2017-05-29 22:22:50 +03:00
|
|
|
{
|
2022-09-16 03:11:15 +03:00
|
|
|
if (WARN_ON_ONCE(!i->data_source))
|
2017-05-29 22:22:50 +03:00
|
|
|
return 0;
|
2022-09-16 03:11:15 +03:00
|
|
|
|
2021-05-02 18:35:03 +03:00
|
|
|
iterate_and_advance(i, bytes, base, len, off,
|
|
|
|
__copy_from_user_flushcache(addr + off, base, len),
|
|
|
|
memcpy_flushcache(addr + off, base, len)
|
2017-05-29 22:22:50 +03:00
|
|
|
)
|
|
|
|
|
|
|
|
return bytes;
|
|
|
|
}
|
Merge branch 'uaccess-work.iov_iter' of git://git.kernel.org/pub/scm/linux/kernel/git/viro/vfs
Pull iov_iter hardening from Al Viro:
"This is the iov_iter/uaccess/hardening pile.
For one thing, it trims the inline part of copy_to_user/copy_from_user
to the minimum that *does* need to be inlined - object size checks,
basically. For another, it sanitizes the checks for iov_iter
primitives. There are 4 groups of checks: access_ok(), might_fault(),
object size and KASAN.
- access_ok() had been verified by whoever had set the iov_iter up.
However, that has happened in a function far away, so proving that
there's no path to actual copying bypassing those checks is hard
and proving that iov_iter has not been buggered in the meanwhile is
also not pleasant. So we want those redone in actual
copyin/copyout.
- might_fault() is better off consolidated - we know whether it needs
to be checked as soon as we enter iov_iter primitive and observe
the iov_iter flavour. No need to wait until the copyin/copyout. The
call chains are short enough to make sure we won't miss anything -
in fact, it's more robust that way, since there are cases where we
do e.g. forced fault-in before getting to copyin/copyout. It's not
quite what we need to check (in particular, combination of
iovec-backed and set_fs(KERNEL_DS) is almost certainly a bug, not a
cause to skip checks), but that's for later series. For now let's
keep might_fault().
- KASAN checks belong in copyin/copyout - at the same level where
other iov_iter flavours would've hit them in memcpy().
- object size checks should apply to *all* iov_iter flavours, not
just iovec-backed ones.
There are two groups of primitives - one gets the kernel object
described as pointer + size (copy_to_iter(), etc.) while another gets
it as page + offset + size (copy_page_to_iter(), etc.)
For the first group the checks are best done where we actually have a
chance to find the object size. In other words, those belong in inline
wrappers in uio.h, before calling into iov_iter.c. Same kind as we
have for inlined part of copy_to_user().
For the second group there is no object to look at - offset in page is
just a number, it bears no type information. So we do them in the
common helper called by iov_iter.c primitives of that kind. All it
currently does is checking that we are not trying to access outside of
the compound page; eventually we might want to add some sanity checks
on the page involved.
So the things we need in copyin/copyout part of iov_iter.c do not
quite match anything in uaccess.h (we want no zeroing, we *do* want
access_ok() and KASAN and we want no might_fault() or object size
checks done on that level). OTOH, these needs are simple enough to
provide a couple of helpers (static in iov_iter.c) doing just what we
need..."
* 'uaccess-work.iov_iter' of git://git.kernel.org/pub/scm/linux/kernel/git/viro/vfs:
iov_iter: saner checks on copyin/copyout
iov_iter: sanity checks for copy to/from page primitives
iov_iter/hardening: move object size checks to inlined part
copy_{to,from}_user(): consolidate object size checks
copy_{from,to}_user(): move kasan checks and might_fault() out-of-line
2017-07-08 06:39:20 +03:00
|
|
|
EXPORT_SYMBOL_GPL(_copy_from_iter_flushcache);
|
2017-05-29 22:22:50 +03:00
|
|
|
#endif
|
|
|
|
|
2017-06-30 04:52:57 +03:00
|
|
|
static inline bool page_copy_sane(struct page *page, size_t offset, size_t n)
|
|
|
|
{
|
2019-02-26 21:42:39 +03:00
|
|
|
struct page *head;
|
|
|
|
size_t v = n + offset;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* The general case needs to access the page order in order
|
|
|
|
* to compute the page size.
|
|
|
|
* However, we mostly deal with order-0 pages and thus can
|
|
|
|
* avoid a possible cache line miss for requests that fit all
|
|
|
|
* page orders.
|
|
|
|
*/
|
|
|
|
if (n <= v && v <= PAGE_SIZE)
|
|
|
|
return true;
|
|
|
|
|
|
|
|
head = compound_head(page);
|
|
|
|
v += (page - head) << PAGE_SHIFT;
|
2017-08-29 21:20:32 +03:00
|
|
|
|
2022-07-29 20:01:57 +03:00
|
|
|
if (WARN_ON(n > v || v > page_size(head)))
|
|
|
|
return false;
|
|
|
|
return true;
|
2017-06-30 04:52:57 +03:00
|
|
|
}
|
2016-11-02 05:09:04 +03:00
|
|
|
|
2021-04-30 03:42:25 +03:00
|
|
|
size_t copy_page_to_iter(struct page *page, size_t offset, size_t bytes,
|
|
|
|
struct iov_iter *i)
|
|
|
|
{
|
|
|
|
size_t res = 0;
|
2022-07-29 20:01:57 +03:00
|
|
|
if (!page_copy_sane(page, offset, bytes))
|
2021-04-30 03:42:25 +03:00
|
|
|
return 0;
|
2022-09-16 03:11:15 +03:00
|
|
|
if (WARN_ON_ONCE(i->data_source))
|
|
|
|
return 0;
|
2021-04-30 03:42:25 +03:00
|
|
|
page += offset / PAGE_SIZE; // first subpage
|
|
|
|
offset %= PAGE_SIZE;
|
|
|
|
while (1) {
|
2022-06-24 00:21:37 +03:00
|
|
|
void *kaddr = kmap_local_page(page);
|
|
|
|
size_t n = min(bytes, (size_t)PAGE_SIZE - offset);
|
|
|
|
n = _copy_to_iter(kaddr + offset, n, i);
|
|
|
|
kunmap_local(kaddr);
|
2021-04-30 03:42:25 +03:00
|
|
|
res += n;
|
|
|
|
bytes -= n;
|
|
|
|
if (!bytes || !n)
|
|
|
|
break;
|
|
|
|
offset += n;
|
|
|
|
if (offset == PAGE_SIZE) {
|
|
|
|
page++;
|
|
|
|
offset = 0;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return res;
|
|
|
|
}
|
2014-04-05 07:12:29 +04:00
|
|
|
EXPORT_SYMBOL(copy_page_to_iter);
|
|
|
|
|
2023-03-22 21:57:03 +03:00
|
|
|
size_t copy_page_to_iter_nofault(struct page *page, unsigned offset, size_t bytes,
|
|
|
|
struct iov_iter *i)
|
|
|
|
{
|
|
|
|
size_t res = 0;
|
|
|
|
|
|
|
|
if (!page_copy_sane(page, offset, bytes))
|
|
|
|
return 0;
|
|
|
|
if (WARN_ON_ONCE(i->data_source))
|
|
|
|
return 0;
|
|
|
|
page += offset / PAGE_SIZE; // first subpage
|
|
|
|
offset %= PAGE_SIZE;
|
|
|
|
while (1) {
|
|
|
|
void *kaddr = kmap_local_page(page);
|
|
|
|
size_t n = min(bytes, (size_t)PAGE_SIZE - offset);
|
|
|
|
|
|
|
|
iterate_and_advance(i, n, base, len, off,
|
|
|
|
copyout_nofault(base, kaddr + offset + off, len),
|
|
|
|
memcpy(base, kaddr + offset + off, len)
|
|
|
|
)
|
|
|
|
kunmap_local(kaddr);
|
|
|
|
res += n;
|
|
|
|
bytes -= n;
|
|
|
|
if (!bytes || !n)
|
|
|
|
break;
|
|
|
|
offset += n;
|
|
|
|
if (offset == PAGE_SIZE) {
|
|
|
|
page++;
|
|
|
|
offset = 0;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return res;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(copy_page_to_iter_nofault);
|
|
|
|
|
2014-04-05 07:12:29 +04:00
|
|
|
size_t copy_page_from_iter(struct page *page, size_t offset, size_t bytes,
|
|
|
|
struct iov_iter *i)
|
|
|
|
{
|
2022-07-29 19:54:53 +03:00
|
|
|
size_t res = 0;
|
|
|
|
if (!page_copy_sane(page, offset, bytes))
|
|
|
|
return 0;
|
|
|
|
page += offset / PAGE_SIZE; // first subpage
|
|
|
|
offset %= PAGE_SIZE;
|
|
|
|
while (1) {
|
2021-04-27 19:33:24 +03:00
|
|
|
void *kaddr = kmap_local_page(page);
|
2022-07-29 19:54:53 +03:00
|
|
|
size_t n = min(bytes, (size_t)PAGE_SIZE - offset);
|
|
|
|
n = _copy_from_iter(kaddr + offset, n, i);
|
2021-04-27 19:33:24 +03:00
|
|
|
kunmap_local(kaddr);
|
2022-07-29 19:54:53 +03:00
|
|
|
res += n;
|
|
|
|
bytes -= n;
|
|
|
|
if (!bytes || !n)
|
|
|
|
break;
|
|
|
|
offset += n;
|
|
|
|
if (offset == PAGE_SIZE) {
|
|
|
|
page++;
|
|
|
|
offset = 0;
|
|
|
|
}
|
2021-06-03 00:25:59 +03:00
|
|
|
}
|
2022-07-29 19:54:53 +03:00
|
|
|
return res;
|
2014-04-05 07:12:29 +04:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(copy_page_from_iter);
|
|
|
|
|
2014-08-01 17:27:22 +04:00
|
|
|
size_t iov_iter_zero(size_t bytes, struct iov_iter *i)
|
|
|
|
{
|
2021-05-02 18:35:03 +03:00
|
|
|
iterate_and_advance(i, bytes, base, len, count,
|
|
|
|
clear_user(base, len),
|
|
|
|
memset(base, 0, len)
|
2014-11-27 22:18:54 +03:00
|
|
|
)
|
|
|
|
|
|
|
|
return bytes;
|
2014-08-01 17:27:22 +04:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(iov_iter_zero);
|
|
|
|
|
2023-07-10 01:17:33 +03:00
|
|
|
size_t copy_page_from_iter_atomic(struct page *page, size_t offset,
|
2023-06-06 23:43:32 +03:00
|
|
|
size_t bytes, struct iov_iter *i)
|
2014-04-05 07:12:29 +04:00
|
|
|
{
|
2023-06-06 23:43:32 +03:00
|
|
|
size_t n, copied = 0;
|
2023-07-16 06:03:52 +03:00
|
|
|
|
|
|
|
if (!page_copy_sane(page, offset, bytes))
|
2017-06-30 04:52:57 +03:00
|
|
|
return 0;
|
2023-07-16 06:03:52 +03:00
|
|
|
if (WARN_ON_ONCE(!i->data_source))
|
2016-09-22 23:33:12 +03:00
|
|
|
return 0;
|
2023-07-16 06:03:52 +03:00
|
|
|
|
2023-06-06 23:43:32 +03:00
|
|
|
do {
|
|
|
|
char *p;
|
2023-07-16 06:03:52 +03:00
|
|
|
|
2023-06-06 23:43:32 +03:00
|
|
|
n = bytes - copied;
|
|
|
|
if (PageHighMem(page)) {
|
|
|
|
page += offset / PAGE_SIZE;
|
|
|
|
offset %= PAGE_SIZE;
|
|
|
|
n = min_t(size_t, n, PAGE_SIZE - offset);
|
|
|
|
}
|
|
|
|
|
|
|
|
p = kmap_atomic(page) + offset;
|
|
|
|
iterate_and_advance(i, n, base, len, off,
|
|
|
|
copyin(p + off, base, len),
|
|
|
|
memcpy_from_iter(i, p + off, base, len)
|
|
|
|
)
|
|
|
|
kunmap_atomic(p);
|
|
|
|
copied += n;
|
|
|
|
offset += n;
|
|
|
|
} while (PageHighMem(page) && copied != bytes && n > 0);
|
|
|
|
|
|
|
|
return copied;
|
2014-04-05 07:12:29 +04:00
|
|
|
}
|
2021-04-30 17:26:41 +03:00
|
|
|
EXPORT_SYMBOL(copy_page_from_iter_atomic);
|
2014-04-05 07:12:29 +04:00
|
|
|
|
2021-01-09 19:03:01 +03:00
|
|
|
static void iov_iter_bvec_advance(struct iov_iter *i, size_t size)
|
|
|
|
{
|
2022-06-07 06:44:33 +03:00
|
|
|
const struct bio_vec *bvec, *end;
|
2021-01-09 19:03:01 +03:00
|
|
|
|
2022-06-07 06:44:33 +03:00
|
|
|
if (!i->count)
|
|
|
|
return;
|
|
|
|
i->count -= size;
|
|
|
|
|
|
|
|
size += i->iov_offset;
|
2021-01-09 19:03:01 +03:00
|
|
|
|
2022-06-07 06:44:33 +03:00
|
|
|
for (bvec = i->bvec, end = bvec + i->nr_segs; bvec < end; bvec++) {
|
|
|
|
if (likely(size < bvec->bv_len))
|
|
|
|
break;
|
|
|
|
size -= bvec->bv_len;
|
|
|
|
}
|
|
|
|
i->iov_offset = size;
|
|
|
|
i->nr_segs -= bvec - i->bvec;
|
|
|
|
i->bvec = bvec;
|
2021-01-09 19:03:01 +03:00
|
|
|
}
|
|
|
|
|
2021-04-23 19:58:53 +03:00
|
|
|
static void iov_iter_iovec_advance(struct iov_iter *i, size_t size)
|
|
|
|
{
|
|
|
|
const struct iovec *iov, *end;
|
|
|
|
|
|
|
|
if (!i->count)
|
|
|
|
return;
|
|
|
|
i->count -= size;
|
|
|
|
|
|
|
|
size += i->iov_offset; // from beginning of current segment
|
2023-03-29 17:52:15 +03:00
|
|
|
for (iov = iter_iov(i), end = iov + i->nr_segs; iov < end; iov++) {
|
2021-04-23 19:58:53 +03:00
|
|
|
if (likely(size < iov->iov_len))
|
|
|
|
break;
|
|
|
|
size -= iov->iov_len;
|
|
|
|
}
|
|
|
|
i->iov_offset = size;
|
2023-03-29 17:52:15 +03:00
|
|
|
i->nr_segs -= iov - iter_iov(i);
|
|
|
|
i->__iov = iov;
|
2021-04-23 19:58:53 +03:00
|
|
|
}
|
|
|
|
|
2014-04-05 07:12:29 +04:00
|
|
|
void iov_iter_advance(struct iov_iter *i, size_t size)
|
|
|
|
{
|
2021-04-24 05:24:08 +03:00
|
|
|
if (unlikely(i->count < size))
|
|
|
|
size = i->count;
|
2022-05-22 21:59:25 +03:00
|
|
|
if (likely(iter_is_ubuf(i)) || unlikely(iov_iter_is_xarray(i))) {
|
|
|
|
i->iov_offset += size;
|
|
|
|
i->count -= size;
|
|
|
|
} else if (likely(iter_is_iovec(i) || iov_iter_is_kvec(i))) {
|
2021-04-23 19:58:53 +03:00
|
|
|
/* iovec and kvec have identical layouts */
|
|
|
|
iov_iter_iovec_advance(i, size);
|
|
|
|
} else if (iov_iter_is_bvec(i)) {
|
|
|
|
iov_iter_bvec_advance(i, size);
|
|
|
|
} else if (iov_iter_is_discard(i)) {
|
|
|
|
i->count -= size;
|
2021-01-09 19:03:01 +03:00
|
|
|
}
|
2014-04-05 07:12:29 +04:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(iov_iter_advance);
|
|
|
|
|
2017-02-18 02:42:24 +03:00
|
|
|
void iov_iter_revert(struct iov_iter *i, size_t unroll)
|
|
|
|
{
|
|
|
|
if (!unroll)
|
|
|
|
return;
|
2017-05-08 20:54:47 +03:00
|
|
|
if (WARN_ON(unroll > MAX_RW_COUNT))
|
|
|
|
return;
|
2017-02-18 02:42:24 +03:00
|
|
|
i->count += unroll;
|
2018-10-20 02:57:56 +03:00
|
|
|
if (unlikely(iov_iter_is_discard(i)))
|
|
|
|
return;
|
2017-02-18 02:42:24 +03:00
|
|
|
if (unroll <= i->iov_offset) {
|
|
|
|
i->iov_offset -= unroll;
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
unroll -= i->iov_offset;
|
2022-05-22 21:59:25 +03:00
|
|
|
if (iov_iter_is_xarray(i) || iter_is_ubuf(i)) {
|
2020-02-10 13:00:21 +03:00
|
|
|
BUG(); /* We should never go beyond the start of the specified
|
|
|
|
* range since we might then be straying into pages that
|
|
|
|
* aren't pinned.
|
|
|
|
*/
|
|
|
|
} else if (iov_iter_is_bvec(i)) {
|
2017-02-18 02:42:24 +03:00
|
|
|
const struct bio_vec *bvec = i->bvec;
|
|
|
|
while (1) {
|
|
|
|
size_t n = (--bvec)->bv_len;
|
|
|
|
i->nr_segs++;
|
|
|
|
if (unroll <= n) {
|
|
|
|
i->bvec = bvec;
|
|
|
|
i->iov_offset = n - unroll;
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
unroll -= n;
|
|
|
|
}
|
|
|
|
} else { /* same logics for iovec and kvec */
|
2023-03-29 17:52:15 +03:00
|
|
|
const struct iovec *iov = iter_iov(i);
|
2017-02-18 02:42:24 +03:00
|
|
|
while (1) {
|
|
|
|
size_t n = (--iov)->iov_len;
|
|
|
|
i->nr_segs++;
|
|
|
|
if (unroll <= n) {
|
2023-03-29 17:52:15 +03:00
|
|
|
i->__iov = iov;
|
2017-02-18 02:42:24 +03:00
|
|
|
i->iov_offset = n - unroll;
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
unroll -= n;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(iov_iter_revert);
|
|
|
|
|
2014-04-05 07:12:29 +04:00
|
|
|
/*
|
|
|
|
* Return the count of just the current iov_iter segment.
|
|
|
|
*/
|
|
|
|
size_t iov_iter_single_seg_count(const struct iov_iter *i)
|
|
|
|
{
|
2021-06-03 00:25:59 +03:00
|
|
|
if (i->nr_segs > 1) {
|
|
|
|
if (likely(iter_is_iovec(i) || iov_iter_is_kvec(i)))
|
2023-03-29 17:52:15 +03:00
|
|
|
return min(i->count, iter_iov(i)->iov_len - i->iov_offset);
|
2021-06-03 00:25:59 +03:00
|
|
|
if (iov_iter_is_bvec(i))
|
|
|
|
return min(i->count, i->bvec->bv_len - i->iov_offset);
|
|
|
|
}
|
|
|
|
return i->count;
|
2014-04-05 07:12:29 +04:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(iov_iter_single_seg_count);
|
|
|
|
|
2018-10-20 02:57:56 +03:00
|
|
|
void iov_iter_kvec(struct iov_iter *i, unsigned int direction,
|
2015-01-23 09:08:07 +03:00
|
|
|
const struct kvec *kvec, unsigned long nr_segs,
|
2014-11-24 22:46:11 +03:00
|
|
|
size_t count)
|
|
|
|
{
|
2018-10-20 02:57:56 +03:00
|
|
|
WARN_ON(direction & ~(READ | WRITE));
|
2021-04-22 21:50:39 +03:00
|
|
|
*i = (struct iov_iter){
|
|
|
|
.iter_type = ITER_KVEC,
|
mm: hwpoison: coredump: support recovery from dump_user_range()
dump_user_range() is used to copy the user page to a coredump file, but if
a hardware memory error occurred during copy, which called from
__kernel_write_iter() in dump_user_range(), it crashes,
CPU: 112 PID: 7014 Comm: mca-recover Not tainted 6.3.0-rc2 #425
pc : __memcpy+0x110/0x260
lr : _copy_from_iter+0x3bc/0x4c8
...
Call trace:
__memcpy+0x110/0x260
copy_page_from_iter+0xcc/0x130
pipe_write+0x164/0x6d8
__kernel_write_iter+0x9c/0x210
dump_user_range+0xc8/0x1d8
elf_core_dump+0x308/0x368
do_coredump+0x2e8/0xa40
get_signal+0x59c/0x788
do_signal+0x118/0x1f8
do_notify_resume+0xf0/0x280
el0_da+0x130/0x138
el0t_64_sync_handler+0x68/0xc0
el0t_64_sync+0x188/0x190
Generally, the '->write_iter' of file ops will use copy_page_from_iter()
and copy_page_from_iter_atomic(), change memcpy() to copy_mc_to_kernel()
in both of them to handle #MC during source read, which stop coredump
processing and kill the task instead of kernel panic, but the source
address may not always a user address, so introduce a new copy_mc flag in
struct iov_iter{} to indicate that the iter could do a safe memory copy,
also introduce the helpers to set/cleck the flag, for now, it's only used
in coredump's dump_user_range(), but it could expand to any other
scenarios to fix the similar issue.
Link: https://lkml.kernel.org/r/20230417045323.11054-1-wangkefeng.wang@huawei.com
Signed-off-by: Kefeng Wang <wangkefeng.wang@huawei.com>
Cc: Alexander Viro <viro@zeniv.linux.org.uk>
Cc: Christian Brauner <brauner@kernel.org>
Cc: Miaohe Lin <linmiaohe@huawei.com>
Cc: Naoya Horiguchi <naoya.horiguchi@nec.com>
Cc: Tong Tiangen <tongtiangen@huawei.com>
Cc: Jens Axboe <axboe@kernel.dk>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
2023-04-17 07:53:23 +03:00
|
|
|
.copy_mc = false,
|
2021-04-22 21:50:39 +03:00
|
|
|
.data_source = direction,
|
|
|
|
.kvec = kvec,
|
|
|
|
.nr_segs = nr_segs,
|
|
|
|
.iov_offset = 0,
|
|
|
|
.count = count
|
|
|
|
};
|
2014-11-24 22:46:11 +03:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(iov_iter_kvec);
|
|
|
|
|
2018-10-20 02:57:56 +03:00
|
|
|
void iov_iter_bvec(struct iov_iter *i, unsigned int direction,
|
2015-01-23 09:08:07 +03:00
|
|
|
const struct bio_vec *bvec, unsigned long nr_segs,
|
|
|
|
size_t count)
|
|
|
|
{
|
2018-10-20 02:57:56 +03:00
|
|
|
WARN_ON(direction & ~(READ | WRITE));
|
2021-04-22 21:50:39 +03:00
|
|
|
*i = (struct iov_iter){
|
|
|
|
.iter_type = ITER_BVEC,
|
mm: hwpoison: coredump: support recovery from dump_user_range()
dump_user_range() is used to copy the user page to a coredump file, but if
a hardware memory error occurred during copy, which called from
__kernel_write_iter() in dump_user_range(), it crashes,
CPU: 112 PID: 7014 Comm: mca-recover Not tainted 6.3.0-rc2 #425
pc : __memcpy+0x110/0x260
lr : _copy_from_iter+0x3bc/0x4c8
...
Call trace:
__memcpy+0x110/0x260
copy_page_from_iter+0xcc/0x130
pipe_write+0x164/0x6d8
__kernel_write_iter+0x9c/0x210
dump_user_range+0xc8/0x1d8
elf_core_dump+0x308/0x368
do_coredump+0x2e8/0xa40
get_signal+0x59c/0x788
do_signal+0x118/0x1f8
do_notify_resume+0xf0/0x280
el0_da+0x130/0x138
el0t_64_sync_handler+0x68/0xc0
el0t_64_sync+0x188/0x190
Generally, the '->write_iter' of file ops will use copy_page_from_iter()
and copy_page_from_iter_atomic(), change memcpy() to copy_mc_to_kernel()
in both of them to handle #MC during source read, which stop coredump
processing and kill the task instead of kernel panic, but the source
address may not always a user address, so introduce a new copy_mc flag in
struct iov_iter{} to indicate that the iter could do a safe memory copy,
also introduce the helpers to set/cleck the flag, for now, it's only used
in coredump's dump_user_range(), but it could expand to any other
scenarios to fix the similar issue.
Link: https://lkml.kernel.org/r/20230417045323.11054-1-wangkefeng.wang@huawei.com
Signed-off-by: Kefeng Wang <wangkefeng.wang@huawei.com>
Cc: Alexander Viro <viro@zeniv.linux.org.uk>
Cc: Christian Brauner <brauner@kernel.org>
Cc: Miaohe Lin <linmiaohe@huawei.com>
Cc: Naoya Horiguchi <naoya.horiguchi@nec.com>
Cc: Tong Tiangen <tongtiangen@huawei.com>
Cc: Jens Axboe <axboe@kernel.dk>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
2023-04-17 07:53:23 +03:00
|
|
|
.copy_mc = false,
|
2021-04-22 21:50:39 +03:00
|
|
|
.data_source = direction,
|
|
|
|
.bvec = bvec,
|
|
|
|
.nr_segs = nr_segs,
|
|
|
|
.iov_offset = 0,
|
|
|
|
.count = count
|
|
|
|
};
|
2015-01-23 09:08:07 +03:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(iov_iter_bvec);
|
|
|
|
|
2020-02-10 13:00:21 +03:00
|
|
|
/**
|
|
|
|
* iov_iter_xarray - Initialise an I/O iterator to use the pages in an xarray
|
|
|
|
* @i: The iterator to initialise.
|
|
|
|
* @direction: The direction of the transfer.
|
|
|
|
* @xarray: The xarray to access.
|
|
|
|
* @start: The start file position.
|
|
|
|
* @count: The size of the I/O buffer in bytes.
|
|
|
|
*
|
|
|
|
* Set up an I/O iterator to either draw data out of the pages attached to an
|
|
|
|
* inode or to inject data into those pages. The pages *must* be prevented
|
|
|
|
* from evaporation, either by taking a ref on them or locking them by the
|
|
|
|
* caller.
|
|
|
|
*/
|
|
|
|
void iov_iter_xarray(struct iov_iter *i, unsigned int direction,
|
|
|
|
struct xarray *xarray, loff_t start, size_t count)
|
|
|
|
{
|
|
|
|
BUG_ON(direction & ~1);
|
2021-04-22 21:50:39 +03:00
|
|
|
*i = (struct iov_iter) {
|
|
|
|
.iter_type = ITER_XARRAY,
|
mm: hwpoison: coredump: support recovery from dump_user_range()
dump_user_range() is used to copy the user page to a coredump file, but if
a hardware memory error occurred during copy, which called from
__kernel_write_iter() in dump_user_range(), it crashes,
CPU: 112 PID: 7014 Comm: mca-recover Not tainted 6.3.0-rc2 #425
pc : __memcpy+0x110/0x260
lr : _copy_from_iter+0x3bc/0x4c8
...
Call trace:
__memcpy+0x110/0x260
copy_page_from_iter+0xcc/0x130
pipe_write+0x164/0x6d8
__kernel_write_iter+0x9c/0x210
dump_user_range+0xc8/0x1d8
elf_core_dump+0x308/0x368
do_coredump+0x2e8/0xa40
get_signal+0x59c/0x788
do_signal+0x118/0x1f8
do_notify_resume+0xf0/0x280
el0_da+0x130/0x138
el0t_64_sync_handler+0x68/0xc0
el0t_64_sync+0x188/0x190
Generally, the '->write_iter' of file ops will use copy_page_from_iter()
and copy_page_from_iter_atomic(), change memcpy() to copy_mc_to_kernel()
in both of them to handle #MC during source read, which stop coredump
processing and kill the task instead of kernel panic, but the source
address may not always a user address, so introduce a new copy_mc flag in
struct iov_iter{} to indicate that the iter could do a safe memory copy,
also introduce the helpers to set/cleck the flag, for now, it's only used
in coredump's dump_user_range(), but it could expand to any other
scenarios to fix the similar issue.
Link: https://lkml.kernel.org/r/20230417045323.11054-1-wangkefeng.wang@huawei.com
Signed-off-by: Kefeng Wang <wangkefeng.wang@huawei.com>
Cc: Alexander Viro <viro@zeniv.linux.org.uk>
Cc: Christian Brauner <brauner@kernel.org>
Cc: Miaohe Lin <linmiaohe@huawei.com>
Cc: Naoya Horiguchi <naoya.horiguchi@nec.com>
Cc: Tong Tiangen <tongtiangen@huawei.com>
Cc: Jens Axboe <axboe@kernel.dk>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
2023-04-17 07:53:23 +03:00
|
|
|
.copy_mc = false,
|
2021-04-22 21:50:39 +03:00
|
|
|
.data_source = direction,
|
|
|
|
.xarray = xarray,
|
|
|
|
.xarray_start = start,
|
|
|
|
.count = count,
|
|
|
|
.iov_offset = 0
|
|
|
|
};
|
2020-02-10 13:00:21 +03:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(iov_iter_xarray);
|
|
|
|
|
2018-10-20 02:57:56 +03:00
|
|
|
/**
|
|
|
|
* iov_iter_discard - Initialise an I/O iterator that discards data
|
|
|
|
* @i: The iterator to initialise.
|
|
|
|
* @direction: The direction of the transfer.
|
|
|
|
* @count: The size of the I/O buffer in bytes.
|
|
|
|
*
|
|
|
|
* Set up an I/O iterator that just discards everything that's written to it.
|
|
|
|
* It's only available as a READ iterator.
|
|
|
|
*/
|
|
|
|
void iov_iter_discard(struct iov_iter *i, unsigned int direction, size_t count)
|
|
|
|
{
|
|
|
|
BUG_ON(direction != READ);
|
2021-04-22 21:50:39 +03:00
|
|
|
*i = (struct iov_iter){
|
|
|
|
.iter_type = ITER_DISCARD,
|
mm: hwpoison: coredump: support recovery from dump_user_range()
dump_user_range() is used to copy the user page to a coredump file, but if
a hardware memory error occurred during copy, which called from
__kernel_write_iter() in dump_user_range(), it crashes,
CPU: 112 PID: 7014 Comm: mca-recover Not tainted 6.3.0-rc2 #425
pc : __memcpy+0x110/0x260
lr : _copy_from_iter+0x3bc/0x4c8
...
Call trace:
__memcpy+0x110/0x260
copy_page_from_iter+0xcc/0x130
pipe_write+0x164/0x6d8
__kernel_write_iter+0x9c/0x210
dump_user_range+0xc8/0x1d8
elf_core_dump+0x308/0x368
do_coredump+0x2e8/0xa40
get_signal+0x59c/0x788
do_signal+0x118/0x1f8
do_notify_resume+0xf0/0x280
el0_da+0x130/0x138
el0t_64_sync_handler+0x68/0xc0
el0t_64_sync+0x188/0x190
Generally, the '->write_iter' of file ops will use copy_page_from_iter()
and copy_page_from_iter_atomic(), change memcpy() to copy_mc_to_kernel()
in both of them to handle #MC during source read, which stop coredump
processing and kill the task instead of kernel panic, but the source
address may not always a user address, so introduce a new copy_mc flag in
struct iov_iter{} to indicate that the iter could do a safe memory copy,
also introduce the helpers to set/cleck the flag, for now, it's only used
in coredump's dump_user_range(), but it could expand to any other
scenarios to fix the similar issue.
Link: https://lkml.kernel.org/r/20230417045323.11054-1-wangkefeng.wang@huawei.com
Signed-off-by: Kefeng Wang <wangkefeng.wang@huawei.com>
Cc: Alexander Viro <viro@zeniv.linux.org.uk>
Cc: Christian Brauner <brauner@kernel.org>
Cc: Miaohe Lin <linmiaohe@huawei.com>
Cc: Naoya Horiguchi <naoya.horiguchi@nec.com>
Cc: Tong Tiangen <tongtiangen@huawei.com>
Cc: Jens Axboe <axboe@kernel.dk>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
2023-04-17 07:53:23 +03:00
|
|
|
.copy_mc = false,
|
2021-04-22 21:50:39 +03:00
|
|
|
.data_source = false,
|
|
|
|
.count = count,
|
|
|
|
.iov_offset = 0
|
|
|
|
};
|
2018-10-20 02:57:56 +03:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(iov_iter_discard);
|
|
|
|
|
2022-06-10 22:58:27 +03:00
|
|
|
static bool iov_iter_aligned_iovec(const struct iov_iter *i, unsigned addr_mask,
|
|
|
|
unsigned len_mask)
|
|
|
|
{
|
|
|
|
size_t size = i->count;
|
|
|
|
size_t skip = i->iov_offset;
|
|
|
|
unsigned k;
|
|
|
|
|
|
|
|
for (k = 0; k < i->nr_segs; k++, skip = 0) {
|
2023-03-29 17:52:15 +03:00
|
|
|
const struct iovec *iov = iter_iov(i) + k;
|
|
|
|
size_t len = iov->iov_len - skip;
|
2022-06-10 22:58:27 +03:00
|
|
|
|
|
|
|
if (len > size)
|
|
|
|
len = size;
|
|
|
|
if (len & len_mask)
|
|
|
|
return false;
|
2023-03-29 17:52:15 +03:00
|
|
|
if ((unsigned long)(iov->iov_base + skip) & addr_mask)
|
2022-06-10 22:58:27 +03:00
|
|
|
return false;
|
|
|
|
|
|
|
|
size -= len;
|
|
|
|
if (!size)
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
static bool iov_iter_aligned_bvec(const struct iov_iter *i, unsigned addr_mask,
|
|
|
|
unsigned len_mask)
|
|
|
|
{
|
|
|
|
size_t size = i->count;
|
|
|
|
unsigned skip = i->iov_offset;
|
|
|
|
unsigned k;
|
|
|
|
|
|
|
|
for (k = 0; k < i->nr_segs; k++, skip = 0) {
|
|
|
|
size_t len = i->bvec[k].bv_len - skip;
|
|
|
|
|
|
|
|
if (len > size)
|
|
|
|
len = size;
|
|
|
|
if (len & len_mask)
|
|
|
|
return false;
|
|
|
|
if ((unsigned long)(i->bvec[k].bv_offset + skip) & addr_mask)
|
|
|
|
return false;
|
|
|
|
|
|
|
|
size -= len;
|
|
|
|
if (!size)
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* iov_iter_is_aligned() - Check if the addresses and lengths of each segments
|
|
|
|
* are aligned to the parameters.
|
|
|
|
*
|
|
|
|
* @i: &struct iov_iter to restore
|
|
|
|
* @addr_mask: bit mask to check against the iov element's addresses
|
|
|
|
* @len_mask: bit mask to check against the iov element's lengths
|
|
|
|
*
|
|
|
|
* Return: false if any addresses or lengths intersect with the provided masks
|
|
|
|
*/
|
|
|
|
bool iov_iter_is_aligned(const struct iov_iter *i, unsigned addr_mask,
|
|
|
|
unsigned len_mask)
|
|
|
|
{
|
2022-05-22 21:59:25 +03:00
|
|
|
if (likely(iter_is_ubuf(i))) {
|
|
|
|
if (i->count & len_mask)
|
|
|
|
return false;
|
|
|
|
if ((unsigned long)(i->ubuf + i->iov_offset) & addr_mask)
|
|
|
|
return false;
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2022-06-10 22:58:27 +03:00
|
|
|
if (likely(iter_is_iovec(i) || iov_iter_is_kvec(i)))
|
|
|
|
return iov_iter_aligned_iovec(i, addr_mask, len_mask);
|
|
|
|
|
|
|
|
if (iov_iter_is_bvec(i))
|
|
|
|
return iov_iter_aligned_bvec(i, addr_mask, len_mask);
|
|
|
|
|
|
|
|
if (iov_iter_is_xarray(i)) {
|
|
|
|
if (i->count & len_mask)
|
|
|
|
return false;
|
|
|
|
if ((i->xarray_start + i->iov_offset) & addr_mask)
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(iov_iter_is_aligned);
|
|
|
|
|
2021-04-25 07:44:35 +03:00
|
|
|
static unsigned long iov_iter_alignment_iovec(const struct iov_iter *i)
|
2014-04-05 07:12:29 +04:00
|
|
|
{
|
iov_iter.c: macros for iterating over iov_iter
iterate_all_kinds(iter, size, ident, step_iovec, step_bvec)
iterates through the ranges covered by iter (up to size bytes total),
repeating step_iovec or step_bvec for each of those. ident is
declared in expansion of that thing, either as struct iovec or
struct bvec, and it contains the range we are currently looking
at. step_bvec should be a void expression, step_iovec - a size_t
one, with non-zero meaning "stop here, that many bytes from this
range left". In the end, the amount actually handled is stored
in size.
iov_iter_copy_from_user_atomic() and iov_iter_alignment() converted
to it.
Signed-off-by: Al Viro <viro@zeniv.linux.org.uk>
2014-11-27 21:51:41 +03:00
|
|
|
unsigned long res = 0;
|
|
|
|
size_t size = i->count;
|
2021-04-25 07:44:35 +03:00
|
|
|
size_t skip = i->iov_offset;
|
|
|
|
unsigned k;
|
|
|
|
|
|
|
|
for (k = 0; k < i->nr_segs; k++, skip = 0) {
|
2023-03-29 17:52:15 +03:00
|
|
|
const struct iovec *iov = iter_iov(i) + k;
|
|
|
|
size_t len = iov->iov_len - skip;
|
2021-04-25 07:44:35 +03:00
|
|
|
if (len) {
|
2023-03-29 17:52:15 +03:00
|
|
|
res |= (unsigned long)iov->iov_base + skip;
|
2021-04-25 07:44:35 +03:00
|
|
|
if (len > size)
|
|
|
|
len = size;
|
|
|
|
res |= len;
|
|
|
|
size -= len;
|
|
|
|
if (!size)
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return res;
|
|
|
|
}
|
iov_iter.c: macros for iterating over iov_iter
iterate_all_kinds(iter, size, ident, step_iovec, step_bvec)
iterates through the ranges covered by iter (up to size bytes total),
repeating step_iovec or step_bvec for each of those. ident is
declared in expansion of that thing, either as struct iovec or
struct bvec, and it contains the range we are currently looking
at. step_bvec should be a void expression, step_iovec - a size_t
one, with non-zero meaning "stop here, that many bytes from this
range left". In the end, the amount actually handled is stored
in size.
iov_iter_copy_from_user_atomic() and iov_iter_alignment() converted
to it.
Signed-off-by: Al Viro <viro@zeniv.linux.org.uk>
2014-11-27 21:51:41 +03:00
|
|
|
|
2021-04-25 07:44:35 +03:00
|
|
|
static unsigned long iov_iter_alignment_bvec(const struct iov_iter *i)
|
|
|
|
{
|
|
|
|
unsigned res = 0;
|
|
|
|
size_t size = i->count;
|
|
|
|
unsigned skip = i->iov_offset;
|
|
|
|
unsigned k;
|
|
|
|
|
|
|
|
for (k = 0; k < i->nr_segs; k++, skip = 0) {
|
|
|
|
size_t len = i->bvec[k].bv_len - skip;
|
|
|
|
res |= (unsigned long)i->bvec[k].bv_offset + skip;
|
|
|
|
if (len > size)
|
|
|
|
len = size;
|
|
|
|
res |= len;
|
|
|
|
size -= len;
|
|
|
|
if (!size)
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
return res;
|
|
|
|
}
|
|
|
|
|
|
|
|
unsigned long iov_iter_alignment(const struct iov_iter *i)
|
|
|
|
{
|
2022-05-22 21:59:25 +03:00
|
|
|
if (likely(iter_is_ubuf(i))) {
|
|
|
|
size_t size = i->count;
|
|
|
|
if (size)
|
|
|
|
return ((unsigned long)i->ubuf + i->iov_offset) | size;
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2021-04-25 07:44:35 +03:00
|
|
|
/* iovec and kvec have identical layouts */
|
|
|
|
if (likely(iter_is_iovec(i) || iov_iter_is_kvec(i)))
|
|
|
|
return iov_iter_alignment_iovec(i);
|
|
|
|
|
|
|
|
if (iov_iter_is_bvec(i))
|
|
|
|
return iov_iter_alignment_bvec(i);
|
|
|
|
|
|
|
|
if (iov_iter_is_xarray(i))
|
2021-04-26 00:02:38 +03:00
|
|
|
return (i->xarray_start + i->iov_offset) | i->count;
|
2021-04-25 07:44:35 +03:00
|
|
|
|
|
|
|
return 0;
|
2014-04-05 07:12:29 +04:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(iov_iter_alignment);
|
|
|
|
|
2016-04-09 02:05:19 +03:00
|
|
|
unsigned long iov_iter_gap_alignment(const struct iov_iter *i)
|
|
|
|
{
|
2016-12-22 05:55:02 +03:00
|
|
|
unsigned long res = 0;
|
2021-04-25 08:03:16 +03:00
|
|
|
unsigned long v = 0;
|
2016-04-09 02:05:19 +03:00
|
|
|
size_t size = i->count;
|
2021-04-25 08:03:16 +03:00
|
|
|
unsigned k;
|
2016-04-09 02:05:19 +03:00
|
|
|
|
2022-05-22 21:59:25 +03:00
|
|
|
if (iter_is_ubuf(i))
|
|
|
|
return 0;
|
|
|
|
|
2021-04-25 08:03:16 +03:00
|
|
|
if (WARN_ON(!iter_is_iovec(i)))
|
2016-09-22 23:33:12 +03:00
|
|
|
return ~0U;
|
|
|
|
|
2021-04-25 08:03:16 +03:00
|
|
|
for (k = 0; k < i->nr_segs; k++) {
|
2023-03-29 17:52:15 +03:00
|
|
|
const struct iovec *iov = iter_iov(i) + k;
|
|
|
|
if (iov->iov_len) {
|
|
|
|
unsigned long base = (unsigned long)iov->iov_base;
|
2021-04-25 08:03:16 +03:00
|
|
|
if (v) // if not the first one
|
|
|
|
res |= base | v; // this start | previous end
|
2023-03-29 17:52:15 +03:00
|
|
|
v = base + iov->iov_len;
|
|
|
|
if (size <= iov->iov_len)
|
2021-04-25 08:03:16 +03:00
|
|
|
break;
|
2023-03-29 17:52:15 +03:00
|
|
|
size -= iov->iov_len;
|
2021-04-25 08:03:16 +03:00
|
|
|
}
|
|
|
|
}
|
2016-12-22 05:55:02 +03:00
|
|
|
return res;
|
2016-04-09 02:05:19 +03:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(iov_iter_gap_alignment);
|
|
|
|
|
2022-06-17 21:45:41 +03:00
|
|
|
static int want_pages_array(struct page ***res, size_t size,
|
|
|
|
size_t start, unsigned int maxpages)
|
2022-06-17 20:35:35 +03:00
|
|
|
{
|
2022-06-17 21:45:41 +03:00
|
|
|
unsigned int count = DIV_ROUND_UP(size + start, PAGE_SIZE);
|
|
|
|
|
|
|
|
if (count > maxpages)
|
|
|
|
count = maxpages;
|
|
|
|
WARN_ON(!count); // caller should've prevented that
|
|
|
|
if (!*res) {
|
|
|
|
*res = kvmalloc_array(count, sizeof(struct page *), GFP_KERNEL);
|
|
|
|
if (!*res)
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
return count;
|
2022-06-17 20:35:35 +03:00
|
|
|
}
|
|
|
|
|
2020-02-10 13:00:21 +03:00
|
|
|
static ssize_t iter_xarray_populate_pages(struct page **pages, struct xarray *xa,
|
|
|
|
pgoff_t index, unsigned int nr_pages)
|
|
|
|
{
|
|
|
|
XA_STATE(xas, xa, index);
|
|
|
|
struct page *page;
|
|
|
|
unsigned int ret = 0;
|
|
|
|
|
|
|
|
rcu_read_lock();
|
|
|
|
for (page = xas_load(&xas); page; page = xas_next(&xas)) {
|
|
|
|
if (xas_retry(&xas, page))
|
|
|
|
continue;
|
|
|
|
|
|
|
|
/* Has the page moved or been split? */
|
|
|
|
if (unlikely(page != xas_reload(&xas))) {
|
|
|
|
xas_reset(&xas);
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
|
|
|
pages[ret] = find_subpage(page, xas.xa_index);
|
|
|
|
get_page(pages[ret]);
|
|
|
|
if (++ret == nr_pages)
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
rcu_read_unlock();
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
static ssize_t iter_xarray_get_pages(struct iov_iter *i,
|
2022-06-17 20:48:03 +03:00
|
|
|
struct page ***pages, size_t maxsize,
|
2020-02-10 13:00:21 +03:00
|
|
|
unsigned maxpages, size_t *_start_offset)
|
|
|
|
{
|
2022-06-17 21:45:41 +03:00
|
|
|
unsigned nr, offset, count;
|
|
|
|
pgoff_t index;
|
2020-02-10 13:00:21 +03:00
|
|
|
loff_t pos;
|
|
|
|
|
|
|
|
pos = i->xarray_start + i->iov_offset;
|
|
|
|
index = pos >> PAGE_SHIFT;
|
|
|
|
offset = pos & ~PAGE_MASK;
|
|
|
|
*_start_offset = offset;
|
|
|
|
|
2022-06-17 21:45:41 +03:00
|
|
|
count = want_pages_array(pages, maxsize, offset, maxpages);
|
|
|
|
if (!count)
|
|
|
|
return -ENOMEM;
|
2022-06-17 20:48:03 +03:00
|
|
|
nr = iter_xarray_populate_pages(*pages, i->xarray, index, count);
|
2020-02-10 13:00:21 +03:00
|
|
|
if (nr == 0)
|
|
|
|
return 0;
|
|
|
|
|
2022-06-10 20:05:12 +03:00
|
|
|
maxsize = min_t(size_t, nr * PAGE_SIZE - offset, maxsize);
|
2022-06-11 11:04:33 +03:00
|
|
|
i->iov_offset += maxsize;
|
|
|
|
i->count -= maxsize;
|
2022-06-10 20:05:12 +03:00
|
|
|
return maxsize;
|
2020-02-10 13:00:21 +03:00
|
|
|
}
|
|
|
|
|
2022-05-22 21:59:25 +03:00
|
|
|
/* must be done on non-empty ITER_UBUF or ITER_IOVEC one */
|
2022-06-17 23:07:49 +03:00
|
|
|
static unsigned long first_iovec_segment(const struct iov_iter *i, size_t *size)
|
2021-04-25 16:14:44 +03:00
|
|
|
{
|
|
|
|
size_t skip;
|
|
|
|
long k;
|
|
|
|
|
2022-05-22 21:59:25 +03:00
|
|
|
if (iter_is_ubuf(i))
|
|
|
|
return (unsigned long)i->ubuf + i->iov_offset;
|
|
|
|
|
2021-04-25 16:14:44 +03:00
|
|
|
for (k = 0, skip = i->iov_offset; k < i->nr_segs; k++, skip = 0) {
|
2023-03-29 17:52:15 +03:00
|
|
|
const struct iovec *iov = iter_iov(i) + k;
|
|
|
|
size_t len = iov->iov_len - skip;
|
2021-04-25 16:14:44 +03:00
|
|
|
|
|
|
|
if (unlikely(!len))
|
|
|
|
continue;
|
2022-06-21 23:10:37 +03:00
|
|
|
if (*size > len)
|
|
|
|
*size = len;
|
2023-03-29 17:52:15 +03:00
|
|
|
return (unsigned long)iov->iov_base + skip;
|
2021-04-25 16:14:44 +03:00
|
|
|
}
|
|
|
|
BUG(); // if it had been empty, we wouldn't get called
|
|
|
|
}
|
|
|
|
|
|
|
|
/* must be done on non-empty ITER_BVEC one */
|
|
|
|
static struct page *first_bvec_segment(const struct iov_iter *i,
|
2022-06-21 23:10:37 +03:00
|
|
|
size_t *size, size_t *start)
|
2021-04-25 16:14:44 +03:00
|
|
|
{
|
|
|
|
struct page *page;
|
|
|
|
size_t skip = i->iov_offset, len;
|
|
|
|
|
|
|
|
len = i->bvec->bv_len - skip;
|
2022-06-21 23:10:37 +03:00
|
|
|
if (*size > len)
|
|
|
|
*size = len;
|
2021-04-25 16:14:44 +03:00
|
|
|
skip += i->bvec->bv_offset;
|
|
|
|
page = i->bvec->bv_page + skip / PAGE_SIZE;
|
2022-06-21 22:55:19 +03:00
|
|
|
*start = skip % PAGE_SIZE;
|
2021-04-25 16:14:44 +03:00
|
|
|
return page;
|
|
|
|
}
|
|
|
|
|
2022-06-17 20:54:15 +03:00
|
|
|
static ssize_t __iov_iter_get_pages_alloc(struct iov_iter *i,
|
|
|
|
struct page ***pages, size_t maxsize,
|
2023-06-14 17:03:41 +03:00
|
|
|
unsigned int maxpages, size_t *start)
|
2014-04-05 07:12:29 +04:00
|
|
|
{
|
2023-01-19 15:47:23 +03:00
|
|
|
unsigned int n, gup_flags = 0;
|
2021-04-25 16:14:44 +03:00
|
|
|
|
2014-11-27 22:12:09 +03:00
|
|
|
if (maxsize > i->count)
|
|
|
|
maxsize = i->count;
|
2022-06-17 20:54:15 +03:00
|
|
|
if (!maxsize)
|
2021-04-25 16:14:44 +03:00
|
|
|
return 0;
|
2022-06-11 23:44:21 +03:00
|
|
|
if (maxsize > MAX_RW_COUNT)
|
|
|
|
maxsize = MAX_RW_COUNT;
|
2014-11-27 22:12:09 +03:00
|
|
|
|
2022-05-22 21:59:25 +03:00
|
|
|
if (likely(user_backed_iter(i))) {
|
2021-04-25 16:14:44 +03:00
|
|
|
unsigned long addr;
|
2022-06-17 21:45:41 +03:00
|
|
|
int res;
|
2014-11-27 22:12:09 +03:00
|
|
|
|
2021-07-12 13:06:14 +03:00
|
|
|
if (iov_iter_rw(i) != WRITE)
|
|
|
|
gup_flags |= FOLL_WRITE;
|
|
|
|
if (i->nofault)
|
|
|
|
gup_flags |= FOLL_NOFAULT;
|
|
|
|
|
2022-06-17 23:07:49 +03:00
|
|
|
addr = first_iovec_segment(i, &maxsize);
|
|
|
|
*start = addr % PAGE_SIZE;
|
|
|
|
addr &= PAGE_MASK;
|
2022-06-17 21:45:41 +03:00
|
|
|
n = want_pages_array(pages, maxsize, *start, maxpages);
|
|
|
|
if (!n)
|
|
|
|
return -ENOMEM;
|
2022-06-17 20:54:15 +03:00
|
|
|
res = get_user_pages_fast(addr, n, gup_flags, *pages);
|
2021-07-21 20:03:47 +03:00
|
|
|
if (unlikely(res <= 0))
|
2014-11-27 22:12:09 +03:00
|
|
|
return res;
|
2022-06-10 20:05:12 +03:00
|
|
|
maxsize = min_t(size_t, maxsize, res * PAGE_SIZE - *start);
|
|
|
|
iov_iter_advance(i, maxsize);
|
|
|
|
return maxsize;
|
2021-04-25 16:14:44 +03:00
|
|
|
}
|
|
|
|
if (iov_iter_is_bvec(i)) {
|
2022-06-17 20:54:15 +03:00
|
|
|
struct page **p;
|
2021-04-25 16:14:44 +03:00
|
|
|
struct page *page;
|
|
|
|
|
2022-06-21 23:10:37 +03:00
|
|
|
page = first_bvec_segment(i, &maxsize, start);
|
2022-06-17 21:45:41 +03:00
|
|
|
n = want_pages_array(pages, maxsize, *start, maxpages);
|
|
|
|
if (!n)
|
|
|
|
return -ENOMEM;
|
2022-06-17 20:54:15 +03:00
|
|
|
p = *pages;
|
2022-06-21 22:55:19 +03:00
|
|
|
for (int k = 0; k < n; k++)
|
2022-06-10 20:05:12 +03:00
|
|
|
get_page(p[k] = page + k);
|
|
|
|
maxsize = min_t(size_t, maxsize, n * PAGE_SIZE - *start);
|
2022-06-11 11:04:33 +03:00
|
|
|
i->count -= maxsize;
|
|
|
|
i->iov_offset += maxsize;
|
|
|
|
if (i->iov_offset == i->bvec->bv_len) {
|
|
|
|
i->iov_offset = 0;
|
|
|
|
i->bvec++;
|
|
|
|
i->nr_segs--;
|
|
|
|
}
|
2022-06-10 20:05:12 +03:00
|
|
|
return maxsize;
|
2021-04-25 16:14:44 +03:00
|
|
|
}
|
|
|
|
if (iov_iter_is_xarray(i))
|
2022-06-17 20:54:15 +03:00
|
|
|
return iter_xarray_get_pages(i, pages, maxsize, maxpages, start);
|
2021-04-25 16:14:44 +03:00
|
|
|
return -EFAULT;
|
2014-04-05 07:12:29 +04:00
|
|
|
}
|
|
|
|
|
2023-06-14 17:03:41 +03:00
|
|
|
ssize_t iov_iter_get_pages2(struct iov_iter *i, struct page **pages,
|
|
|
|
size_t maxsize, unsigned maxpages, size_t *start)
|
2014-04-05 07:12:29 +04:00
|
|
|
{
|
2022-06-17 20:54:15 +03:00
|
|
|
if (!maxpages)
|
2021-04-25 16:14:44 +03:00
|
|
|
return 0;
|
2022-06-17 20:54:15 +03:00
|
|
|
BUG_ON(!pages);
|
2021-04-25 16:14:44 +03:00
|
|
|
|
2023-06-14 17:03:41 +03:00
|
|
|
return __iov_iter_get_pages_alloc(i, &pages, maxsize, maxpages, start);
|
2014-04-05 07:12:29 +04:00
|
|
|
}
|
2022-06-10 20:05:12 +03:00
|
|
|
EXPORT_SYMBOL(iov_iter_get_pages2);
|
2022-06-11 03:38:20 +03:00
|
|
|
|
2023-06-14 17:03:41 +03:00
|
|
|
ssize_t iov_iter_get_pages_alloc2(struct iov_iter *i,
|
|
|
|
struct page ***pages, size_t maxsize, size_t *start)
|
2022-06-11 03:38:20 +03:00
|
|
|
{
|
|
|
|
ssize_t len;
|
|
|
|
|
|
|
|
*pages = NULL;
|
|
|
|
|
2023-06-14 17:03:41 +03:00
|
|
|
len = __iov_iter_get_pages_alloc(i, pages, maxsize, ~0U, start);
|
2022-06-11 03:38:20 +03:00
|
|
|
if (len <= 0) {
|
|
|
|
kvfree(*pages);
|
|
|
|
*pages = NULL;
|
|
|
|
}
|
|
|
|
return len;
|
|
|
|
}
|
2022-06-10 20:05:12 +03:00
|
|
|
EXPORT_SYMBOL(iov_iter_get_pages_alloc2);
|
2014-04-05 07:12:29 +04:00
|
|
|
|
2014-11-24 09:08:00 +03:00
|
|
|
size_t csum_and_copy_from_iter(void *addr, size_t bytes, __wsum *csum,
|
|
|
|
struct iov_iter *i)
|
|
|
|
{
|
|
|
|
__wsum sum, next;
|
|
|
|
sum = *csum;
|
2022-09-16 03:11:15 +03:00
|
|
|
if (WARN_ON_ONCE(!i->data_source))
|
2016-09-22 23:33:12 +03:00
|
|
|
return 0;
|
2022-09-16 03:11:15 +03:00
|
|
|
|
2021-05-02 18:35:03 +03:00
|
|
|
iterate_and_advance(i, bytes, base, len, off, ({
|
|
|
|
next = csum_and_copy_from_user(base, addr + off, len);
|
2021-04-30 20:40:48 +03:00
|
|
|
sum = csum_block_add(sum, next, off);
|
2021-05-02 18:35:03 +03:00
|
|
|
next ? 0 : len;
|
2014-11-24 09:08:00 +03:00
|
|
|
}), ({
|
2021-05-02 18:35:03 +03:00
|
|
|
sum = csum_and_memcpy(addr + off, base, len, sum, off);
|
2014-11-24 09:08:00 +03:00
|
|
|
})
|
|
|
|
)
|
|
|
|
*csum = sum;
|
|
|
|
return bytes;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(csum_and_copy_from_iter);
|
|
|
|
|
2021-02-03 22:29:52 +03:00
|
|
|
size_t csum_and_copy_to_iter(const void *addr, size_t bytes, void *_csstate,
|
2014-11-24 09:08:00 +03:00
|
|
|
struct iov_iter *i)
|
|
|
|
{
|
2021-02-03 22:29:52 +03:00
|
|
|
struct csum_state *csstate = _csstate;
|
2014-11-24 09:08:00 +03:00
|
|
|
__wsum sum, next;
|
2018-11-26 00:24:16 +03:00
|
|
|
|
2022-09-16 03:11:15 +03:00
|
|
|
if (WARN_ON_ONCE(i->data_source))
|
|
|
|
return 0;
|
2018-11-26 00:24:16 +03:00
|
|
|
if (unlikely(iov_iter_is_discard(i))) {
|
2022-09-16 03:20:59 +03:00
|
|
|
// can't use csum_memcpy() for that one - data is not copied
|
|
|
|
csstate->csum = csum_block_add(csstate->csum,
|
|
|
|
csum_partial(addr, bytes, 0),
|
|
|
|
csstate->off);
|
|
|
|
csstate->off += bytes;
|
|
|
|
return bytes;
|
2016-09-22 23:33:12 +03:00
|
|
|
}
|
2021-05-03 00:24:40 +03:00
|
|
|
|
|
|
|
sum = csum_shift(csstate->csum, csstate->off);
|
2023-05-22 16:50:17 +03:00
|
|
|
iterate_and_advance(i, bytes, base, len, off, ({
|
2021-05-02 18:35:03 +03:00
|
|
|
next = csum_and_copy_to_user(addr + off, base, len);
|
2021-04-30 20:40:48 +03:00
|
|
|
sum = csum_block_add(sum, next, off);
|
2021-05-02 18:35:03 +03:00
|
|
|
next ? 0 : len;
|
2014-11-24 09:08:00 +03:00
|
|
|
}), ({
|
2021-05-02 18:35:03 +03:00
|
|
|
sum = csum_and_memcpy(base, addr + off, len, sum, off);
|
2014-11-24 09:08:00 +03:00
|
|
|
})
|
|
|
|
)
|
2021-06-05 17:19:30 +03:00
|
|
|
csstate->csum = csum_shift(sum, csstate->off);
|
|
|
|
csstate->off += bytes;
|
2014-11-24 09:08:00 +03:00
|
|
|
return bytes;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(csum_and_copy_to_iter);
|
|
|
|
|
2018-12-04 04:52:09 +03:00
|
|
|
size_t hash_and_copy_to_iter(const void *addr, size_t bytes, void *hashp,
|
|
|
|
struct iov_iter *i)
|
|
|
|
{
|
2020-06-12 09:57:37 +03:00
|
|
|
#ifdef CONFIG_CRYPTO_HASH
|
2018-12-04 04:52:09 +03:00
|
|
|
struct ahash_request *hash = hashp;
|
|
|
|
struct scatterlist sg;
|
|
|
|
size_t copied;
|
|
|
|
|
|
|
|
copied = copy_to_iter(addr, bytes, i);
|
|
|
|
sg_init_one(&sg, addr, copied);
|
|
|
|
ahash_request_set_crypt(hash, &sg, NULL, copied);
|
|
|
|
crypto_ahash_update(hash);
|
|
|
|
return copied;
|
2019-04-04 05:31:14 +03:00
|
|
|
#else
|
|
|
|
return 0;
|
|
|
|
#endif
|
2018-12-04 04:52:09 +03:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(hash_and_copy_to_iter);
|
|
|
|
|
2021-04-25 23:00:48 +03:00
|
|
|
static int iov_npages(const struct iov_iter *i, int maxpages)
|
2014-04-05 07:12:29 +04:00
|
|
|
{
|
2021-04-25 23:00:48 +03:00
|
|
|
size_t skip = i->iov_offset, size = i->count;
|
|
|
|
const struct iovec *p;
|
2014-11-27 22:09:46 +03:00
|
|
|
int npages = 0;
|
|
|
|
|
2023-03-29 17:52:15 +03:00
|
|
|
for (p = iter_iov(i); size; skip = 0, p++) {
|
2021-04-25 23:00:48 +03:00
|
|
|
unsigned offs = offset_in_page(p->iov_base + skip);
|
|
|
|
size_t len = min(p->iov_len - skip, size);
|
2014-11-27 22:09:46 +03:00
|
|
|
|
2021-04-25 23:00:48 +03:00
|
|
|
if (len) {
|
|
|
|
size -= len;
|
|
|
|
npages += DIV_ROUND_UP(offs + len, PAGE_SIZE);
|
|
|
|
if (unlikely(npages > maxpages))
|
|
|
|
return maxpages;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return npages;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int bvec_npages(const struct iov_iter *i, int maxpages)
|
|
|
|
{
|
|
|
|
size_t skip = i->iov_offset, size = i->count;
|
|
|
|
const struct bio_vec *p;
|
|
|
|
int npages = 0;
|
|
|
|
|
|
|
|
for (p = i->bvec; size; skip = 0, p++) {
|
|
|
|
unsigned offs = (p->bv_offset + skip) % PAGE_SIZE;
|
|
|
|
size_t len = min(p->bv_len - skip, size);
|
|
|
|
|
|
|
|
size -= len;
|
|
|
|
npages += DIV_ROUND_UP(offs + len, PAGE_SIZE);
|
|
|
|
if (unlikely(npages > maxpages))
|
|
|
|
return maxpages;
|
|
|
|
}
|
|
|
|
return npages;
|
|
|
|
}
|
|
|
|
|
|
|
|
int iov_iter_npages(const struct iov_iter *i, int maxpages)
|
|
|
|
{
|
|
|
|
if (unlikely(!i->count))
|
|
|
|
return 0;
|
2022-05-22 21:59:25 +03:00
|
|
|
if (likely(iter_is_ubuf(i))) {
|
|
|
|
unsigned offs = offset_in_page(i->ubuf + i->iov_offset);
|
|
|
|
int npages = DIV_ROUND_UP(offs + i->count, PAGE_SIZE);
|
|
|
|
return min(npages, maxpages);
|
|
|
|
}
|
2021-04-25 23:00:48 +03:00
|
|
|
/* iovec and kvec have identical layouts */
|
|
|
|
if (likely(iter_is_iovec(i) || iov_iter_is_kvec(i)))
|
|
|
|
return iov_npages(i, maxpages);
|
|
|
|
if (iov_iter_is_bvec(i))
|
|
|
|
return bvec_npages(i, maxpages);
|
|
|
|
if (iov_iter_is_xarray(i)) {
|
2021-05-03 18:05:29 +03:00
|
|
|
unsigned offset = (i->xarray_start + i->iov_offset) % PAGE_SIZE;
|
|
|
|
int npages = DIV_ROUND_UP(offset + i->count, PAGE_SIZE);
|
2021-04-25 23:00:48 +03:00
|
|
|
return min(npages, maxpages);
|
|
|
|
}
|
|
|
|
return 0;
|
2014-04-05 07:12:29 +04:00
|
|
|
}
|
2014-03-19 09:16:16 +04:00
|
|
|
EXPORT_SYMBOL(iov_iter_npages);
|
2015-02-01 04:08:47 +03:00
|
|
|
|
|
|
|
const void *dup_iter(struct iov_iter *new, struct iov_iter *old, gfp_t flags)
|
|
|
|
{
|
|
|
|
*new = *old;
|
2018-10-22 15:07:28 +03:00
|
|
|
if (iov_iter_is_bvec(new))
|
2015-02-01 04:08:47 +03:00
|
|
|
return new->bvec = kmemdup(new->bvec,
|
|
|
|
new->nr_segs * sizeof(struct bio_vec),
|
|
|
|
flags);
|
2022-05-22 21:59:25 +03:00
|
|
|
else if (iov_iter_is_kvec(new) || iter_is_iovec(new))
|
2015-02-01 04:08:47 +03:00
|
|
|
/* iovec and kvec have identical layout */
|
2023-03-29 17:52:15 +03:00
|
|
|
return new->__iov = kmemdup(new->__iov,
|
2015-02-01 04:08:47 +03:00
|
|
|
new->nr_segs * sizeof(struct iovec),
|
|
|
|
flags);
|
2022-05-22 21:59:25 +03:00
|
|
|
return NULL;
|
2015-02-01 04:08:47 +03:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(dup_iter);
|
saner iov_iter initialization primitives
iovec-backed iov_iter instances are assumed to satisfy several properties:
* no more than UIO_MAXIOV elements in iovec array
* total size of all ranges is no more than MAX_RW_COUNT
* all ranges pass access_ok().
The problem is, invariants of data structures should be established in the
primitives creating those data structures, not in the code using those
primitives. And iov_iter_init() violates that principle. For a while we
managed to get away with that, but once the use of iov_iter started to
spread, it didn't take long for shit to hit the fan - missed check in
sys_sendto() had introduced a roothole.
We _do_ have primitives for importing and validating iovecs (both native and
compat ones) and those primitives are almost always followed by shoving the
resulting iovec into iov_iter. Life would be considerably simpler (and safer)
if we combined those primitives with initializing iov_iter.
That gives us two new primitives - import_iovec() and compat_import_iovec().
Calling conventions:
iovec = iov_array;
err = import_iovec(direction, uvec, nr_segs,
ARRAY_SIZE(iov_array), &iovec,
&iter);
imports user vector into kernel space (into iov_array if it fits, allocated
if it doesn't fit or if iovec was NULL), validates it and sets iter up to
refer to it. On success 0 is returned and allocated kernel copy (or NULL
if the array had fit into caller-supplied one) is returned via iovec.
On failure all allocations are undone and -E... is returned. If the total
size of ranges exceeds MAX_RW_COUNT, the excess is silently truncated.
compat_import_iovec() expects uvec to be a pointer to user array of compat_iovec;
otherwise it's identical to import_iovec().
Finally, import_single_range() sets iov_iter backed by single-element iovec
covering a user-supplied range -
err = import_single_range(direction, address, size, iovec, &iter);
does validation and sets iter up. Again, size in excess of MAX_RW_COUNT gets
silently truncated.
Next commits will be switching the things up to use of those and reducing
the amount of iov_iter_init() instances.
Signed-off-by: Al Viro <viro@zeniv.linux.org.uk>
2015-03-22 00:45:43 +03:00
|
|
|
|
2023-04-12 19:46:48 +03:00
|
|
|
static __noclone int copy_compat_iovec_from_user(struct iovec *iov,
|
2020-09-25 07:51:40 +03:00
|
|
|
const struct iovec __user *uvec, unsigned long nr_segs)
|
|
|
|
{
|
|
|
|
const struct compat_iovec __user *uiov =
|
|
|
|
(const struct compat_iovec __user *)uvec;
|
|
|
|
int ret = -EFAULT, i;
|
|
|
|
|
2021-01-11 20:19:26 +03:00
|
|
|
if (!user_access_begin(uiov, nr_segs * sizeof(*uiov)))
|
2020-09-25 07:51:40 +03:00
|
|
|
return -EFAULT;
|
|
|
|
|
|
|
|
for (i = 0; i < nr_segs; i++) {
|
|
|
|
compat_uptr_t buf;
|
|
|
|
compat_ssize_t len;
|
|
|
|
|
|
|
|
unsafe_get_user(len, &uiov[i].iov_len, uaccess_end);
|
|
|
|
unsafe_get_user(buf, &uiov[i].iov_base, uaccess_end);
|
|
|
|
|
|
|
|
/* check for compat_size_t not fitting in compat_ssize_t .. */
|
|
|
|
if (len < 0) {
|
|
|
|
ret = -EINVAL;
|
|
|
|
goto uaccess_end;
|
|
|
|
}
|
|
|
|
iov[i].iov_base = compat_ptr(buf);
|
|
|
|
iov[i].iov_len = len;
|
|
|
|
}
|
|
|
|
|
|
|
|
ret = 0;
|
|
|
|
uaccess_end:
|
|
|
|
user_access_end();
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
2023-06-16 15:43:55 +03:00
|
|
|
static __noclone int copy_iovec_from_user(struct iovec *iov,
|
2023-03-31 00:53:51 +03:00
|
|
|
const struct iovec __user *uiov, unsigned long nr_segs)
|
2020-09-25 07:51:39 +03:00
|
|
|
{
|
2023-03-31 00:53:51 +03:00
|
|
|
int ret = -EFAULT;
|
2020-09-25 07:51:39 +03:00
|
|
|
|
2023-03-31 00:53:51 +03:00
|
|
|
if (!user_access_begin(uiov, nr_segs * sizeof(*uiov)))
|
2020-09-25 07:51:40 +03:00
|
|
|
return -EFAULT;
|
2020-09-25 07:51:39 +03:00
|
|
|
|
2023-03-31 00:53:51 +03:00
|
|
|
do {
|
|
|
|
void __user *buf;
|
|
|
|
ssize_t len;
|
|
|
|
|
|
|
|
unsafe_get_user(len, &uiov->iov_len, uaccess_end);
|
|
|
|
unsafe_get_user(buf, &uiov->iov_base, uaccess_end);
|
|
|
|
|
|
|
|
/* check for size_t not fitting in ssize_t .. */
|
|
|
|
if (unlikely(len < 0)) {
|
|
|
|
ret = -EINVAL;
|
|
|
|
goto uaccess_end;
|
|
|
|
}
|
|
|
|
iov->iov_base = buf;
|
|
|
|
iov->iov_len = len;
|
|
|
|
|
|
|
|
uiov++; iov++;
|
|
|
|
} while (--nr_segs);
|
|
|
|
|
|
|
|
ret = 0;
|
|
|
|
uaccess_end:
|
|
|
|
user_access_end();
|
|
|
|
return ret;
|
2020-09-25 07:51:40 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
struct iovec *iovec_from_user(const struct iovec __user *uvec,
|
|
|
|
unsigned long nr_segs, unsigned long fast_segs,
|
|
|
|
struct iovec *fast_iov, bool compat)
|
|
|
|
{
|
|
|
|
struct iovec *iov = fast_iov;
|
|
|
|
int ret;
|
|
|
|
|
2020-09-25 07:51:39 +03:00
|
|
|
/*
|
2020-09-25 07:51:40 +03:00
|
|
|
* SuS says "The readv() function *may* fail if the iovcnt argument was
|
|
|
|
* less than or equal to 0, or greater than {IOV_MAX}. Linux has
|
|
|
|
* traditionally returned zero for zero segments, so...
|
2020-09-25 07:51:39 +03:00
|
|
|
*/
|
2020-09-25 07:51:40 +03:00
|
|
|
if (nr_segs == 0)
|
|
|
|
return iov;
|
|
|
|
if (nr_segs > UIO_MAXIOV)
|
|
|
|
return ERR_PTR(-EINVAL);
|
2020-09-25 07:51:39 +03:00
|
|
|
if (nr_segs > fast_segs) {
|
|
|
|
iov = kmalloc_array(nr_segs, sizeof(struct iovec), GFP_KERNEL);
|
2020-09-25 07:51:40 +03:00
|
|
|
if (!iov)
|
|
|
|
return ERR_PTR(-ENOMEM);
|
2020-09-25 07:51:39 +03:00
|
|
|
}
|
2020-09-25 07:51:40 +03:00
|
|
|
|
2023-03-31 00:53:51 +03:00
|
|
|
if (unlikely(compat))
|
2020-09-25 07:51:40 +03:00
|
|
|
ret = copy_compat_iovec_from_user(iov, uvec, nr_segs);
|
|
|
|
else
|
|
|
|
ret = copy_iovec_from_user(iov, uvec, nr_segs);
|
|
|
|
if (ret) {
|
|
|
|
if (iov != fast_iov)
|
|
|
|
kfree(iov);
|
|
|
|
return ERR_PTR(ret);
|
|
|
|
}
|
|
|
|
|
|
|
|
return iov;
|
|
|
|
}
|
|
|
|
|
2023-03-24 23:37:19 +03:00
|
|
|
/*
|
|
|
|
* Single segment iovec supplied by the user, import it as ITER_UBUF.
|
|
|
|
*/
|
|
|
|
static ssize_t __import_iovec_ubuf(int type, const struct iovec __user *uvec,
|
|
|
|
struct iovec **iovp, struct iov_iter *i,
|
|
|
|
bool compat)
|
|
|
|
{
|
|
|
|
struct iovec *iov = *iovp;
|
|
|
|
ssize_t ret;
|
|
|
|
|
|
|
|
if (compat)
|
|
|
|
ret = copy_compat_iovec_from_user(iov, uvec, 1);
|
|
|
|
else
|
|
|
|
ret = copy_iovec_from_user(iov, uvec, 1);
|
|
|
|
if (unlikely(ret))
|
|
|
|
return ret;
|
|
|
|
|
|
|
|
ret = import_ubuf(type, iov->iov_base, iov->iov_len, i);
|
|
|
|
if (unlikely(ret))
|
|
|
|
return ret;
|
|
|
|
*iovp = NULL;
|
|
|
|
return i->count;
|
|
|
|
}
|
|
|
|
|
2020-09-25 07:51:40 +03:00
|
|
|
ssize_t __import_iovec(int type, const struct iovec __user *uvec,
|
|
|
|
unsigned nr_segs, unsigned fast_segs, struct iovec **iovp,
|
|
|
|
struct iov_iter *i, bool compat)
|
|
|
|
{
|
|
|
|
ssize_t total_len = 0;
|
|
|
|
unsigned long seg;
|
|
|
|
struct iovec *iov;
|
|
|
|
|
2023-03-24 23:37:19 +03:00
|
|
|
if (nr_segs == 1)
|
|
|
|
return __import_iovec_ubuf(type, uvec, iovp, i, compat);
|
|
|
|
|
2020-09-25 07:51:40 +03:00
|
|
|
iov = iovec_from_user(uvec, nr_segs, fast_segs, *iovp, compat);
|
|
|
|
if (IS_ERR(iov)) {
|
|
|
|
*iovp = NULL;
|
|
|
|
return PTR_ERR(iov);
|
2020-09-25 07:51:39 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
2020-09-25 07:51:40 +03:00
|
|
|
* According to the Single Unix Specification we should return EINVAL if
|
|
|
|
* an element length is < 0 when cast to ssize_t or if the total length
|
|
|
|
* would overflow the ssize_t return value of the system call.
|
2020-09-25 07:51:39 +03:00
|
|
|
*
|
|
|
|
* Linux caps all read/write calls to MAX_RW_COUNT, and avoids the
|
|
|
|
* overflow case.
|
|
|
|
*/
|
|
|
|
for (seg = 0; seg < nr_segs; seg++) {
|
|
|
|
ssize_t len = (ssize_t)iov[seg].iov_len;
|
|
|
|
|
2020-09-25 07:51:40 +03:00
|
|
|
if (!access_ok(iov[seg].iov_base, len)) {
|
|
|
|
if (iov != *iovp)
|
|
|
|
kfree(iov);
|
|
|
|
*iovp = NULL;
|
|
|
|
return -EFAULT;
|
2020-09-25 07:51:39 +03:00
|
|
|
}
|
2020-09-25 07:51:40 +03:00
|
|
|
|
|
|
|
if (len > MAX_RW_COUNT - total_len) {
|
|
|
|
len = MAX_RW_COUNT - total_len;
|
2020-09-25 07:51:39 +03:00
|
|
|
iov[seg].iov_len = len;
|
|
|
|
}
|
2020-09-25 07:51:40 +03:00
|
|
|
total_len += len;
|
2020-09-25 07:51:39 +03:00
|
|
|
}
|
2020-09-25 07:51:40 +03:00
|
|
|
|
|
|
|
iov_iter_init(i, type, iov, nr_segs, total_len);
|
|
|
|
if (iov == *iovp)
|
|
|
|
*iovp = NULL;
|
|
|
|
else
|
|
|
|
*iovp = iov;
|
|
|
|
return total_len;
|
2020-09-25 07:51:39 +03:00
|
|
|
}
|
|
|
|
|
2016-10-08 12:18:07 +03:00
|
|
|
/**
|
|
|
|
* import_iovec() - Copy an array of &struct iovec from userspace
|
|
|
|
* into the kernel, check that it is valid, and initialize a new
|
|
|
|
* &struct iov_iter iterator to access it.
|
|
|
|
*
|
|
|
|
* @type: One of %READ or %WRITE.
|
2020-09-25 07:51:40 +03:00
|
|
|
* @uvec: Pointer to the userspace array.
|
2016-10-08 12:18:07 +03:00
|
|
|
* @nr_segs: Number of elements in userspace array.
|
|
|
|
* @fast_segs: Number of elements in @iov.
|
2020-09-25 07:51:40 +03:00
|
|
|
* @iovp: (input and output parameter) Pointer to pointer to (usually small
|
2016-10-08 12:18:07 +03:00
|
|
|
* on-stack) kernel array.
|
|
|
|
* @i: Pointer to iterator that will be initialized on success.
|
|
|
|
*
|
|
|
|
* If the array pointed to by *@iov is large enough to hold all @nr_segs,
|
|
|
|
* then this function places %NULL in *@iov on return. Otherwise, a new
|
|
|
|
* array will be allocated and the result placed in *@iov. This means that
|
|
|
|
* the caller may call kfree() on *@iov regardless of whether the small
|
|
|
|
* on-stack array was used or not (and regardless of whether this function
|
|
|
|
* returns an error or not).
|
|
|
|
*
|
2019-05-15 01:02:22 +03:00
|
|
|
* Return: Negative error code on error, bytes imported on success
|
2016-10-08 12:18:07 +03:00
|
|
|
*/
|
2020-09-25 07:51:40 +03:00
|
|
|
ssize_t import_iovec(int type, const struct iovec __user *uvec,
|
saner iov_iter initialization primitives
iovec-backed iov_iter instances are assumed to satisfy several properties:
* no more than UIO_MAXIOV elements in iovec array
* total size of all ranges is no more than MAX_RW_COUNT
* all ranges pass access_ok().
The problem is, invariants of data structures should be established in the
primitives creating those data structures, not in the code using those
primitives. And iov_iter_init() violates that principle. For a while we
managed to get away with that, but once the use of iov_iter started to
spread, it didn't take long for shit to hit the fan - missed check in
sys_sendto() had introduced a roothole.
We _do_ have primitives for importing and validating iovecs (both native and
compat ones) and those primitives are almost always followed by shoving the
resulting iovec into iov_iter. Life would be considerably simpler (and safer)
if we combined those primitives with initializing iov_iter.
That gives us two new primitives - import_iovec() and compat_import_iovec().
Calling conventions:
iovec = iov_array;
err = import_iovec(direction, uvec, nr_segs,
ARRAY_SIZE(iov_array), &iovec,
&iter);
imports user vector into kernel space (into iov_array if it fits, allocated
if it doesn't fit or if iovec was NULL), validates it and sets iter up to
refer to it. On success 0 is returned and allocated kernel copy (or NULL
if the array had fit into caller-supplied one) is returned via iovec.
On failure all allocations are undone and -E... is returned. If the total
size of ranges exceeds MAX_RW_COUNT, the excess is silently truncated.
compat_import_iovec() expects uvec to be a pointer to user array of compat_iovec;
otherwise it's identical to import_iovec().
Finally, import_single_range() sets iov_iter backed by single-element iovec
covering a user-supplied range -
err = import_single_range(direction, address, size, iovec, &iter);
does validation and sets iter up. Again, size in excess of MAX_RW_COUNT gets
silently truncated.
Next commits will be switching the things up to use of those and reducing
the amount of iov_iter_init() instances.
Signed-off-by: Al Viro <viro@zeniv.linux.org.uk>
2015-03-22 00:45:43 +03:00
|
|
|
unsigned nr_segs, unsigned fast_segs,
|
2020-09-25 07:51:40 +03:00
|
|
|
struct iovec **iovp, struct iov_iter *i)
|
saner iov_iter initialization primitives
iovec-backed iov_iter instances are assumed to satisfy several properties:
* no more than UIO_MAXIOV elements in iovec array
* total size of all ranges is no more than MAX_RW_COUNT
* all ranges pass access_ok().
The problem is, invariants of data structures should be established in the
primitives creating those data structures, not in the code using those
primitives. And iov_iter_init() violates that principle. For a while we
managed to get away with that, but once the use of iov_iter started to
spread, it didn't take long for shit to hit the fan - missed check in
sys_sendto() had introduced a roothole.
We _do_ have primitives for importing and validating iovecs (both native and
compat ones) and those primitives are almost always followed by shoving the
resulting iovec into iov_iter. Life would be considerably simpler (and safer)
if we combined those primitives with initializing iov_iter.
That gives us two new primitives - import_iovec() and compat_import_iovec().
Calling conventions:
iovec = iov_array;
err = import_iovec(direction, uvec, nr_segs,
ARRAY_SIZE(iov_array), &iovec,
&iter);
imports user vector into kernel space (into iov_array if it fits, allocated
if it doesn't fit or if iovec was NULL), validates it and sets iter up to
refer to it. On success 0 is returned and allocated kernel copy (or NULL
if the array had fit into caller-supplied one) is returned via iovec.
On failure all allocations are undone and -E... is returned. If the total
size of ranges exceeds MAX_RW_COUNT, the excess is silently truncated.
compat_import_iovec() expects uvec to be a pointer to user array of compat_iovec;
otherwise it's identical to import_iovec().
Finally, import_single_range() sets iov_iter backed by single-element iovec
covering a user-supplied range -
err = import_single_range(direction, address, size, iovec, &iter);
does validation and sets iter up. Again, size in excess of MAX_RW_COUNT gets
silently truncated.
Next commits will be switching the things up to use of those and reducing
the amount of iov_iter_init() instances.
Signed-off-by: Al Viro <viro@zeniv.linux.org.uk>
2015-03-22 00:45:43 +03:00
|
|
|
{
|
2020-09-25 07:51:41 +03:00
|
|
|
return __import_iovec(type, uvec, nr_segs, fast_segs, iovp, i,
|
|
|
|
in_compat_syscall());
|
saner iov_iter initialization primitives
iovec-backed iov_iter instances are assumed to satisfy several properties:
* no more than UIO_MAXIOV elements in iovec array
* total size of all ranges is no more than MAX_RW_COUNT
* all ranges pass access_ok().
The problem is, invariants of data structures should be established in the
primitives creating those data structures, not in the code using those
primitives. And iov_iter_init() violates that principle. For a while we
managed to get away with that, but once the use of iov_iter started to
spread, it didn't take long for shit to hit the fan - missed check in
sys_sendto() had introduced a roothole.
We _do_ have primitives for importing and validating iovecs (both native and
compat ones) and those primitives are almost always followed by shoving the
resulting iovec into iov_iter. Life would be considerably simpler (and safer)
if we combined those primitives with initializing iov_iter.
That gives us two new primitives - import_iovec() and compat_import_iovec().
Calling conventions:
iovec = iov_array;
err = import_iovec(direction, uvec, nr_segs,
ARRAY_SIZE(iov_array), &iovec,
&iter);
imports user vector into kernel space (into iov_array if it fits, allocated
if it doesn't fit or if iovec was NULL), validates it and sets iter up to
refer to it. On success 0 is returned and allocated kernel copy (or NULL
if the array had fit into caller-supplied one) is returned via iovec.
On failure all allocations are undone and -E... is returned. If the total
size of ranges exceeds MAX_RW_COUNT, the excess is silently truncated.
compat_import_iovec() expects uvec to be a pointer to user array of compat_iovec;
otherwise it's identical to import_iovec().
Finally, import_single_range() sets iov_iter backed by single-element iovec
covering a user-supplied range -
err = import_single_range(direction, address, size, iovec, &iter);
does validation and sets iter up. Again, size in excess of MAX_RW_COUNT gets
silently truncated.
Next commits will be switching the things up to use of those and reducing
the amount of iov_iter_init() instances.
Signed-off-by: Al Viro <viro@zeniv.linux.org.uk>
2015-03-22 00:45:43 +03:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(import_iovec);
|
|
|
|
|
|
|
|
int import_single_range(int rw, void __user *buf, size_t len,
|
|
|
|
struct iovec *iov, struct iov_iter *i)
|
|
|
|
{
|
|
|
|
if (len > MAX_RW_COUNT)
|
|
|
|
len = MAX_RW_COUNT;
|
Remove 'type' argument from access_ok() function
Nobody has actually used the type (VERIFY_READ vs VERIFY_WRITE) argument
of the user address range verification function since we got rid of the
old racy i386-only code to walk page tables by hand.
It existed because the original 80386 would not honor the write protect
bit when in kernel mode, so you had to do COW by hand before doing any
user access. But we haven't supported that in a long time, and these
days the 'type' argument is a purely historical artifact.
A discussion about extending 'user_access_begin()' to do the range
checking resulted this patch, because there is no way we're going to
move the old VERIFY_xyz interface to that model. And it's best done at
the end of the merge window when I've done most of my merges, so let's
just get this done once and for all.
This patch was mostly done with a sed-script, with manual fix-ups for
the cases that weren't of the trivial 'access_ok(VERIFY_xyz' form.
There were a couple of notable cases:
- csky still had the old "verify_area()" name as an alias.
- the iter_iov code had magical hardcoded knowledge of the actual
values of VERIFY_{READ,WRITE} (not that they mattered, since nothing
really used it)
- microblaze used the type argument for a debug printout
but other than those oddities this should be a total no-op patch.
I tried to fix up all architectures, did fairly extensive grepping for
access_ok() uses, and the changes are trivial, but I may have missed
something. Any missed conversion should be trivially fixable, though.
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2019-01-04 05:57:57 +03:00
|
|
|
if (unlikely(!access_ok(buf, len)))
|
saner iov_iter initialization primitives
iovec-backed iov_iter instances are assumed to satisfy several properties:
* no more than UIO_MAXIOV elements in iovec array
* total size of all ranges is no more than MAX_RW_COUNT
* all ranges pass access_ok().
The problem is, invariants of data structures should be established in the
primitives creating those data structures, not in the code using those
primitives. And iov_iter_init() violates that principle. For a while we
managed to get away with that, but once the use of iov_iter started to
spread, it didn't take long for shit to hit the fan - missed check in
sys_sendto() had introduced a roothole.
We _do_ have primitives for importing and validating iovecs (both native and
compat ones) and those primitives are almost always followed by shoving the
resulting iovec into iov_iter. Life would be considerably simpler (and safer)
if we combined those primitives with initializing iov_iter.
That gives us two new primitives - import_iovec() and compat_import_iovec().
Calling conventions:
iovec = iov_array;
err = import_iovec(direction, uvec, nr_segs,
ARRAY_SIZE(iov_array), &iovec,
&iter);
imports user vector into kernel space (into iov_array if it fits, allocated
if it doesn't fit or if iovec was NULL), validates it and sets iter up to
refer to it. On success 0 is returned and allocated kernel copy (or NULL
if the array had fit into caller-supplied one) is returned via iovec.
On failure all allocations are undone and -E... is returned. If the total
size of ranges exceeds MAX_RW_COUNT, the excess is silently truncated.
compat_import_iovec() expects uvec to be a pointer to user array of compat_iovec;
otherwise it's identical to import_iovec().
Finally, import_single_range() sets iov_iter backed by single-element iovec
covering a user-supplied range -
err = import_single_range(direction, address, size, iovec, &iter);
does validation and sets iter up. Again, size in excess of MAX_RW_COUNT gets
silently truncated.
Next commits will be switching the things up to use of those and reducing
the amount of iov_iter_init() instances.
Signed-off-by: Al Viro <viro@zeniv.linux.org.uk>
2015-03-22 00:45:43 +03:00
|
|
|
return -EFAULT;
|
|
|
|
|
2023-03-24 23:35:49 +03:00
|
|
|
iov_iter_ubuf(i, rw, buf, len);
|
saner iov_iter initialization primitives
iovec-backed iov_iter instances are assumed to satisfy several properties:
* no more than UIO_MAXIOV elements in iovec array
* total size of all ranges is no more than MAX_RW_COUNT
* all ranges pass access_ok().
The problem is, invariants of data structures should be established in the
primitives creating those data structures, not in the code using those
primitives. And iov_iter_init() violates that principle. For a while we
managed to get away with that, but once the use of iov_iter started to
spread, it didn't take long for shit to hit the fan - missed check in
sys_sendto() had introduced a roothole.
We _do_ have primitives for importing and validating iovecs (both native and
compat ones) and those primitives are almost always followed by shoving the
resulting iovec into iov_iter. Life would be considerably simpler (and safer)
if we combined those primitives with initializing iov_iter.
That gives us two new primitives - import_iovec() and compat_import_iovec().
Calling conventions:
iovec = iov_array;
err = import_iovec(direction, uvec, nr_segs,
ARRAY_SIZE(iov_array), &iovec,
&iter);
imports user vector into kernel space (into iov_array if it fits, allocated
if it doesn't fit or if iovec was NULL), validates it and sets iter up to
refer to it. On success 0 is returned and allocated kernel copy (or NULL
if the array had fit into caller-supplied one) is returned via iovec.
On failure all allocations are undone and -E... is returned. If the total
size of ranges exceeds MAX_RW_COUNT, the excess is silently truncated.
compat_import_iovec() expects uvec to be a pointer to user array of compat_iovec;
otherwise it's identical to import_iovec().
Finally, import_single_range() sets iov_iter backed by single-element iovec
covering a user-supplied range -
err = import_single_range(direction, address, size, iovec, &iter);
does validation and sets iter up. Again, size in excess of MAX_RW_COUNT gets
silently truncated.
Next commits will be switching the things up to use of those and reducing
the amount of iov_iter_init() instances.
Signed-off-by: Al Viro <viro@zeniv.linux.org.uk>
2015-03-22 00:45:43 +03:00
|
|
|
return 0;
|
|
|
|
}
|
2015-12-07 04:38:56 +03:00
|
|
|
EXPORT_SYMBOL(import_single_range);
|
2021-09-10 20:18:36 +03:00
|
|
|
|
2023-01-05 22:07:30 +03:00
|
|
|
int import_ubuf(int rw, void __user *buf, size_t len, struct iov_iter *i)
|
|
|
|
{
|
|
|
|
if (len > MAX_RW_COUNT)
|
|
|
|
len = MAX_RW_COUNT;
|
|
|
|
if (unlikely(!access_ok(buf, len)))
|
|
|
|
return -EFAULT;
|
|
|
|
|
|
|
|
iov_iter_ubuf(i, rw, buf, len);
|
|
|
|
return 0;
|
|
|
|
}
|
2023-08-15 22:01:12 +03:00
|
|
|
EXPORT_SYMBOL_GPL(import_ubuf);
|
2023-01-05 22:07:30 +03:00
|
|
|
|
2021-09-10 20:18:36 +03:00
|
|
|
/**
|
|
|
|
* iov_iter_restore() - Restore a &struct iov_iter to the same state as when
|
|
|
|
* iov_iter_save_state() was called.
|
|
|
|
*
|
|
|
|
* @i: &struct iov_iter to restore
|
|
|
|
* @state: state to restore from
|
|
|
|
*
|
|
|
|
* Used after iov_iter_save_state() to bring restore @i, if operations may
|
|
|
|
* have advanced it.
|
|
|
|
*
|
|
|
|
* Note: only works on ITER_IOVEC, ITER_BVEC, and ITER_KVEC
|
|
|
|
*/
|
|
|
|
void iov_iter_restore(struct iov_iter *i, struct iov_iter_state *state)
|
|
|
|
{
|
2023-01-05 22:07:33 +03:00
|
|
|
if (WARN_ON_ONCE(!iov_iter_is_bvec(i) && !iter_is_iovec(i) &&
|
|
|
|
!iter_is_ubuf(i)) && !iov_iter_is_kvec(i))
|
2021-09-10 20:18:36 +03:00
|
|
|
return;
|
|
|
|
i->iov_offset = state->iov_offset;
|
|
|
|
i->count = state->count;
|
2022-05-22 21:59:25 +03:00
|
|
|
if (iter_is_ubuf(i))
|
|
|
|
return;
|
2021-09-10 20:18:36 +03:00
|
|
|
/*
|
|
|
|
* For the *vec iters, nr_segs + iov is constant - if we increment
|
|
|
|
* the vec, then we also decrement the nr_segs count. Hence we don't
|
|
|
|
* need to track both of these, just one is enough and we can deduct
|
|
|
|
* the other from that. ITER_KVEC and ITER_IOVEC are the same struct
|
|
|
|
* size, so we can just increment the iov pointer as they are unionzed.
|
|
|
|
* ITER_BVEC _may_ be the same size on some archs, but on others it is
|
|
|
|
* not. Be safe and handle it separately.
|
|
|
|
*/
|
|
|
|
BUILD_BUG_ON(sizeof(struct iovec) != sizeof(struct kvec));
|
|
|
|
if (iov_iter_is_bvec(i))
|
|
|
|
i->bvec -= state->nr_segs - i->nr_segs;
|
|
|
|
else
|
2023-03-29 17:52:15 +03:00
|
|
|
i->__iov -= state->nr_segs - i->nr_segs;
|
2021-09-10 20:18:36 +03:00
|
|
|
i->nr_segs = state->nr_segs;
|
|
|
|
}
|
2022-10-28 23:50:30 +03:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Extract a list of contiguous pages from an ITER_XARRAY iterator. This does not
|
|
|
|
* get references on the pages, nor does it get a pin on them.
|
|
|
|
*/
|
|
|
|
static ssize_t iov_iter_extract_xarray_pages(struct iov_iter *i,
|
|
|
|
struct page ***pages, size_t maxsize,
|
|
|
|
unsigned int maxpages,
|
|
|
|
iov_iter_extraction_t extraction_flags,
|
|
|
|
size_t *offset0)
|
|
|
|
{
|
|
|
|
struct page *page, **p;
|
|
|
|
unsigned int nr = 0, offset;
|
|
|
|
loff_t pos = i->xarray_start + i->iov_offset;
|
|
|
|
pgoff_t index = pos >> PAGE_SHIFT;
|
|
|
|
XA_STATE(xas, i->xarray, index);
|
|
|
|
|
|
|
|
offset = pos & ~PAGE_MASK;
|
|
|
|
*offset0 = offset;
|
|
|
|
|
|
|
|
maxpages = want_pages_array(pages, maxsize, offset, maxpages);
|
|
|
|
if (!maxpages)
|
|
|
|
return -ENOMEM;
|
|
|
|
p = *pages;
|
|
|
|
|
|
|
|
rcu_read_lock();
|
|
|
|
for (page = xas_load(&xas); page; page = xas_next(&xas)) {
|
|
|
|
if (xas_retry(&xas, page))
|
|
|
|
continue;
|
|
|
|
|
|
|
|
/* Has the page moved or been split? */
|
|
|
|
if (unlikely(page != xas_reload(&xas))) {
|
|
|
|
xas_reset(&xas);
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
|
|
|
p[nr++] = find_subpage(page, xas.xa_index);
|
|
|
|
if (nr == maxpages)
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
rcu_read_unlock();
|
|
|
|
|
|
|
|
maxsize = min_t(size_t, nr * PAGE_SIZE - offset, maxsize);
|
|
|
|
iov_iter_advance(i, maxsize);
|
|
|
|
return maxsize;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Extract a list of contiguous pages from an ITER_BVEC iterator. This does
|
|
|
|
* not get references on the pages, nor does it get a pin on them.
|
|
|
|
*/
|
|
|
|
static ssize_t iov_iter_extract_bvec_pages(struct iov_iter *i,
|
|
|
|
struct page ***pages, size_t maxsize,
|
|
|
|
unsigned int maxpages,
|
|
|
|
iov_iter_extraction_t extraction_flags,
|
|
|
|
size_t *offset0)
|
|
|
|
{
|
|
|
|
struct page **p, *page;
|
|
|
|
size_t skip = i->iov_offset, offset;
|
|
|
|
int k;
|
|
|
|
|
|
|
|
for (;;) {
|
|
|
|
if (i->nr_segs == 0)
|
|
|
|
return 0;
|
|
|
|
maxsize = min(maxsize, i->bvec->bv_len - skip);
|
|
|
|
if (maxsize)
|
|
|
|
break;
|
|
|
|
i->iov_offset = 0;
|
|
|
|
i->nr_segs--;
|
|
|
|
i->bvec++;
|
|
|
|
skip = 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
skip += i->bvec->bv_offset;
|
|
|
|
page = i->bvec->bv_page + skip / PAGE_SIZE;
|
|
|
|
offset = skip % PAGE_SIZE;
|
|
|
|
*offset0 = offset;
|
|
|
|
|
|
|
|
maxpages = want_pages_array(pages, maxsize, offset, maxpages);
|
|
|
|
if (!maxpages)
|
|
|
|
return -ENOMEM;
|
|
|
|
p = *pages;
|
|
|
|
for (k = 0; k < maxpages; k++)
|
|
|
|
p[k] = page + k;
|
|
|
|
|
|
|
|
maxsize = min_t(size_t, maxsize, maxpages * PAGE_SIZE - offset);
|
|
|
|
iov_iter_advance(i, maxsize);
|
|
|
|
return maxsize;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Extract a list of virtually contiguous pages from an ITER_KVEC iterator.
|
|
|
|
* This does not get references on the pages, nor does it get a pin on them.
|
|
|
|
*/
|
|
|
|
static ssize_t iov_iter_extract_kvec_pages(struct iov_iter *i,
|
|
|
|
struct page ***pages, size_t maxsize,
|
|
|
|
unsigned int maxpages,
|
|
|
|
iov_iter_extraction_t extraction_flags,
|
|
|
|
size_t *offset0)
|
|
|
|
{
|
|
|
|
struct page **p, *page;
|
|
|
|
const void *kaddr;
|
|
|
|
size_t skip = i->iov_offset, offset, len;
|
|
|
|
int k;
|
|
|
|
|
|
|
|
for (;;) {
|
|
|
|
if (i->nr_segs == 0)
|
|
|
|
return 0;
|
|
|
|
maxsize = min(maxsize, i->kvec->iov_len - skip);
|
|
|
|
if (maxsize)
|
|
|
|
break;
|
|
|
|
i->iov_offset = 0;
|
|
|
|
i->nr_segs--;
|
|
|
|
i->kvec++;
|
|
|
|
skip = 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
kaddr = i->kvec->iov_base + skip;
|
|
|
|
offset = (unsigned long)kaddr & ~PAGE_MASK;
|
|
|
|
*offset0 = offset;
|
|
|
|
|
|
|
|
maxpages = want_pages_array(pages, maxsize, offset, maxpages);
|
|
|
|
if (!maxpages)
|
|
|
|
return -ENOMEM;
|
|
|
|
p = *pages;
|
|
|
|
|
|
|
|
kaddr -= offset;
|
|
|
|
len = offset + maxsize;
|
|
|
|
for (k = 0; k < maxpages; k++) {
|
|
|
|
size_t seg = min_t(size_t, len, PAGE_SIZE);
|
|
|
|
|
|
|
|
if (is_vmalloc_or_module_addr(kaddr))
|
|
|
|
page = vmalloc_to_page(kaddr);
|
|
|
|
else
|
|
|
|
page = virt_to_page(kaddr);
|
|
|
|
|
|
|
|
p[k] = page;
|
|
|
|
len -= seg;
|
|
|
|
kaddr += PAGE_SIZE;
|
|
|
|
}
|
|
|
|
|
|
|
|
maxsize = min_t(size_t, maxsize, maxpages * PAGE_SIZE - offset);
|
|
|
|
iov_iter_advance(i, maxsize);
|
|
|
|
return maxsize;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Extract a list of contiguous pages from a user iterator and get a pin on
|
|
|
|
* each of them. This should only be used if the iterator is user-backed
|
|
|
|
* (IOBUF/UBUF).
|
|
|
|
*
|
|
|
|
* It does not get refs on the pages, but the pages must be unpinned by the
|
|
|
|
* caller once the transfer is complete.
|
|
|
|
*
|
|
|
|
* This is safe to be used where background IO/DMA *is* going to be modifying
|
|
|
|
* the buffer; using a pin rather than a ref makes forces fork() to give the
|
|
|
|
* child a copy of the page.
|
|
|
|
*/
|
|
|
|
static ssize_t iov_iter_extract_user_pages(struct iov_iter *i,
|
|
|
|
struct page ***pages,
|
|
|
|
size_t maxsize,
|
|
|
|
unsigned int maxpages,
|
|
|
|
iov_iter_extraction_t extraction_flags,
|
|
|
|
size_t *offset0)
|
|
|
|
{
|
|
|
|
unsigned long addr;
|
|
|
|
unsigned int gup_flags = 0;
|
|
|
|
size_t offset;
|
|
|
|
int res;
|
|
|
|
|
|
|
|
if (i->data_source == ITER_DEST)
|
|
|
|
gup_flags |= FOLL_WRITE;
|
|
|
|
if (extraction_flags & ITER_ALLOW_P2PDMA)
|
|
|
|
gup_flags |= FOLL_PCI_P2PDMA;
|
|
|
|
if (i->nofault)
|
|
|
|
gup_flags |= FOLL_NOFAULT;
|
|
|
|
|
|
|
|
addr = first_iovec_segment(i, &maxsize);
|
|
|
|
*offset0 = offset = addr % PAGE_SIZE;
|
|
|
|
addr &= PAGE_MASK;
|
|
|
|
maxpages = want_pages_array(pages, maxsize, offset, maxpages);
|
|
|
|
if (!maxpages)
|
|
|
|
return -ENOMEM;
|
|
|
|
res = pin_user_pages_fast(addr, maxpages, gup_flags, *pages);
|
|
|
|
if (unlikely(res <= 0))
|
|
|
|
return res;
|
|
|
|
maxsize = min_t(size_t, maxsize, res * PAGE_SIZE - offset);
|
|
|
|
iov_iter_advance(i, maxsize);
|
|
|
|
return maxsize;
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* iov_iter_extract_pages - Extract a list of contiguous pages from an iterator
|
|
|
|
* @i: The iterator to extract from
|
|
|
|
* @pages: Where to return the list of pages
|
|
|
|
* @maxsize: The maximum amount of iterator to extract
|
|
|
|
* @maxpages: The maximum size of the list of pages
|
|
|
|
* @extraction_flags: Flags to qualify request
|
|
|
|
* @offset0: Where to return the starting offset into (*@pages)[0]
|
|
|
|
*
|
|
|
|
* Extract a list of contiguous pages from the current point of the iterator,
|
|
|
|
* advancing the iterator. The maximum number of pages and the maximum amount
|
|
|
|
* of page contents can be set.
|
|
|
|
*
|
|
|
|
* If *@pages is NULL, a page list will be allocated to the required size and
|
|
|
|
* *@pages will be set to its base. If *@pages is not NULL, it will be assumed
|
|
|
|
* that the caller allocated a page list at least @maxpages in size and this
|
|
|
|
* will be filled in.
|
|
|
|
*
|
|
|
|
* @extraction_flags can have ITER_ALLOW_P2PDMA set to request peer-to-peer DMA
|
|
|
|
* be allowed on the pages extracted.
|
|
|
|
*
|
|
|
|
* The iov_iter_extract_will_pin() function can be used to query how cleanup
|
|
|
|
* should be performed.
|
|
|
|
*
|
|
|
|
* Extra refs or pins on the pages may be obtained as follows:
|
|
|
|
*
|
|
|
|
* (*) If the iterator is user-backed (ITER_IOVEC/ITER_UBUF), pins will be
|
|
|
|
* added to the pages, but refs will not be taken.
|
|
|
|
* iov_iter_extract_will_pin() will return true.
|
|
|
|
*
|
|
|
|
* (*) If the iterator is ITER_KVEC, ITER_BVEC or ITER_XARRAY, the pages are
|
|
|
|
* merely listed; no extra refs or pins are obtained.
|
|
|
|
* iov_iter_extract_will_pin() will return 0.
|
|
|
|
*
|
|
|
|
* Note also:
|
|
|
|
*
|
|
|
|
* (*) Use with ITER_DISCARD is not supported as that has no content.
|
|
|
|
*
|
|
|
|
* On success, the function sets *@pages to the new pagelist, if allocated, and
|
|
|
|
* sets *offset0 to the offset into the first page.
|
|
|
|
*
|
|
|
|
* It may also return -ENOMEM and -EFAULT.
|
|
|
|
*/
|
|
|
|
ssize_t iov_iter_extract_pages(struct iov_iter *i,
|
|
|
|
struct page ***pages,
|
|
|
|
size_t maxsize,
|
|
|
|
unsigned int maxpages,
|
|
|
|
iov_iter_extraction_t extraction_flags,
|
|
|
|
size_t *offset0)
|
|
|
|
{
|
|
|
|
maxsize = min_t(size_t, min_t(size_t, maxsize, i->count), MAX_RW_COUNT);
|
|
|
|
if (!maxsize)
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
if (likely(user_backed_iter(i)))
|
|
|
|
return iov_iter_extract_user_pages(i, pages, maxsize,
|
|
|
|
maxpages, extraction_flags,
|
|
|
|
offset0);
|
|
|
|
if (iov_iter_is_kvec(i))
|
|
|
|
return iov_iter_extract_kvec_pages(i, pages, maxsize,
|
|
|
|
maxpages, extraction_flags,
|
|
|
|
offset0);
|
|
|
|
if (iov_iter_is_bvec(i))
|
|
|
|
return iov_iter_extract_bvec_pages(i, pages, maxsize,
|
|
|
|
maxpages, extraction_flags,
|
|
|
|
offset0);
|
|
|
|
if (iov_iter_is_xarray(i))
|
|
|
|
return iov_iter_extract_xarray_pages(i, pages, maxsize,
|
|
|
|
maxpages, extraction_flags,
|
|
|
|
offset0);
|
|
|
|
return -EFAULT;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(iov_iter_extract_pages);
|