mirror of
https://github.com/torvalds/linux.git
synced 2025-12-01 07:26:02 +07:00
selftests/mm/uffd: refactor non-composite global vars into struct
Refactor macros and non-composite global variable definitions into a struct that is defined at the start of a test and is passed around instead of relying on global vars. Link: https://lkml.kernel.org/r/20250829155600.2000-1-ujwal.kundur@gmail.com Signed-off-by: Ujwal Kundur <ujwal.kundur@gmail.com> Acked-by: Peter Xu <peterx@redhat.com> Reviewed-by: Brendan Jackman <jackmanb@google.com> Cc: David Hildenbrand <david@redhat.com> Cc: Liam Howlett <liam.howlett@oracle.com> Cc: Lorenzo Stoakes <lorenzo.stoakes@oracle.com> Cc: Michal Hocko <mhocko@suse.com> Cc: Mike Rapoport <rppt@kernel.org> Cc: Shuah Khan <shuah@kernel.org> Cc: Suren Baghdasaryan <surenb@google.com> Cc: Vlastimil Babka <vbabka@suse.cz> Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
This commit is contained in:
committed by
Andrew Morton
parent
2f5bd89ba9
commit
4dfd4bba85
@@ -7,18 +7,29 @@
|
||||
|
||||
#include "uffd-common.h"
|
||||
|
||||
#define BASE_PMD_ADDR ((void *)(1UL << 30))
|
||||
|
||||
volatile bool test_uffdio_copy_eexist = true;
|
||||
unsigned long nr_parallel, nr_pages, nr_pages_per_cpu, page_size;
|
||||
char *area_src, *area_src_alias, *area_dst, *area_dst_alias, *area_remap;
|
||||
int uffd = -1, uffd_flags, finished, *pipefd, test_type;
|
||||
bool map_shared;
|
||||
bool test_uffdio_wp = true;
|
||||
unsigned long long *count_verify;
|
||||
uffd_test_ops_t *uffd_test_ops;
|
||||
uffd_test_case_ops_t *uffd_test_case_ops;
|
||||
atomic_bool ready_for_fork;
|
||||
|
||||
#define BASE_PMD_ADDR ((void *)(1UL << 30))
|
||||
|
||||
/* pthread_mutex_t starts at page offset 0 */
|
||||
pthread_mutex_t *area_mutex(char *area, unsigned long nr, uffd_global_test_opts_t *gopts)
|
||||
{
|
||||
return (pthread_mutex_t *) (area + nr * gopts->page_size);
|
||||
}
|
||||
|
||||
/*
|
||||
* count is placed in the page after pthread_mutex_t naturally aligned
|
||||
* to avoid non alignment faults on non-x86 archs.
|
||||
*/
|
||||
volatile unsigned long long *area_count(char *area, unsigned long nr,
|
||||
uffd_global_test_opts_t *gopts)
|
||||
{
|
||||
return (volatile unsigned long long *)
|
||||
((unsigned long)(area + nr * gopts->page_size +
|
||||
sizeof(pthread_mutex_t) + sizeof(unsigned long long) - 1) &
|
||||
~(unsigned long)(sizeof(unsigned long long) - 1));
|
||||
}
|
||||
|
||||
static int uffd_mem_fd_create(off_t mem_size, bool hugetlb)
|
||||
{
|
||||
@@ -40,15 +51,15 @@ static int uffd_mem_fd_create(off_t mem_size, bool hugetlb)
|
||||
return mem_fd;
|
||||
}
|
||||
|
||||
static void anon_release_pages(char *rel_area)
|
||||
static void anon_release_pages(uffd_global_test_opts_t *gopts, char *rel_area)
|
||||
{
|
||||
if (madvise(rel_area, nr_pages * page_size, MADV_DONTNEED))
|
||||
if (madvise(rel_area, gopts->nr_pages * gopts->page_size, MADV_DONTNEED))
|
||||
err("madvise(MADV_DONTNEED) failed");
|
||||
}
|
||||
|
||||
static int anon_allocate_area(void **alloc_area, bool is_src)
|
||||
static int anon_allocate_area(uffd_global_test_opts_t *gopts, void **alloc_area, bool is_src)
|
||||
{
|
||||
*alloc_area = mmap(NULL, nr_pages * page_size, PROT_READ | PROT_WRITE,
|
||||
*alloc_area = mmap(NULL, gopts->nr_pages * gopts->page_size, PROT_READ | PROT_WRITE,
|
||||
MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
|
||||
if (*alloc_area == MAP_FAILED) {
|
||||
*alloc_area = NULL;
|
||||
@@ -57,31 +68,32 @@ static int anon_allocate_area(void **alloc_area, bool is_src)
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void noop_alias_mapping(__u64 *start, size_t len, unsigned long offset)
|
||||
static void noop_alias_mapping(uffd_global_test_opts_t *gopts, __u64 *start,
|
||||
size_t len, unsigned long offset)
|
||||
{
|
||||
}
|
||||
|
||||
static void hugetlb_release_pages(char *rel_area)
|
||||
static void hugetlb_release_pages(uffd_global_test_opts_t *gopts, char *rel_area)
|
||||
{
|
||||
if (!map_shared) {
|
||||
if (madvise(rel_area, nr_pages * page_size, MADV_DONTNEED))
|
||||
if (!gopts->map_shared) {
|
||||
if (madvise(rel_area, gopts->nr_pages * gopts->page_size, MADV_DONTNEED))
|
||||
err("madvise(MADV_DONTNEED) failed");
|
||||
} else {
|
||||
if (madvise(rel_area, nr_pages * page_size, MADV_REMOVE))
|
||||
if (madvise(rel_area, gopts->nr_pages * gopts->page_size, MADV_REMOVE))
|
||||
err("madvise(MADV_REMOVE) failed");
|
||||
}
|
||||
}
|
||||
|
||||
static int hugetlb_allocate_area(void **alloc_area, bool is_src)
|
||||
static int hugetlb_allocate_area(uffd_global_test_opts_t *gopts, void **alloc_area, bool is_src)
|
||||
{
|
||||
off_t size = nr_pages * page_size;
|
||||
off_t size = gopts->nr_pages * gopts->page_size;
|
||||
off_t offset = is_src ? 0 : size;
|
||||
void *area_alias = NULL;
|
||||
char **alloc_area_alias;
|
||||
int mem_fd = uffd_mem_fd_create(size * 2, true);
|
||||
|
||||
*alloc_area = mmap(NULL, size, PROT_READ | PROT_WRITE,
|
||||
(map_shared ? MAP_SHARED : MAP_PRIVATE) |
|
||||
(gopts->map_shared ? MAP_SHARED : MAP_PRIVATE) |
|
||||
(is_src ? 0 : MAP_NORESERVE),
|
||||
mem_fd, offset);
|
||||
if (*alloc_area == MAP_FAILED) {
|
||||
@@ -89,7 +101,7 @@ static int hugetlb_allocate_area(void **alloc_area, bool is_src)
|
||||
return -errno;
|
||||
}
|
||||
|
||||
if (map_shared) {
|
||||
if (gopts->map_shared) {
|
||||
area_alias = mmap(NULL, size, PROT_READ | PROT_WRITE,
|
||||
MAP_SHARED, mem_fd, offset);
|
||||
if (area_alias == MAP_FAILED)
|
||||
@@ -97,9 +109,9 @@ static int hugetlb_allocate_area(void **alloc_area, bool is_src)
|
||||
}
|
||||
|
||||
if (is_src) {
|
||||
alloc_area_alias = &area_src_alias;
|
||||
alloc_area_alias = &gopts->area_src_alias;
|
||||
} else {
|
||||
alloc_area_alias = &area_dst_alias;
|
||||
alloc_area_alias = &gopts->area_dst_alias;
|
||||
}
|
||||
if (area_alias)
|
||||
*alloc_area_alias = area_alias;
|
||||
@@ -108,24 +120,25 @@ static int hugetlb_allocate_area(void **alloc_area, bool is_src)
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void hugetlb_alias_mapping(__u64 *start, size_t len, unsigned long offset)
|
||||
static void hugetlb_alias_mapping(uffd_global_test_opts_t *gopts, __u64 *start,
|
||||
size_t len, unsigned long offset)
|
||||
{
|
||||
if (!map_shared)
|
||||
if (!gopts->map_shared)
|
||||
return;
|
||||
|
||||
*start = (unsigned long) area_dst_alias + offset;
|
||||
*start = (unsigned long) gopts->area_dst_alias + offset;
|
||||
}
|
||||
|
||||
static void shmem_release_pages(char *rel_area)
|
||||
static void shmem_release_pages(uffd_global_test_opts_t *gopts, char *rel_area)
|
||||
{
|
||||
if (madvise(rel_area, nr_pages * page_size, MADV_REMOVE))
|
||||
if (madvise(rel_area, gopts->nr_pages * gopts->page_size, MADV_REMOVE))
|
||||
err("madvise(MADV_REMOVE) failed");
|
||||
}
|
||||
|
||||
static int shmem_allocate_area(void **alloc_area, bool is_src)
|
||||
static int shmem_allocate_area(uffd_global_test_opts_t *gopts, void **alloc_area, bool is_src)
|
||||
{
|
||||
void *area_alias = NULL;
|
||||
size_t bytes = nr_pages * page_size, hpage_size = read_pmd_pagesize();
|
||||
size_t bytes = gopts->nr_pages * gopts->page_size, hpage_size = read_pmd_pagesize();
|
||||
unsigned long offset = is_src ? 0 : bytes;
|
||||
char *p = NULL, *p_alias = NULL;
|
||||
int mem_fd = uffd_mem_fd_create(bytes * 2, false);
|
||||
@@ -159,22 +172,23 @@ static int shmem_allocate_area(void **alloc_area, bool is_src)
|
||||
err("mmap of anonymous memory failed at %p", p_alias);
|
||||
|
||||
if (is_src)
|
||||
area_src_alias = area_alias;
|
||||
gopts->area_src_alias = area_alias;
|
||||
else
|
||||
area_dst_alias = area_alias;
|
||||
gopts->area_dst_alias = area_alias;
|
||||
|
||||
close(mem_fd);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void shmem_alias_mapping(__u64 *start, size_t len, unsigned long offset)
|
||||
static void shmem_alias_mapping(uffd_global_test_opts_t *gopts, __u64 *start,
|
||||
size_t len, unsigned long offset)
|
||||
{
|
||||
*start = (unsigned long)area_dst_alias + offset;
|
||||
*start = (unsigned long)gopts->area_dst_alias + offset;
|
||||
}
|
||||
|
||||
static void shmem_check_pmd_mapping(void *p, int expect_nr_hpages)
|
||||
static void shmem_check_pmd_mapping(uffd_global_test_opts_t *gopts, void *p, int expect_nr_hpages)
|
||||
{
|
||||
if (!check_huge_shmem(area_dst_alias, expect_nr_hpages,
|
||||
if (!check_huge_shmem(gopts->area_dst_alias, expect_nr_hpages,
|
||||
read_pmd_pagesize()))
|
||||
err("Did not find expected %d number of hugepages",
|
||||
expect_nr_hpages);
|
||||
@@ -234,18 +248,18 @@ void uffd_stats_report(struct uffd_args *args, int n_cpus)
|
||||
printf("\n");
|
||||
}
|
||||
|
||||
int userfaultfd_open(uint64_t *features)
|
||||
int userfaultfd_open(uffd_global_test_opts_t *gopts, uint64_t *features)
|
||||
{
|
||||
struct uffdio_api uffdio_api;
|
||||
|
||||
uffd = uffd_open(UFFD_FLAGS);
|
||||
if (uffd < 0)
|
||||
gopts->uffd = uffd_open(UFFD_FLAGS);
|
||||
if (gopts->uffd < 0)
|
||||
return -1;
|
||||
uffd_flags = fcntl(uffd, F_GETFD, NULL);
|
||||
gopts->uffd_flags = fcntl(gopts->uffd, F_GETFD, NULL);
|
||||
|
||||
uffdio_api.api = UFFD_API;
|
||||
uffdio_api.features = *features;
|
||||
if (ioctl(uffd, UFFDIO_API, &uffdio_api))
|
||||
if (ioctl(gopts->uffd, UFFDIO_API, &uffdio_api))
|
||||
/* Probably lack of CAP_PTRACE? */
|
||||
return -1;
|
||||
if (uffdio_api.api != UFFD_API)
|
||||
@@ -255,59 +269,63 @@ int userfaultfd_open(uint64_t *features)
|
||||
return 0;
|
||||
}
|
||||
|
||||
static inline void munmap_area(void **area)
|
||||
static inline void munmap_area(uffd_global_test_opts_t *gopts, void **area)
|
||||
{
|
||||
if (*area)
|
||||
if (munmap(*area, nr_pages * page_size))
|
||||
if (munmap(*area, gopts->nr_pages * gopts->page_size))
|
||||
err("munmap");
|
||||
|
||||
*area = NULL;
|
||||
}
|
||||
|
||||
void uffd_test_ctx_clear(void)
|
||||
void uffd_test_ctx_clear(uffd_global_test_opts_t *gopts)
|
||||
{
|
||||
size_t i;
|
||||
|
||||
if (pipefd) {
|
||||
for (i = 0; i < nr_parallel * 2; ++i) {
|
||||
if (close(pipefd[i]))
|
||||
if (gopts->pipefd) {
|
||||
for (i = 0; i < gopts->nr_parallel * 2; ++i) {
|
||||
if (close(gopts->pipefd[i]))
|
||||
err("close pipefd");
|
||||
}
|
||||
free(pipefd);
|
||||
pipefd = NULL;
|
||||
free(gopts->pipefd);
|
||||
gopts->pipefd = NULL;
|
||||
}
|
||||
|
||||
if (count_verify) {
|
||||
free(count_verify);
|
||||
count_verify = NULL;
|
||||
if (gopts->count_verify) {
|
||||
free(gopts->count_verify);
|
||||
gopts->count_verify = NULL;
|
||||
}
|
||||
|
||||
if (uffd != -1) {
|
||||
if (close(uffd))
|
||||
if (gopts->uffd != -1) {
|
||||
if (close(gopts->uffd))
|
||||
err("close uffd");
|
||||
uffd = -1;
|
||||
gopts->uffd = -1;
|
||||
}
|
||||
|
||||
munmap_area((void **)&area_src);
|
||||
munmap_area((void **)&area_src_alias);
|
||||
munmap_area((void **)&area_dst);
|
||||
munmap_area((void **)&area_dst_alias);
|
||||
munmap_area((void **)&area_remap);
|
||||
munmap_area(gopts, (void **)&gopts->area_src);
|
||||
munmap_area(gopts, (void **)&gopts->area_src_alias);
|
||||
munmap_area(gopts, (void **)&gopts->area_dst);
|
||||
munmap_area(gopts, (void **)&gopts->area_dst_alias);
|
||||
munmap_area(gopts, (void **)&gopts->area_remap);
|
||||
}
|
||||
|
||||
int uffd_test_ctx_init(uint64_t features, const char **errmsg)
|
||||
int uffd_test_ctx_init(uffd_global_test_opts_t *gopts, uint64_t features, const char **errmsg)
|
||||
{
|
||||
unsigned long nr, cpu;
|
||||
int ret;
|
||||
|
||||
gopts->area_src_alias = NULL;
|
||||
gopts->area_dst_alias = NULL;
|
||||
gopts->area_remap = NULL;
|
||||
|
||||
if (uffd_test_case_ops && uffd_test_case_ops->pre_alloc) {
|
||||
ret = uffd_test_case_ops->pre_alloc(errmsg);
|
||||
ret = uffd_test_case_ops->pre_alloc(gopts, errmsg);
|
||||
if (ret)
|
||||
return ret;
|
||||
}
|
||||
|
||||
ret = uffd_test_ops->allocate_area((void **)&area_src, true);
|
||||
ret |= uffd_test_ops->allocate_area((void **)&area_dst, false);
|
||||
ret = uffd_test_ops->allocate_area(gopts, (void **) &gopts->area_src, true);
|
||||
ret |= uffd_test_ops->allocate_area(gopts, (void **) &gopts->area_dst, false);
|
||||
if (ret) {
|
||||
if (errmsg)
|
||||
*errmsg = "memory allocation failed";
|
||||
@@ -315,26 +333,26 @@ int uffd_test_ctx_init(uint64_t features, const char **errmsg)
|
||||
}
|
||||
|
||||
if (uffd_test_case_ops && uffd_test_case_ops->post_alloc) {
|
||||
ret = uffd_test_case_ops->post_alloc(errmsg);
|
||||
ret = uffd_test_case_ops->post_alloc(gopts, errmsg);
|
||||
if (ret)
|
||||
return ret;
|
||||
}
|
||||
|
||||
ret = userfaultfd_open(&features);
|
||||
ret = userfaultfd_open(gopts, &features);
|
||||
if (ret) {
|
||||
if (errmsg)
|
||||
*errmsg = "possible lack of privilege";
|
||||
return ret;
|
||||
}
|
||||
|
||||
count_verify = malloc(nr_pages * sizeof(unsigned long long));
|
||||
if (!count_verify)
|
||||
gopts->count_verify = malloc(gopts->nr_pages * sizeof(unsigned long long));
|
||||
if (!gopts->count_verify)
|
||||
err("count_verify");
|
||||
|
||||
for (nr = 0; nr < nr_pages; nr++) {
|
||||
*area_mutex(area_src, nr) =
|
||||
for (nr = 0; nr < gopts->nr_pages; nr++) {
|
||||
*area_mutex(gopts->area_src, nr, gopts) =
|
||||
(pthread_mutex_t)PTHREAD_MUTEX_INITIALIZER;
|
||||
count_verify[nr] = *area_count(area_src, nr) = 1;
|
||||
gopts->count_verify[nr] = *area_count(gopts->area_src, nr, gopts) = 1;
|
||||
/*
|
||||
* In the transition between 255 to 256, powerpc will
|
||||
* read out of order in my_bcmp and see both bytes as
|
||||
@@ -342,7 +360,7 @@ int uffd_test_ctx_init(uint64_t features, const char **errmsg)
|
||||
* after the count, to avoid my_bcmp to trigger false
|
||||
* positives.
|
||||
*/
|
||||
*(area_count(area_src, nr) + 1) = 1;
|
||||
*(area_count(gopts->area_src, nr, gopts) + 1) = 1;
|
||||
}
|
||||
|
||||
/*
|
||||
@@ -363,13 +381,13 @@ int uffd_test_ctx_init(uint64_t features, const char **errmsg)
|
||||
* proactively split the thp and drop any accidentally initialized
|
||||
* pages within area_dst.
|
||||
*/
|
||||
uffd_test_ops->release_pages(area_dst);
|
||||
uffd_test_ops->release_pages(gopts, gopts->area_dst);
|
||||
|
||||
pipefd = malloc(sizeof(int) * nr_parallel * 2);
|
||||
if (!pipefd)
|
||||
gopts->pipefd = malloc(sizeof(int) * gopts->nr_parallel * 2);
|
||||
if (!gopts->pipefd)
|
||||
err("pipefd");
|
||||
for (cpu = 0; cpu < nr_parallel; cpu++)
|
||||
if (pipe2(&pipefd[cpu * 2], O_CLOEXEC | O_NONBLOCK))
|
||||
for (cpu = 0; cpu < gopts->nr_parallel; cpu++)
|
||||
if (pipe2(&gopts->pipefd[cpu * 2], O_CLOEXEC | O_NONBLOCK))
|
||||
err("pipe");
|
||||
|
||||
return 0;
|
||||
@@ -416,9 +434,9 @@ static void continue_range(int ufd, __u64 start, __u64 len, bool wp)
|
||||
ret, (int64_t) req.mapped);
|
||||
}
|
||||
|
||||
int uffd_read_msg(int ufd, struct uffd_msg *msg)
|
||||
int uffd_read_msg(uffd_global_test_opts_t *gopts, struct uffd_msg *msg)
|
||||
{
|
||||
int ret = read(uffd, msg, sizeof(*msg));
|
||||
int ret = read(gopts->uffd, msg, sizeof(*msg));
|
||||
|
||||
if (ret != sizeof(*msg)) {
|
||||
if (ret < 0) {
|
||||
@@ -433,7 +451,8 @@ int uffd_read_msg(int ufd, struct uffd_msg *msg)
|
||||
return 0;
|
||||
}
|
||||
|
||||
void uffd_handle_page_fault(struct uffd_msg *msg, struct uffd_args *args)
|
||||
void uffd_handle_page_fault(uffd_global_test_opts_t *gopts, struct uffd_msg *msg,
|
||||
struct uffd_args *args)
|
||||
{
|
||||
unsigned long offset;
|
||||
|
||||
@@ -442,7 +461,7 @@ void uffd_handle_page_fault(struct uffd_msg *msg, struct uffd_args *args)
|
||||
|
||||
if (msg->arg.pagefault.flags & UFFD_PAGEFAULT_FLAG_WP) {
|
||||
/* Write protect page faults */
|
||||
wp_range(uffd, msg->arg.pagefault.address, page_size, false);
|
||||
wp_range(gopts->uffd, msg->arg.pagefault.address, gopts->page_size, false);
|
||||
args->wp_faults++;
|
||||
} else if (msg->arg.pagefault.flags & UFFD_PAGEFAULT_FLAG_MINOR) {
|
||||
uint8_t *area;
|
||||
@@ -460,12 +479,12 @@ void uffd_handle_page_fault(struct uffd_msg *msg, struct uffd_args *args)
|
||||
* (UFFD-registered).
|
||||
*/
|
||||
|
||||
area = (uint8_t *)(area_dst +
|
||||
((char *)msg->arg.pagefault.address -
|
||||
area_dst_alias));
|
||||
for (b = 0; b < page_size; ++b)
|
||||
area = (uint8_t *)(gopts->area_dst +
|
||||
((char *)msg->arg.pagefault.address -
|
||||
gopts->area_dst_alias));
|
||||
for (b = 0; b < gopts->page_size; ++b)
|
||||
area[b] = ~area[b];
|
||||
continue_range(uffd, msg->arg.pagefault.address, page_size,
|
||||
continue_range(gopts->uffd, msg->arg.pagefault.address, gopts->page_size,
|
||||
args->apply_wp);
|
||||
args->minor_faults++;
|
||||
} else {
|
||||
@@ -493,10 +512,10 @@ void uffd_handle_page_fault(struct uffd_msg *msg, struct uffd_args *args)
|
||||
if (msg->arg.pagefault.flags & UFFD_PAGEFAULT_FLAG_WRITE)
|
||||
err("unexpected write fault");
|
||||
|
||||
offset = (char *)(unsigned long)msg->arg.pagefault.address - area_dst;
|
||||
offset &= ~(page_size-1);
|
||||
offset = (char *)(unsigned long)msg->arg.pagefault.address - gopts->area_dst;
|
||||
offset &= ~(gopts->page_size-1);
|
||||
|
||||
if (copy_page(uffd, offset, args->apply_wp))
|
||||
if (copy_page(gopts, offset, args->apply_wp))
|
||||
args->missing_faults++;
|
||||
}
|
||||
}
|
||||
@@ -504,6 +523,7 @@ void uffd_handle_page_fault(struct uffd_msg *msg, struct uffd_args *args)
|
||||
void *uffd_poll_thread(void *arg)
|
||||
{
|
||||
struct uffd_args *args = (struct uffd_args *)arg;
|
||||
uffd_global_test_opts_t *gopts = args->gopts;
|
||||
unsigned long cpu = args->cpu;
|
||||
struct pollfd pollfd[2];
|
||||
struct uffd_msg msg;
|
||||
@@ -514,12 +534,12 @@ void *uffd_poll_thread(void *arg)
|
||||
if (!args->handle_fault)
|
||||
args->handle_fault = uffd_handle_page_fault;
|
||||
|
||||
pollfd[0].fd = uffd;
|
||||
pollfd[0].fd = gopts->uffd;
|
||||
pollfd[0].events = POLLIN;
|
||||
pollfd[1].fd = pipefd[cpu*2];
|
||||
pollfd[1].fd = gopts->pipefd[cpu*2];
|
||||
pollfd[1].events = POLLIN;
|
||||
|
||||
ready_for_fork = true;
|
||||
gopts->ready_for_fork = true;
|
||||
|
||||
for (;;) {
|
||||
ret = poll(pollfd, 2, -1);
|
||||
@@ -537,30 +557,30 @@ void *uffd_poll_thread(void *arg)
|
||||
}
|
||||
if (!(pollfd[0].revents & POLLIN))
|
||||
err("pollfd[0].revents %d", pollfd[0].revents);
|
||||
if (uffd_read_msg(uffd, &msg))
|
||||
if (uffd_read_msg(gopts, &msg))
|
||||
continue;
|
||||
switch (msg.event) {
|
||||
default:
|
||||
err("unexpected msg event %u\n", msg.event);
|
||||
break;
|
||||
case UFFD_EVENT_PAGEFAULT:
|
||||
args->handle_fault(&msg, args);
|
||||
args->handle_fault(gopts, &msg, args);
|
||||
break;
|
||||
case UFFD_EVENT_FORK:
|
||||
close(uffd);
|
||||
uffd = msg.arg.fork.ufd;
|
||||
pollfd[0].fd = uffd;
|
||||
close(gopts->uffd);
|
||||
gopts->uffd = msg.arg.fork.ufd;
|
||||
pollfd[0].fd = gopts->uffd;
|
||||
break;
|
||||
case UFFD_EVENT_REMOVE:
|
||||
uffd_reg.range.start = msg.arg.remove.start;
|
||||
uffd_reg.range.len = msg.arg.remove.end -
|
||||
msg.arg.remove.start;
|
||||
if (ioctl(uffd, UFFDIO_UNREGISTER, &uffd_reg.range))
|
||||
if (ioctl(gopts->uffd, UFFDIO_UNREGISTER, &uffd_reg.range))
|
||||
err("remove failure");
|
||||
break;
|
||||
case UFFD_EVENT_REMAP:
|
||||
area_remap = area_dst; /* save for later unmap */
|
||||
area_dst = (char *)(unsigned long)msg.arg.remap.to;
|
||||
gopts->area_remap = gopts->area_dst; /* save for later unmap */
|
||||
gopts->area_dst = (char *)(unsigned long)msg.arg.remap.to;
|
||||
break;
|
||||
}
|
||||
}
|
||||
@@ -568,17 +588,18 @@ void *uffd_poll_thread(void *arg)
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static void retry_copy_page(int ufd, struct uffdio_copy *uffdio_copy,
|
||||
static void retry_copy_page(uffd_global_test_opts_t *gopts, struct uffdio_copy *uffdio_copy,
|
||||
unsigned long offset)
|
||||
{
|
||||
uffd_test_ops->alias_mapping(&uffdio_copy->dst,
|
||||
uffd_test_ops->alias_mapping(gopts,
|
||||
&uffdio_copy->dst,
|
||||
uffdio_copy->len,
|
||||
offset);
|
||||
if (ioctl(ufd, UFFDIO_COPY, uffdio_copy)) {
|
||||
if (ioctl(gopts->uffd, UFFDIO_COPY, uffdio_copy)) {
|
||||
/* real retval in ufdio_copy.copy */
|
||||
if (uffdio_copy->copy != -EEXIST)
|
||||
err("UFFDIO_COPY retry error: %"PRId64,
|
||||
(int64_t)uffdio_copy->copy);
|
||||
(int64_t)uffdio_copy->copy);
|
||||
} else {
|
||||
err("UFFDIO_COPY retry unexpected: %"PRId64,
|
||||
(int64_t)uffdio_copy->copy);
|
||||
@@ -597,60 +618,60 @@ static void wake_range(int ufd, unsigned long addr, unsigned long len)
|
||||
addr), exit(1);
|
||||
}
|
||||
|
||||
int __copy_page(int ufd, unsigned long offset, bool retry, bool wp)
|
||||
int __copy_page(uffd_global_test_opts_t *gopts, unsigned long offset, bool retry, bool wp)
|
||||
{
|
||||
struct uffdio_copy uffdio_copy;
|
||||
|
||||
if (offset >= nr_pages * page_size)
|
||||
if (offset >= gopts->nr_pages * gopts->page_size)
|
||||
err("unexpected offset %lu\n", offset);
|
||||
uffdio_copy.dst = (unsigned long) area_dst + offset;
|
||||
uffdio_copy.src = (unsigned long) area_src + offset;
|
||||
uffdio_copy.len = page_size;
|
||||
uffdio_copy.dst = (unsigned long) gopts->area_dst + offset;
|
||||
uffdio_copy.src = (unsigned long) gopts->area_src + offset;
|
||||
uffdio_copy.len = gopts->page_size;
|
||||
if (wp)
|
||||
uffdio_copy.mode = UFFDIO_COPY_MODE_WP;
|
||||
else
|
||||
uffdio_copy.mode = 0;
|
||||
uffdio_copy.copy = 0;
|
||||
if (ioctl(ufd, UFFDIO_COPY, &uffdio_copy)) {
|
||||
if (ioctl(gopts->uffd, UFFDIO_COPY, &uffdio_copy)) {
|
||||
/* real retval in ufdio_copy.copy */
|
||||
if (uffdio_copy.copy != -EEXIST)
|
||||
err("UFFDIO_COPY error: %"PRId64,
|
||||
(int64_t)uffdio_copy.copy);
|
||||
wake_range(ufd, uffdio_copy.dst, page_size);
|
||||
} else if (uffdio_copy.copy != page_size) {
|
||||
wake_range(gopts->uffd, uffdio_copy.dst, gopts->page_size);
|
||||
} else if (uffdio_copy.copy != gopts->page_size) {
|
||||
err("UFFDIO_COPY error: %"PRId64, (int64_t)uffdio_copy.copy);
|
||||
} else {
|
||||
if (test_uffdio_copy_eexist && retry) {
|
||||
test_uffdio_copy_eexist = false;
|
||||
retry_copy_page(ufd, &uffdio_copy, offset);
|
||||
if (gopts->test_uffdio_copy_eexist && retry) {
|
||||
gopts->test_uffdio_copy_eexist = false;
|
||||
retry_copy_page(gopts, &uffdio_copy, offset);
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
int copy_page(int ufd, unsigned long offset, bool wp)
|
||||
int copy_page(uffd_global_test_opts_t *gopts, unsigned long offset, bool wp)
|
||||
{
|
||||
return __copy_page(ufd, offset, false, wp);
|
||||
return __copy_page(gopts, offset, false, wp);
|
||||
}
|
||||
|
||||
int move_page(int ufd, unsigned long offset, unsigned long len)
|
||||
int move_page(uffd_global_test_opts_t *gopts, unsigned long offset, unsigned long len)
|
||||
{
|
||||
struct uffdio_move uffdio_move;
|
||||
|
||||
if (offset + len > nr_pages * page_size)
|
||||
if (offset + len > gopts->nr_pages * gopts->page_size)
|
||||
err("unexpected offset %lu and length %lu\n", offset, len);
|
||||
uffdio_move.dst = (unsigned long) area_dst + offset;
|
||||
uffdio_move.src = (unsigned long) area_src + offset;
|
||||
uffdio_move.dst = (unsigned long) gopts->area_dst + offset;
|
||||
uffdio_move.src = (unsigned long) gopts->area_src + offset;
|
||||
uffdio_move.len = len;
|
||||
uffdio_move.mode = UFFDIO_MOVE_MODE_ALLOW_SRC_HOLES;
|
||||
uffdio_move.move = 0;
|
||||
if (ioctl(ufd, UFFDIO_MOVE, &uffdio_move)) {
|
||||
if (ioctl(gopts->uffd, UFFDIO_MOVE, &uffdio_move)) {
|
||||
/* real retval in uffdio_move.move */
|
||||
if (uffdio_move.move != -EEXIST)
|
||||
err("UFFDIO_MOVE error: %"PRId64,
|
||||
(int64_t)uffdio_move.move);
|
||||
wake_range(ufd, uffdio_move.dst, len);
|
||||
wake_range(gopts->uffd, uffdio_move.dst, len);
|
||||
} else if (uffdio_move.move != len) {
|
||||
err("UFFDIO_MOVE error: %"PRId64, (int64_t)uffdio_move.move);
|
||||
} else
|
||||
|
||||
@@ -56,20 +56,17 @@
|
||||
|
||||
#define err(fmt, ...) errexit(1, fmt, ##__VA_ARGS__)
|
||||
|
||||
/* pthread_mutex_t starts at page offset 0 */
|
||||
#define area_mutex(___area, ___nr) \
|
||||
((pthread_mutex_t *) ((___area) + (___nr)*page_size))
|
||||
/*
|
||||
* count is placed in the page after pthread_mutex_t naturally aligned
|
||||
* to avoid non alignment faults on non-x86 archs.
|
||||
*/
|
||||
#define area_count(___area, ___nr) \
|
||||
((volatile unsigned long long *) ((unsigned long) \
|
||||
((___area) + (___nr)*page_size + \
|
||||
sizeof(pthread_mutex_t) + \
|
||||
sizeof(unsigned long long) - 1) & \
|
||||
~(unsigned long)(sizeof(unsigned long long) \
|
||||
- 1)))
|
||||
struct uffd_global_test_opts {
|
||||
unsigned long nr_parallel, nr_pages, nr_pages_per_cpu, page_size;
|
||||
char *area_src, *area_src_alias, *area_dst, *area_dst_alias, *area_remap;
|
||||
int uffd, uffd_flags, finished, *pipefd, test_type;
|
||||
bool map_shared;
|
||||
bool test_uffdio_wp;
|
||||
unsigned long long *count_verify;
|
||||
volatile bool test_uffdio_copy_eexist;
|
||||
atomic_bool ready_for_fork;
|
||||
};
|
||||
typedef struct uffd_global_test_opts uffd_global_test_opts_t;
|
||||
|
||||
/* Userfaultfd test statistics */
|
||||
struct uffd_args {
|
||||
@@ -79,50 +76,55 @@ struct uffd_args {
|
||||
unsigned long missing_faults;
|
||||
unsigned long wp_faults;
|
||||
unsigned long minor_faults;
|
||||
struct uffd_global_test_opts *gopts;
|
||||
|
||||
/* A custom fault handler; defaults to uffd_handle_page_fault. */
|
||||
void (*handle_fault)(struct uffd_msg *msg, struct uffd_args *args);
|
||||
void (*handle_fault)(struct uffd_global_test_opts *gopts,
|
||||
struct uffd_msg *msg,
|
||||
struct uffd_args *args);
|
||||
};
|
||||
|
||||
struct uffd_test_ops {
|
||||
int (*allocate_area)(void **alloc_area, bool is_src);
|
||||
void (*release_pages)(char *rel_area);
|
||||
void (*alias_mapping)(__u64 *start, size_t len, unsigned long offset);
|
||||
void (*check_pmd_mapping)(void *p, int expect_nr_hpages);
|
||||
int (*allocate_area)(uffd_global_test_opts_t *gopts, void **alloc_area, bool is_src);
|
||||
void (*release_pages)(uffd_global_test_opts_t *gopts, char *rel_area);
|
||||
void (*alias_mapping)(uffd_global_test_opts_t *gopts,
|
||||
__u64 *start,
|
||||
size_t len,
|
||||
unsigned long offset);
|
||||
void (*check_pmd_mapping)(uffd_global_test_opts_t *gopts, void *p, int expect_nr_hpages);
|
||||
};
|
||||
typedef struct uffd_test_ops uffd_test_ops_t;
|
||||
|
||||
struct uffd_test_case_ops {
|
||||
int (*pre_alloc)(const char **errmsg);
|
||||
int (*post_alloc)(const char **errmsg);
|
||||
int (*pre_alloc)(uffd_global_test_opts_t *gopts, const char **errmsg);
|
||||
int (*post_alloc)(uffd_global_test_opts_t *gopts, const char **errmsg);
|
||||
};
|
||||
typedef struct uffd_test_case_ops uffd_test_case_ops_t;
|
||||
|
||||
extern unsigned long nr_parallel, nr_pages, nr_pages_per_cpu, page_size;
|
||||
extern char *area_src, *area_src_alias, *area_dst, *area_dst_alias, *area_remap;
|
||||
extern int uffd, uffd_flags, finished, *pipefd, test_type;
|
||||
extern bool map_shared;
|
||||
extern bool test_uffdio_wp;
|
||||
extern unsigned long long *count_verify;
|
||||
extern volatile bool test_uffdio_copy_eexist;
|
||||
extern atomic_bool ready_for_fork;
|
||||
|
||||
extern uffd_global_test_opts_t *uffd_gtest_opts;
|
||||
extern uffd_test_ops_t anon_uffd_test_ops;
|
||||
extern uffd_test_ops_t shmem_uffd_test_ops;
|
||||
extern uffd_test_ops_t hugetlb_uffd_test_ops;
|
||||
extern uffd_test_ops_t *uffd_test_ops;
|
||||
extern uffd_test_case_ops_t *uffd_test_case_ops;
|
||||
|
||||
pthread_mutex_t *area_mutex(char *area, unsigned long nr, uffd_global_test_opts_t *gopts);
|
||||
volatile unsigned long long *area_count(char *area,
|
||||
unsigned long nr,
|
||||
uffd_global_test_opts_t *gopts);
|
||||
|
||||
void uffd_stats_report(struct uffd_args *args, int n_cpus);
|
||||
int uffd_test_ctx_init(uint64_t features, const char **errmsg);
|
||||
void uffd_test_ctx_clear(void);
|
||||
int userfaultfd_open(uint64_t *features);
|
||||
int uffd_read_msg(int ufd, struct uffd_msg *msg);
|
||||
int uffd_test_ctx_init(uffd_global_test_opts_t *gopts, uint64_t features, const char **errmsg);
|
||||
void uffd_test_ctx_clear(uffd_global_test_opts_t *gopts);
|
||||
int userfaultfd_open(uffd_global_test_opts_t *gopts, uint64_t *features);
|
||||
int uffd_read_msg(uffd_global_test_opts_t *gopts, struct uffd_msg *msg);
|
||||
void wp_range(int ufd, __u64 start, __u64 len, bool wp);
|
||||
void uffd_handle_page_fault(struct uffd_msg *msg, struct uffd_args *args);
|
||||
int __copy_page(int ufd, unsigned long offset, bool retry, bool wp);
|
||||
int copy_page(int ufd, unsigned long offset, bool wp);
|
||||
int move_page(int ufd, unsigned long offset, unsigned long len);
|
||||
void uffd_handle_page_fault(uffd_global_test_opts_t *gopts,
|
||||
struct uffd_msg *msg,
|
||||
struct uffd_args *args);
|
||||
int __copy_page(uffd_global_test_opts_t *gopts, unsigned long offset, bool retry, bool wp);
|
||||
int copy_page(uffd_global_test_opts_t *gopts, unsigned long offset, bool wp);
|
||||
int move_page(uffd_global_test_opts_t *gopts, unsigned long offset, unsigned long len);
|
||||
void *uffd_poll_thread(void *arg);
|
||||
|
||||
int uffd_open_dev(unsigned int flags);
|
||||
|
||||
@@ -44,6 +44,12 @@ uint64_t features;
|
||||
#define BOUNCE_VERIFY (1<<2)
|
||||
#define BOUNCE_POLL (1<<3)
|
||||
static int bounces;
|
||||
/* defined globally for this particular test as the sigalrm handler
|
||||
* depends on test_uffdio_*_eexist.
|
||||
* XXX: define gopts in main() when we figure out a way to deal with
|
||||
* test_uffdio_*_eexist.
|
||||
*/
|
||||
static uffd_global_test_opts_t *gopts;
|
||||
|
||||
/* exercise the test_uffdio_*_eexist every ALARM_INTERVAL_SECS */
|
||||
#define ALARM_INTERVAL_SECS 10
|
||||
@@ -76,54 +82,58 @@ static void usage(void)
|
||||
exit(1);
|
||||
}
|
||||
|
||||
static void uffd_stats_reset(struct uffd_args *args, unsigned long n_cpus)
|
||||
static void uffd_stats_reset(uffd_global_test_opts_t *gopts, struct uffd_args *args,
|
||||
unsigned long n_cpus)
|
||||
{
|
||||
int i;
|
||||
|
||||
for (i = 0; i < n_cpus; i++) {
|
||||
args[i].cpu = i;
|
||||
args[i].apply_wp = test_uffdio_wp;
|
||||
args[i].apply_wp = gopts->test_uffdio_wp;
|
||||
args[i].missing_faults = 0;
|
||||
args[i].wp_faults = 0;
|
||||
args[i].minor_faults = 0;
|
||||
args[i].gopts = gopts;
|
||||
}
|
||||
}
|
||||
|
||||
static void *locking_thread(void *arg)
|
||||
{
|
||||
unsigned long cpu = (unsigned long) arg;
|
||||
struct uffd_args *args = (struct uffd_args *) arg;
|
||||
uffd_global_test_opts_t *gopts = args->gopts;
|
||||
unsigned long cpu = (unsigned long) args->cpu;
|
||||
unsigned long page_nr;
|
||||
unsigned long long count;
|
||||
|
||||
if (!(bounces & BOUNCE_RANDOM)) {
|
||||
page_nr = -bounces;
|
||||
if (!(bounces & BOUNCE_RACINGFAULTS))
|
||||
page_nr += cpu * nr_pages_per_cpu;
|
||||
page_nr += cpu * gopts->nr_pages_per_cpu;
|
||||
}
|
||||
|
||||
while (!finished) {
|
||||
while (!gopts->finished) {
|
||||
if (bounces & BOUNCE_RANDOM) {
|
||||
if (getrandom(&page_nr, sizeof(page_nr), 0) != sizeof(page_nr))
|
||||
err("getrandom failed");
|
||||
} else
|
||||
page_nr += 1;
|
||||
page_nr %= nr_pages;
|
||||
pthread_mutex_lock(area_mutex(area_dst, page_nr));
|
||||
count = *area_count(area_dst, page_nr);
|
||||
if (count != count_verify[page_nr])
|
||||
page_nr %= gopts->nr_pages;
|
||||
pthread_mutex_lock(area_mutex(gopts->area_dst, page_nr, gopts));
|
||||
count = *area_count(gopts->area_dst, page_nr, gopts);
|
||||
if (count != gopts->count_verify[page_nr])
|
||||
err("page_nr %lu memory corruption %llu %llu",
|
||||
page_nr, count, count_verify[page_nr]);
|
||||
page_nr, count, gopts->count_verify[page_nr]);
|
||||
count++;
|
||||
*area_count(area_dst, page_nr) = count_verify[page_nr] = count;
|
||||
pthread_mutex_unlock(area_mutex(area_dst, page_nr));
|
||||
*area_count(gopts->area_dst, page_nr, gopts) = gopts->count_verify[page_nr] = count;
|
||||
pthread_mutex_unlock(area_mutex(gopts->area_dst, page_nr, gopts));
|
||||
}
|
||||
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static int copy_page_retry(int ufd, unsigned long offset)
|
||||
static int copy_page_retry(uffd_global_test_opts_t *gopts, unsigned long offset)
|
||||
{
|
||||
return __copy_page(ufd, offset, true, test_uffdio_wp);
|
||||
return __copy_page(gopts, offset, true, gopts->test_uffdio_wp);
|
||||
}
|
||||
|
||||
pthread_mutex_t uffd_read_mutex = PTHREAD_MUTEX_INITIALIZER;
|
||||
@@ -131,15 +141,16 @@ pthread_mutex_t uffd_read_mutex = PTHREAD_MUTEX_INITIALIZER;
|
||||
static void *uffd_read_thread(void *arg)
|
||||
{
|
||||
struct uffd_args *args = (struct uffd_args *)arg;
|
||||
uffd_global_test_opts_t *gopts = args->gopts;
|
||||
struct uffd_msg msg;
|
||||
|
||||
pthread_mutex_unlock(&uffd_read_mutex);
|
||||
/* from here cancellation is ok */
|
||||
|
||||
for (;;) {
|
||||
if (uffd_read_msg(uffd, &msg))
|
||||
if (uffd_read_msg(gopts, &msg))
|
||||
continue;
|
||||
uffd_handle_page_fault(&msg, args);
|
||||
uffd_handle_page_fault(gopts, &msg, args);
|
||||
}
|
||||
|
||||
return NULL;
|
||||
@@ -147,32 +158,34 @@ static void *uffd_read_thread(void *arg)
|
||||
|
||||
static void *background_thread(void *arg)
|
||||
{
|
||||
unsigned long cpu = (unsigned long) arg;
|
||||
struct uffd_args *args = (struct uffd_args *) arg;
|
||||
uffd_global_test_opts_t *gopts = args->gopts;
|
||||
unsigned long cpu = (unsigned long) args->cpu;
|
||||
unsigned long page_nr, start_nr, mid_nr, end_nr;
|
||||
|
||||
start_nr = cpu * nr_pages_per_cpu;
|
||||
end_nr = (cpu+1) * nr_pages_per_cpu;
|
||||
start_nr = cpu * gopts->nr_pages_per_cpu;
|
||||
end_nr = (cpu+1) * gopts->nr_pages_per_cpu;
|
||||
mid_nr = (start_nr + end_nr) / 2;
|
||||
|
||||
/* Copy the first half of the pages */
|
||||
for (page_nr = start_nr; page_nr < mid_nr; page_nr++)
|
||||
copy_page_retry(uffd, page_nr * page_size);
|
||||
copy_page_retry(gopts, page_nr * gopts->page_size);
|
||||
|
||||
/*
|
||||
* If we need to test uffd-wp, set it up now. Then we'll have
|
||||
* at least the first half of the pages mapped already which
|
||||
* can be write-protected for testing
|
||||
*/
|
||||
if (test_uffdio_wp)
|
||||
wp_range(uffd, (unsigned long)area_dst + start_nr * page_size,
|
||||
nr_pages_per_cpu * page_size, true);
|
||||
if (gopts->test_uffdio_wp)
|
||||
wp_range(gopts->uffd, (unsigned long)gopts->area_dst + start_nr * gopts->page_size,
|
||||
gopts->nr_pages_per_cpu * gopts->page_size, true);
|
||||
|
||||
/*
|
||||
* Continue the 2nd half of the page copying, handling write
|
||||
* protection faults if any
|
||||
*/
|
||||
for (page_nr = mid_nr; page_nr < end_nr; page_nr++)
|
||||
copy_page_retry(uffd, page_nr * page_size);
|
||||
copy_page_retry(gopts, page_nr * gopts->page_size);
|
||||
|
||||
return NULL;
|
||||
}
|
||||
@@ -180,17 +193,21 @@ static void *background_thread(void *arg)
|
||||
static int stress(struct uffd_args *args)
|
||||
{
|
||||
unsigned long cpu;
|
||||
pthread_t locking_threads[nr_parallel];
|
||||
pthread_t uffd_threads[nr_parallel];
|
||||
pthread_t background_threads[nr_parallel];
|
||||
uffd_global_test_opts_t *gopts = args->gopts;
|
||||
pthread_t locking_threads[gopts->nr_parallel];
|
||||
pthread_t uffd_threads[gopts->nr_parallel];
|
||||
pthread_t background_threads[gopts->nr_parallel];
|
||||
|
||||
finished = 0;
|
||||
for (cpu = 0; cpu < nr_parallel; cpu++) {
|
||||
gopts->finished = 0;
|
||||
for (cpu = 0; cpu < gopts->nr_parallel; cpu++) {
|
||||
if (pthread_create(&locking_threads[cpu], &attr,
|
||||
locking_thread, (void *)cpu))
|
||||
locking_thread, (void *)&args[cpu]))
|
||||
return 1;
|
||||
if (bounces & BOUNCE_POLL) {
|
||||
if (pthread_create(&uffd_threads[cpu], &attr, uffd_poll_thread, &args[cpu]))
|
||||
if (pthread_create(&uffd_threads[cpu],
|
||||
&attr,
|
||||
uffd_poll_thread,
|
||||
(void *) &args[cpu]))
|
||||
err("uffd_poll_thread create");
|
||||
} else {
|
||||
if (pthread_create(&uffd_threads[cpu], &attr,
|
||||
@@ -200,10 +217,10 @@ static int stress(struct uffd_args *args)
|
||||
pthread_mutex_lock(&uffd_read_mutex);
|
||||
}
|
||||
if (pthread_create(&background_threads[cpu], &attr,
|
||||
background_thread, (void *)cpu))
|
||||
background_thread, (void *)&args[cpu]))
|
||||
return 1;
|
||||
}
|
||||
for (cpu = 0; cpu < nr_parallel; cpu++)
|
||||
for (cpu = 0; cpu < gopts->nr_parallel; cpu++)
|
||||
if (pthread_join(background_threads[cpu], NULL))
|
||||
return 1;
|
||||
|
||||
@@ -216,17 +233,17 @@ static int stress(struct uffd_args *args)
|
||||
* UFFDIO_COPY without writing zero pages into area_dst
|
||||
* because the background threads already completed).
|
||||
*/
|
||||
uffd_test_ops->release_pages(area_src);
|
||||
uffd_test_ops->release_pages(gopts, gopts->area_src);
|
||||
|
||||
finished = 1;
|
||||
for (cpu = 0; cpu < nr_parallel; cpu++)
|
||||
gopts->finished = 1;
|
||||
for (cpu = 0; cpu < gopts->nr_parallel; cpu++)
|
||||
if (pthread_join(locking_threads[cpu], NULL))
|
||||
return 1;
|
||||
|
||||
for (cpu = 0; cpu < nr_parallel; cpu++) {
|
||||
for (cpu = 0; cpu < gopts->nr_parallel; cpu++) {
|
||||
char c;
|
||||
if (bounces & BOUNCE_POLL) {
|
||||
if (write(pipefd[cpu*2+1], &c, 1) != 1)
|
||||
if (write(gopts->pipefd[cpu*2+1], &c, 1) != 1)
|
||||
err("pipefd write error");
|
||||
if (pthread_join(uffd_threads[cpu],
|
||||
(void *)&args[cpu]))
|
||||
@@ -242,26 +259,26 @@ static int stress(struct uffd_args *args)
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int userfaultfd_stress(void)
|
||||
static int userfaultfd_stress(uffd_global_test_opts_t *gopts)
|
||||
{
|
||||
void *area;
|
||||
unsigned long nr;
|
||||
struct uffd_args args[nr_parallel];
|
||||
uint64_t mem_size = nr_pages * page_size;
|
||||
struct uffd_args args[gopts->nr_parallel];
|
||||
uint64_t mem_size = gopts->nr_pages * gopts->page_size;
|
||||
int flags = 0;
|
||||
|
||||
memset(args, 0, sizeof(struct uffd_args) * nr_parallel);
|
||||
memset(args, 0, sizeof(struct uffd_args) * gopts->nr_parallel);
|
||||
|
||||
if (features & UFFD_FEATURE_WP_UNPOPULATED && test_type == TEST_ANON)
|
||||
if (features & UFFD_FEATURE_WP_UNPOPULATED && gopts->test_type == TEST_ANON)
|
||||
flags = UFFD_FEATURE_WP_UNPOPULATED;
|
||||
|
||||
if (uffd_test_ctx_init(flags, NULL))
|
||||
if (uffd_test_ctx_init(gopts, flags, NULL))
|
||||
err("context init failed");
|
||||
|
||||
if (posix_memalign(&area, page_size, page_size))
|
||||
if (posix_memalign(&area, gopts->page_size, gopts->page_size))
|
||||
err("out of memory");
|
||||
zeropage = area;
|
||||
bzero(zeropage, page_size);
|
||||
bzero(zeropage, gopts->page_size);
|
||||
|
||||
pthread_mutex_lock(&uffd_read_mutex);
|
||||
|
||||
@@ -284,18 +301,18 @@ static int userfaultfd_stress(void)
|
||||
fflush(stdout);
|
||||
|
||||
if (bounces & BOUNCE_POLL)
|
||||
fcntl(uffd, F_SETFL, uffd_flags | O_NONBLOCK);
|
||||
fcntl(gopts->uffd, F_SETFL, gopts->uffd_flags | O_NONBLOCK);
|
||||
else
|
||||
fcntl(uffd, F_SETFL, uffd_flags & ~O_NONBLOCK);
|
||||
fcntl(gopts->uffd, F_SETFL, gopts->uffd_flags & ~O_NONBLOCK);
|
||||
|
||||
/* register */
|
||||
if (uffd_register(uffd, area_dst, mem_size,
|
||||
true, test_uffdio_wp, false))
|
||||
if (uffd_register(gopts->uffd, gopts->area_dst, mem_size,
|
||||
true, gopts->test_uffdio_wp, false))
|
||||
err("register failure");
|
||||
|
||||
if (area_dst_alias) {
|
||||
if (uffd_register(uffd, area_dst_alias, mem_size,
|
||||
true, test_uffdio_wp, false))
|
||||
if (gopts->area_dst_alias) {
|
||||
if (uffd_register(gopts->uffd, gopts->area_dst_alias, mem_size,
|
||||
true, gopts->test_uffdio_wp, false))
|
||||
err("register failure alias");
|
||||
}
|
||||
|
||||
@@ -323,87 +340,88 @@ static int userfaultfd_stress(void)
|
||||
* MADV_DONTNEED only after the UFFDIO_REGISTER, so it's
|
||||
* required to MADV_DONTNEED here.
|
||||
*/
|
||||
uffd_test_ops->release_pages(area_dst);
|
||||
uffd_test_ops->release_pages(gopts, gopts->area_dst);
|
||||
|
||||
uffd_stats_reset(args, nr_parallel);
|
||||
uffd_stats_reset(gopts, args, gopts->nr_parallel);
|
||||
|
||||
/* bounce pass */
|
||||
if (stress(args)) {
|
||||
uffd_test_ctx_clear();
|
||||
uffd_test_ctx_clear(gopts);
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* Clear all the write protections if there is any */
|
||||
if (test_uffdio_wp)
|
||||
wp_range(uffd, (unsigned long)area_dst,
|
||||
nr_pages * page_size, false);
|
||||
if (gopts->test_uffdio_wp)
|
||||
wp_range(gopts->uffd, (unsigned long)gopts->area_dst,
|
||||
gopts->nr_pages * gopts->page_size, false);
|
||||
|
||||
/* unregister */
|
||||
if (uffd_unregister(uffd, area_dst, mem_size))
|
||||
if (uffd_unregister(gopts->uffd, gopts->area_dst, mem_size))
|
||||
err("unregister failure");
|
||||
if (area_dst_alias) {
|
||||
if (uffd_unregister(uffd, area_dst_alias, mem_size))
|
||||
if (gopts->area_dst_alias) {
|
||||
if (uffd_unregister(gopts->uffd, gopts->area_dst_alias, mem_size))
|
||||
err("unregister failure alias");
|
||||
}
|
||||
|
||||
/* verification */
|
||||
if (bounces & BOUNCE_VERIFY)
|
||||
for (nr = 0; nr < nr_pages; nr++)
|
||||
if (*area_count(area_dst, nr) != count_verify[nr])
|
||||
for (nr = 0; nr < gopts->nr_pages; nr++)
|
||||
if (*area_count(gopts->area_dst, nr, gopts) !=
|
||||
gopts->count_verify[nr])
|
||||
err("error area_count %llu %llu %lu\n",
|
||||
*area_count(area_src, nr),
|
||||
count_verify[nr], nr);
|
||||
*area_count(gopts->area_src, nr, gopts),
|
||||
gopts->count_verify[nr], nr);
|
||||
|
||||
/* prepare next bounce */
|
||||
swap(area_src, area_dst);
|
||||
swap(gopts->area_src, gopts->area_dst);
|
||||
|
||||
swap(area_src_alias, area_dst_alias);
|
||||
swap(gopts->area_src_alias, gopts->area_dst_alias);
|
||||
|
||||
uffd_stats_report(args, nr_parallel);
|
||||
uffd_stats_report(args, gopts->nr_parallel);
|
||||
}
|
||||
uffd_test_ctx_clear();
|
||||
uffd_test_ctx_clear(gopts);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void set_test_type(const char *type)
|
||||
static void set_test_type(uffd_global_test_opts_t *gopts, const char *type)
|
||||
{
|
||||
if (!strcmp(type, "anon")) {
|
||||
test_type = TEST_ANON;
|
||||
gopts->test_type = TEST_ANON;
|
||||
uffd_test_ops = &anon_uffd_test_ops;
|
||||
} else if (!strcmp(type, "hugetlb")) {
|
||||
test_type = TEST_HUGETLB;
|
||||
gopts->test_type = TEST_HUGETLB;
|
||||
uffd_test_ops = &hugetlb_uffd_test_ops;
|
||||
map_shared = true;
|
||||
gopts->map_shared = true;
|
||||
} else if (!strcmp(type, "hugetlb-private")) {
|
||||
test_type = TEST_HUGETLB;
|
||||
gopts->test_type = TEST_HUGETLB;
|
||||
uffd_test_ops = &hugetlb_uffd_test_ops;
|
||||
} else if (!strcmp(type, "shmem")) {
|
||||
map_shared = true;
|
||||
test_type = TEST_SHMEM;
|
||||
gopts->map_shared = true;
|
||||
gopts->test_type = TEST_SHMEM;
|
||||
uffd_test_ops = &shmem_uffd_test_ops;
|
||||
} else if (!strcmp(type, "shmem-private")) {
|
||||
test_type = TEST_SHMEM;
|
||||
gopts->test_type = TEST_SHMEM;
|
||||
uffd_test_ops = &shmem_uffd_test_ops;
|
||||
}
|
||||
}
|
||||
|
||||
static void parse_test_type_arg(const char *raw_type)
|
||||
static void parse_test_type_arg(uffd_global_test_opts_t *gopts, const char *raw_type)
|
||||
{
|
||||
set_test_type(raw_type);
|
||||
set_test_type(gopts, raw_type);
|
||||
|
||||
if (!test_type)
|
||||
if (!gopts->test_type)
|
||||
err("failed to parse test type argument: '%s'", raw_type);
|
||||
|
||||
if (test_type == TEST_HUGETLB)
|
||||
page_size = default_huge_page_size();
|
||||
if (gopts->test_type == TEST_HUGETLB)
|
||||
gopts->page_size = default_huge_page_size();
|
||||
else
|
||||
page_size = sysconf(_SC_PAGE_SIZE);
|
||||
gopts->page_size = sysconf(_SC_PAGE_SIZE);
|
||||
|
||||
if (!page_size)
|
||||
if (!gopts->page_size)
|
||||
err("Unable to determine page size");
|
||||
if ((unsigned long) area_count(NULL, 0) + sizeof(unsigned long long) * 2
|
||||
> page_size)
|
||||
if ((unsigned long) area_count(NULL, 0, gopts) + sizeof(unsigned long long) * 2
|
||||
> gopts->page_size)
|
||||
err("Impossible to run this test");
|
||||
|
||||
/*
|
||||
@@ -415,21 +433,21 @@ static void parse_test_type_arg(const char *raw_type)
|
||||
if (uffd_get_features(&features) && errno == ENOENT)
|
||||
ksft_exit_skip("failed to get available features (%d)\n", errno);
|
||||
|
||||
test_uffdio_wp = test_uffdio_wp &&
|
||||
gopts->test_uffdio_wp = gopts->test_uffdio_wp &&
|
||||
(features & UFFD_FEATURE_PAGEFAULT_FLAG_WP);
|
||||
|
||||
if (test_type != TEST_ANON && !(features & UFFD_FEATURE_WP_HUGETLBFS_SHMEM))
|
||||
test_uffdio_wp = false;
|
||||
if (gopts->test_type != TEST_ANON && !(features & UFFD_FEATURE_WP_HUGETLBFS_SHMEM))
|
||||
gopts->test_uffdio_wp = false;
|
||||
|
||||
close(uffd);
|
||||
uffd = -1;
|
||||
close(gopts->uffd);
|
||||
gopts->uffd = -1;
|
||||
}
|
||||
|
||||
static void sigalrm(int sig)
|
||||
{
|
||||
if (sig != SIGALRM)
|
||||
abort();
|
||||
test_uffdio_copy_eexist = true;
|
||||
gopts->test_uffdio_copy_eexist = true;
|
||||
alarm(ALARM_INTERVAL_SECS);
|
||||
}
|
||||
|
||||
@@ -438,6 +456,8 @@ int main(int argc, char **argv)
|
||||
unsigned long nr_cpus;
|
||||
size_t bytes;
|
||||
|
||||
gopts = (uffd_global_test_opts_t *) malloc(sizeof(uffd_global_test_opts_t));
|
||||
|
||||
if (argc < 4)
|
||||
usage();
|
||||
|
||||
@@ -445,7 +465,7 @@ int main(int argc, char **argv)
|
||||
err("failed to arm SIGALRM");
|
||||
alarm(ALARM_INTERVAL_SECS);
|
||||
|
||||
parse_test_type_arg(argv[1]);
|
||||
parse_test_type_arg(gopts, argv[1]);
|
||||
bytes = atol(argv[2]) * 1024 * 1024;
|
||||
|
||||
nr_cpus = sysconf(_SC_NPROCESSORS_ONLN);
|
||||
@@ -453,9 +473,9 @@ int main(int argc, char **argv)
|
||||
/* Don't let calculation below go to zero. */
|
||||
ksft_print_msg("_SC_NPROCESSORS_ONLN (%lu) too large, capping nr_threads to 32\n",
|
||||
nr_cpus);
|
||||
nr_parallel = 32;
|
||||
gopts->nr_parallel = 32;
|
||||
} else {
|
||||
nr_parallel = nr_cpus;
|
||||
gopts->nr_parallel = nr_cpus;
|
||||
}
|
||||
|
||||
/*
|
||||
@@ -463,16 +483,16 @@ int main(int argc, char **argv)
|
||||
* Ensure nr_parallel - 1 hugepages on top of that to account
|
||||
* for racy extra reservation of hugepages.
|
||||
*/
|
||||
if (test_type == TEST_HUGETLB &&
|
||||
get_free_hugepages() < 2 * (bytes / page_size) + nr_parallel - 1) {
|
||||
if (gopts->test_type == TEST_HUGETLB &&
|
||||
get_free_hugepages() < 2 * (bytes / gopts->page_size) + gopts->nr_parallel - 1) {
|
||||
printf("skip: Skipping userfaultfd... not enough hugepages\n");
|
||||
return KSFT_SKIP;
|
||||
}
|
||||
|
||||
nr_pages_per_cpu = bytes / page_size / nr_parallel;
|
||||
if (!nr_pages_per_cpu) {
|
||||
gopts->nr_pages_per_cpu = bytes / gopts->page_size / gopts->nr_parallel;
|
||||
if (!gopts->nr_pages_per_cpu) {
|
||||
_err("pages_per_cpu = 0, cannot test (%lu / %lu / %lu)",
|
||||
bytes, page_size, nr_parallel);
|
||||
bytes, gopts->page_size, gopts->nr_parallel);
|
||||
usage();
|
||||
}
|
||||
|
||||
@@ -481,11 +501,11 @@ int main(int argc, char **argv)
|
||||
_err("invalid bounces");
|
||||
usage();
|
||||
}
|
||||
nr_pages = nr_pages_per_cpu * nr_parallel;
|
||||
gopts->nr_pages = gopts->nr_pages_per_cpu * gopts->nr_parallel;
|
||||
|
||||
printf("nr_pages: %lu, nr_pages_per_cpu: %lu\n",
|
||||
nr_pages, nr_pages_per_cpu);
|
||||
return userfaultfd_stress();
|
||||
gopts->nr_pages, gopts->nr_pages_per_cpu);
|
||||
return userfaultfd_stress(gopts);
|
||||
}
|
||||
|
||||
#else /* __NR_userfaultfd */
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -152,7 +152,8 @@ static bool range_is_swapped(void *addr, size_t size)
|
||||
return true;
|
||||
}
|
||||
|
||||
static void test_one_folio(size_t size, bool private, bool swapout, bool hugetlb)
|
||||
static void test_one_folio(uffd_global_test_opts_t *gopts, size_t size, bool private,
|
||||
bool swapout, bool hugetlb)
|
||||
{
|
||||
struct uffdio_writeprotect wp_prms;
|
||||
uint64_t features = 0;
|
||||
@@ -176,21 +177,21 @@ static void test_one_folio(size_t size, bool private, bool swapout, bool hugetlb
|
||||
}
|
||||
|
||||
/* Register range for uffd-wp. */
|
||||
if (userfaultfd_open(&features)) {
|
||||
if (userfaultfd_open(gopts, &features)) {
|
||||
if (errno == ENOENT)
|
||||
ksft_test_result_skip("userfaultfd not available\n");
|
||||
else
|
||||
ksft_test_result_fail("userfaultfd_open() failed\n");
|
||||
goto out;
|
||||
}
|
||||
if (uffd_register(uffd, mem, size, false, true, false)) {
|
||||
if (uffd_register(gopts->uffd, mem, size, false, true, false)) {
|
||||
ksft_test_result_fail("uffd_register() failed\n");
|
||||
goto out;
|
||||
}
|
||||
wp_prms.mode = UFFDIO_WRITEPROTECT_MODE_WP;
|
||||
wp_prms.range.start = (uintptr_t)mem;
|
||||
wp_prms.range.len = size;
|
||||
if (ioctl(uffd, UFFDIO_WRITEPROTECT, &wp_prms)) {
|
||||
if (ioctl(gopts->uffd, UFFDIO_WRITEPROTECT, &wp_prms)) {
|
||||
ksft_test_result_fail("ioctl(UFFDIO_WRITEPROTECT) failed\n");
|
||||
goto out;
|
||||
}
|
||||
@@ -237,9 +238,9 @@ static void test_one_folio(size_t size, bool private, bool swapout, bool hugetlb
|
||||
out:
|
||||
if (mem)
|
||||
munmap(mem, size);
|
||||
if (uffd >= 0) {
|
||||
close(uffd);
|
||||
uffd = -1;
|
||||
if (gopts->uffd >= 0) {
|
||||
close(gopts->uffd);
|
||||
gopts->uffd = -1;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -331,6 +332,7 @@ static const struct testcase testcases[] = {
|
||||
|
||||
int main(int argc, char **argv)
|
||||
{
|
||||
uffd_global_test_opts_t gopts = { 0 };
|
||||
struct thp_settings settings;
|
||||
int i, j, plan = 0;
|
||||
|
||||
@@ -362,8 +364,8 @@ int main(int argc, char **argv)
|
||||
const struct testcase *tc = &testcases[i];
|
||||
|
||||
for (j = 0; j < *tc->nr_sizes; j++)
|
||||
test_one_folio(tc->sizes[j], tc->private, tc->swapout,
|
||||
tc->hugetlb);
|
||||
test_one_folio(&gopts, tc->sizes[j], tc->private,
|
||||
tc->swapout, tc->hugetlb);
|
||||
}
|
||||
|
||||
/* If THP is supported, restore original THP settings. */
|
||||
|
||||
Reference in New Issue
Block a user