tools/testing/selftests/vm/userfaultfd.c

   1 /*
   2  * Stress userfaultfd syscall.
   3  *
   4  *  Copyright (C) 2015  Red Hat, Inc.
   5  *
   6  *  This work is licensed under the terms of the GNU GPL, version 2. See
   7  *  the COPYING file in the top-level directory.
   8  *
   9  * This test allocates two virtual areas and bounces the physical
  10  * memory across the two virtual areas (from area_src to area_dst)
  11  * using userfaultfd.
  12  *
  13  * There are three threads running per CPU:
  14  *
  15  * 1) one per-CPU thread takes a per-page pthread_mutex in a random
  16  *    page of the area_dst (while the physical page may still be in
  17  *    area_src), and increments a per-page counter in the same page,
  18  *    and checks its value against a verification region.
  19  *
  20  * 2) another per-CPU thread handles the userfaults generated by
  21  *    thread 1 above. userfaultfd blocking reads or poll() modes are
  22  *    exercised interleaved.
  23  *
  24  * 3) one last per-CPU thread transfers the memory in the background
  25  *    at maximum bandwidth (if not already transferred by thread
  26  *    2). Each cpu thread takes cares of transferring a portion of the
  27  *    area.
  28  *
  29  * When all threads of type 3 completed the transfer, one bounce is
  30  * complete. area_src and area_dst are then swapped. All threads are
  31  * respawned and so the bounce is immediately restarted in the
  32  * opposite direction.
  33  *
  34  * per-CPU threads 1 by triggering userfaults inside
  35  * pthread_mutex_lock will also verify the atomicity of the memory
  36  * transfer (UFFDIO_COPY).
  37  *
  38  * The program takes two parameters: the amounts of physical memory in
  39  * megabytes (MiB) of the area and the number of bounces to execute.
  40  *
  41  * # 100MiB 99999 bounces
  42  * ./userfaultfd 100 99999
  43  *
  44  * # 1GiB 99 bounces
  45  * ./userfaultfd 1000 99
  46  *
  47  * # 10MiB-~6GiB 999 bounces, continue forever unless an error triggers
  48  * while ./userfaultfd $[RANDOM % 6000 + 10] 999; do true; done
  49  */
  50
  51 #define _GNU_SOURCE
  52 #include <stdio.h>
  53 #include <errno.h>
  54 #include <unistd.h>
  55 #include <stdlib.h>
  56 #include <sys/types.h>
  57 #include <sys/stat.h>
  58 #include <fcntl.h>
  59 #include <time.h>
  60 #include <signal.h>
  61 #include <poll.h>
  62 #include <string.h>
  63 #include <linux/mman.h>
  64 #include <sys/mman.h>
  65 #include <sys/syscall.h>
  66 #include <sys/ioctl.h>
  67 #include <sys/wait.h>
  68 #include <pthread.h>
  69 #include <linux/userfaultfd.h>
  70 #include <setjmp.h>
  71 #include <stdbool.h>
  72
  73 #include "../kselftest.h"
  74
  75 #ifdef __NR_userfaultfd
  76
  77 static unsigned long nr_cpus, nr_pages, nr_pages_per_cpu, page_size;
  78
  79 #define BOUNCE_RANDOM           (1<<0)
  80 #define BOUNCE_RACINGFAULTS     (1<<1)
  81 #define BOUNCE_VERIFY           (1<<2)
  82 #define BOUNCE_POLL             (1<<3)
  83 static int bounces;
  84
  85 #define TEST_ANON       1
  86 #define TEST_HUGETLB    2
  87 #define TEST_SHMEM      3
  88 static int test_type;
  89
  90 /* exercise the test_uffdio_*_eexist every ALARM_INTERVAL_SECS */
  91 #define ALARM_INTERVAL_SECS 10
  92 static volatile bool test_uffdio_copy_eexist = true;
  93 static volatile bool test_uffdio_zeropage_eexist = true;
  94
  95 static bool map_shared;
  96 static int huge_fd;
  97 static char *huge_fd_off0;
  98 static unsigned long long *count_verify;
  99 static int uffd, uffd_flags, finished, *pipefd;
 100 static char *area_src, *area_src_alias, *area_dst, *area_dst_alias;
 101 static char *zeropage;
 102 pthread_attr_t attr;
 103
 104 /* pthread_mutex_t starts at page offset 0 */
 105 #define area_mutex(___area, ___nr)                                      \
 106         ((pthread_mutex_t *) ((___area) + (___nr)*page_size))
 107 /*
 108  * count is placed in the page after pthread_mutex_t naturally aligned
 109  * to avoid non alignment faults on non-x86 archs.
 110  */
 111 #define area_count(___area, ___nr)                                      \
 112         ((volatile unsigned long long *) ((unsigned long)               \
 113                                  ((___area) + (___nr)*page_size +       \
 114                                   sizeof(pthread_mutex_t) +             \
 115                                   sizeof(unsigned long long) - 1) &     \
 116                                  ~(unsigned long)(sizeof(unsigned long long) \
 117                                                   -  1)))
 118
 119 static int anon_release_pages(char *rel_area)
 120 {
 121         int ret = 0;
 122
 123         if (madvise(rel_area, nr_pages * page_size, MADV_DONTNEED)) {
 124                 perror("madvise");
 125                 ret = 1;
 126         }
 127
 128         return ret;
 129 }
 130
 131 static void anon_allocate_area(void **alloc_area)
 132 {
 133         *alloc_area = mmap(NULL, nr_pages * page_size, PROT_READ | PROT_WRITE,
 134                            MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
 135         if (*alloc_area == MAP_FAILED) {
 136                 fprintf(stderr, "mmap of anonymous memory failed");
 137                 *alloc_area = NULL;
 138         }
 139 }
 140
 141 static void noop_alias_mapping(__u64 *start, size_t len, unsigned long offset)
 142 {
 143 }
 144
 145 /* HugeTLB memory */
 146 static int hugetlb_release_pages(char *rel_area)
 147 {
 148         int ret = 0;
 149
 150         if (fallocate(huge_fd, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE,
 151                                 rel_area == huge_fd_off0 ? 0 :
 152                                 nr_pages * page_size,
 153                                 nr_pages * page_size)) {
 154                 perror("fallocate");
 155                 ret = 1;
 156         }
 157
 158         return ret;
 159 }
 160
 161
 162 static void hugetlb_allocate_area(void **alloc_area)
 163 {
 164         void *area_alias = NULL;
 165         char **alloc_area_alias;
 166         *alloc_area = mmap(NULL, nr_pages * page_size, PROT_READ | PROT_WRITE,
 167                            (map_shared ? MAP_SHARED : MAP_PRIVATE) |
 168                            MAP_HUGETLB,
 169                            huge_fd, *alloc_area == area_src ? 0 :
 170                            nr_pages * page_size);
 171         if (*alloc_area == MAP_FAILED) {
 172                 fprintf(stderr, "mmap of hugetlbfs file failed\n");
 173                 *alloc_area = NULL;
 174         }
 175
 176         if (map_shared) {
 177                 area_alias = mmap(NULL, nr_pages * page_size, PROT_READ | PROT_WRITE,
 178                                   MAP_SHARED | MAP_HUGETLB,
 179                                   huge_fd, *alloc_area == area_src ? 0 :
 180                                   nr_pages * page_size);
 181                 if (area_alias == MAP_FAILED) {
 182                         if (munmap(*alloc_area, nr_pages * page_size) < 0)
 183                                 perror("hugetlb munmap"), exit(1);
 184                         *alloc_area = NULL;
 185                         return;
 186                 }
 187         }
 188         if (*alloc_area == area_src) {
 189                 huge_fd_off0 = *alloc_area;
 190                 alloc_area_alias = &area_src_alias;
 191         } else {
 192                 alloc_area_alias = &area_dst_alias;
 193         }
 194         if (area_alias)
 195                 *alloc_area_alias = area_alias;
 196 }
 197
 198 static void hugetlb_alias_mapping(__u64 *start, size_t len, unsigned long offset)
 199 {
 200         if (!map_shared)
 201                 return;
 202         /*
 203          * We can't zap just the pagetable with hugetlbfs because
 204          * MADV_DONTEED won't work. So exercise -EEXIST on a alias
 205          * mapping where the pagetables are not established initially,
 206          * this way we'll exercise the -EEXEC at the fs level.
 207          */
 208         *start = (unsigned long) area_dst_alias + offset;
 209 }
 210
 211 /* Shared memory */
 212 static int shmem_release_pages(char *rel_area)
 213 {
 214         int ret = 0;
 215
 216         if (madvise(rel_area, nr_pages * page_size, MADV_REMOVE)) {
 217                 perror("madvise");
 218                 ret = 1;
 219         }
 220
 221         return ret;
 222 }
 223
 224 static void shmem_allocate_area(void **alloc_area)
 225 {
 226         *alloc_area = mmap(NULL, nr_pages * page_size, PROT_READ | PROT_WRITE,
 227                            MAP_ANONYMOUS | MAP_SHARED, -1, 0);
 228         if (*alloc_area == MAP_FAILED) {
 229                 fprintf(stderr, "shared memory mmap failed\n");
 230                 *alloc_area = NULL;
 231         }
 232 }
 233
 234 struct uffd_test_ops {
 235         unsigned long expected_ioctls;
 236         void (*allocate_area)(void **alloc_area);
 237         int (*release_pages)(char *rel_area);
 238         void (*alias_mapping)(__u64 *start, size_t len, unsigned long offset);
 239 };
 240
 241 #define ANON_EXPECTED_IOCTLS            ((1 << _UFFDIO_WAKE) | \
 242                                          (1 << _UFFDIO_COPY) | \
 243                                          (1 << _UFFDIO_ZEROPAGE))
 244
 245 static struct uffd_test_ops anon_uffd_test_ops = {
 246         .expected_ioctls = ANON_EXPECTED_IOCTLS,
 247         .allocate_area  = anon_allocate_area,
 248         .release_pages  = anon_release_pages,
 249         .alias_mapping = noop_alias_mapping,
 250 };
 251
 252 static struct uffd_test_ops shmem_uffd_test_ops = {
 253         .expected_ioctls = ANON_EXPECTED_IOCTLS,
 254         .allocate_area  = shmem_allocate_area,
 255         .release_pages  = shmem_release_pages,
 256         .alias_mapping = noop_alias_mapping,
 257 };
 258
 259 static struct uffd_test_ops hugetlb_uffd_test_ops = {
 260         .expected_ioctls = UFFD_API_RANGE_IOCTLS_BASIC,
 261         .allocate_area  = hugetlb_allocate_area,
 262         .release_pages  = hugetlb_release_pages,
 263         .alias_mapping = hugetlb_alias_mapping,
 264 };
 265
 266 static struct uffd_test_ops *uffd_test_ops;
 267
 268 static int my_bcmp(char *str1, char *str2, size_t n)
 269 {
 270         unsigned long i;
 271         for (i = 0; i < n; i++)
 272                 if (str1[i] != str2[i])
 273                         return 1;
 274         return 0;
 275 }
 276
 277 static void *locking_thread(void *arg)
 278 {
 279         unsigned long cpu = (unsigned long) arg;
 280         struct random_data rand;
 281         unsigned long page_nr = *(&(page_nr)); /* uninitialized warning */
 282         int32_t rand_nr;
 283         unsigned long long count;
 284         char randstate[64];
 285         unsigned int seed;
 286         time_t start;
 287
 288         if (bounces & BOUNCE_RANDOM) {
 289                 seed = (unsigned int) time(NULL) - bounces;
 290                 if (!(bounces & BOUNCE_RACINGFAULTS))
 291                         seed += cpu;
 292                 bzero(&rand, sizeof(rand));
 293                 bzero(&randstate, sizeof(randstate));
 294                 if (initstate_r(seed, randstate, sizeof(randstate), &rand))
 295                         fprintf(stderr, "srandom_r error\n"), exit(1);
 296         } else {
 297                 page_nr = -bounces;
 298                 if (!(bounces & BOUNCE_RACINGFAULTS))
 299                         page_nr += cpu * nr_pages_per_cpu;
 300         }
 301
 302         while (!finished) {
 303                 if (bounces & BOUNCE_RANDOM) {
 304                         if (random_r(&rand, &rand_nr))
 305                                 fprintf(stderr, "random_r 1 error\n"), exit(1);
 306                         page_nr = rand_nr;
 307                         if (sizeof(page_nr) > sizeof(rand_nr)) {
 308                                 if (random_r(&rand, &rand_nr))
 309                                         fprintf(stderr, "random_r 2 error\n"), exit(1);
 310                                 page_nr |= (((unsigned long) rand_nr) << 16) <<
 311                                            16;
 312                         }
 313                 } else
 314                         page_nr += 1;
 315                 page_nr %= nr_pages;
 316
 317                 start = time(NULL);
 318                 if (bounces & BOUNCE_VERIFY) {
 319                         count = *area_count(area_dst, page_nr);
 320                         if (!count)
 321                                 fprintf(stderr,
 322                                         "page_nr %lu wrong count %Lu %Lu\n",
 323                                         page_nr, count,
 324                                         count_verify[page_nr]), exit(1);
 325
 326
 327                         /*
 328                          * We can't use bcmp (or memcmp) because that
 329                          * returns 0 erroneously if the memory is
 330                          * changing under it (even if the end of the
 331                          * page is never changing and always
 332                          * different).
 333                          */
 334 #if 1
 335                         if (!my_bcmp(area_dst + page_nr * page_size, zeropage,
 336                                      page_size))
 337                                 fprintf(stderr,
 338                                         "my_bcmp page_nr %lu wrong count %Lu %Lu\n",
 339                                         page_nr, count,
 340                                         count_verify[page_nr]), exit(1);
 341 #else
 342                         unsigned long loops;
 343
 344                         loops = 0;
 345                         /* uncomment the below line to test with mutex */
 346                         /* pthread_mutex_lock(area_mutex(area_dst, page_nr)); */
 347                         while (!bcmp(area_dst + page_nr * page_size, zeropage,
 348                                      page_size)) {
 349                                 loops += 1;
 350                                 if (loops > 10)
 351                                         break;
 352                         }
 353                         /* uncomment below line to test with mutex */
 354                         /* pthread_mutex_unlock(area_mutex(area_dst, page_nr)); */
 355                         if (loops) {
 356                                 fprintf(stderr,
 357                                         "page_nr %lu all zero thread %lu %p %lu\n",
 358                                         page_nr, cpu, area_dst + page_nr * page_size,
 359                                         loops);
 360                                 if (loops > 10)
 361                                         exit(1);
 362                         }
 363 #endif
 364                 }
 365
 366                 pthread_mutex_lock(area_mutex(area_dst, page_nr));
 367                 count = *area_count(area_dst, page_nr);
 368                 if (count != count_verify[page_nr]) {
 369                         fprintf(stderr,
 370                                 "page_nr %lu memory corruption %Lu %Lu\n",
 371                                 page_nr, count,
 372                                 count_verify[page_nr]), exit(1);
 373                 }
 374                 count++;
 375                 *area_count(area_dst, page_nr) = count_verify[page_nr] = count;
 376                 pthread_mutex_unlock(area_mutex(area_dst, page_nr));
 377
 378                 if (time(NULL) - start > 1)
 379                         fprintf(stderr,
 380                                 "userfault too slow %ld "
 381                                 "possible false positive with overcommit\n",
 382                                 time(NULL) - start);
 383         }
 384
 385         return NULL;
 386 }
 387
 388 static void retry_copy_page(int ufd, struct uffdio_copy *uffdio_copy,
 389                             unsigned long offset)
 390 {
 391         uffd_test_ops->alias_mapping(&uffdio_copy->dst,
 392                                      uffdio_copy->len,
 393                                      offset);
 394         if (ioctl(ufd, UFFDIO_COPY, uffdio_copy)) {
 395                 /* real retval in ufdio_copy.copy */
 396                 if (uffdio_copy->copy != -EEXIST)
 397                         fprintf(stderr, "UFFDIO_COPY retry error %Ld\n",
 398                                 uffdio_copy->copy), exit(1);
 399         } else {
 400                 fprintf(stderr, "UFFDIO_COPY retry unexpected %Ld\n",
 401                         uffdio_copy->copy), exit(1);
 402         }
 403 }
 404
 405 static int __copy_page(int ufd, unsigned long offset, bool retry)
 406 {
 407         struct uffdio_copy uffdio_copy;
 408
 409         if (offset >= nr_pages * page_size)
 410                 fprintf(stderr, "unexpected offset %lu\n",
 411                         offset), exit(1);
 412         uffdio_copy.dst = (unsigned long) area_dst + offset;
 413         uffdio_copy.src = (unsigned long) area_src + offset;
 414         uffdio_copy.len = page_size;
 415         uffdio_copy.mode = 0;
 416         uffdio_copy.copy = 0;
 417         if (ioctl(ufd, UFFDIO_COPY, &uffdio_copy)) {
 418                 /* real retval in ufdio_copy.copy */
 419                 if (uffdio_copy.copy != -EEXIST)
 420                         fprintf(stderr, "UFFDIO_COPY error %Ld\n",
 421                                 uffdio_copy.copy), exit(1);
 422         } else if (uffdio_copy.copy != page_size) {
 423                 fprintf(stderr, "UFFDIO_COPY unexpected copy %Ld\n",
 424                         uffdio_copy.copy), exit(1);
 425         } else {
 426                 if (test_uffdio_copy_eexist && retry) {
 427                         test_uffdio_copy_eexist = false;
 428                         retry_copy_page(ufd, &uffdio_copy, offset);
 429                 }
 430                 return 1;
 431         }
 432         return 0;
 433 }
 434
 435 static int copy_page_retry(int ufd, unsigned long offset)
 436 {
 437         return __copy_page(ufd, offset, true);
 438 }
 439
 440 static int copy_page(int ufd, unsigned long offset)
 441 {
 442         return __copy_page(ufd, offset, false);
 443 }
 444
 445 static void *uffd_poll_thread(void *arg)
 446 {
 447         unsigned long cpu = (unsigned long) arg;
 448         struct pollfd pollfd[2];
 449         struct uffd_msg msg;
 450         struct uffdio_register uffd_reg;
 451         int ret;
 452         unsigned long offset;
 453         char tmp_chr;
 454         unsigned long userfaults = 0;
 455
 456         pollfd[0].fd = uffd;
 457         pollfd[0].events = POLLIN;
 458         pollfd[1].fd = pipefd[cpu*2];
 459         pollfd[1].events = POLLIN;
 460
 461         for (;;) {
 462                 ret = poll(pollfd, 2, -1);
 463                 if (!ret)
 464                         fprintf(stderr, "poll error %d\n", ret), exit(1);
 465                 if (ret < 0)
 466                         perror("poll"), exit(1);
 467                 if (pollfd[1].revents & POLLIN) {
 468                         if (read(pollfd[1].fd, &tmp_chr, 1) != 1)
 469                                 fprintf(stderr, "read pipefd error\n"),
 470                                         exit(1);
 471                         break;
 472                 }
 473                 if (!(pollfd[0].revents & POLLIN))
 474                         fprintf(stderr, "pollfd[0].revents %d\n",
 475                                 pollfd[0].revents), exit(1);
 476                 ret = read(uffd, &msg, sizeof(msg));
 477                 if (ret < 0) {
 478                         if (errno == EAGAIN)
 479                                 continue;
 480                         perror("nonblocking read error"), exit(1);
 481                 }
 482                 switch (msg.event) {
 483                 default:
 484                         fprintf(stderr, "unexpected msg event %u\n",
 485                                 msg.event), exit(1);
 486                         break;
 487                 case UFFD_EVENT_PAGEFAULT:
 488                         if (msg.arg.pagefault.flags & UFFD_PAGEFAULT_FLAG_WRITE)
 489                                 fprintf(stderr, "unexpected write fault\n"), exit(1);
 490                         offset = (char *)(unsigned long)msg.arg.pagefault.address -
 491                                 area_dst;
 492                         offset &= ~(page_size-1);
 493                         if (copy_page(uffd, offset))
 494                                 userfaults++;
 495                         break;
 496                 case UFFD_EVENT_FORK:
 497                         close(uffd);
 498                         uffd = msg.arg.fork.ufd;
 499                         pollfd[0].fd = uffd;
 500                         break;
 501                 case UFFD_EVENT_REMOVE:
 502                         uffd_reg.range.start = msg.arg.remove.start;
 503                         uffd_reg.range.len = msg.arg.remove.end -
 504                                 msg.arg.remove.start;
 505                         if (ioctl(uffd, UFFDIO_UNREGISTER, &uffd_reg.range))
 506                                 fprintf(stderr, "remove failure\n"), exit(1);
 507                         break;
 508                 case UFFD_EVENT_REMAP:
 509                         area_dst = (char *)(unsigned long)msg.arg.remap.to;
 510                         break;
 511                 }
 512         }
 513         return (void *)userfaults;
 514 }
 515
 516 pthread_mutex_t uffd_read_mutex = PTHREAD_MUTEX_INITIALIZER;
 517
 518 static void *uffd_read_thread(void *arg)
 519 {
 520         unsigned long *this_cpu_userfaults;
 521         struct uffd_msg msg;
 522         unsigned long offset;
 523         int ret;
 524
 525         this_cpu_userfaults = (unsigned long *) arg;
 526         *this_cpu_userfaults = 0;
 527
 528         pthread_mutex_unlock(&uffd_read_mutex);
 529         /* from here cancellation is ok */
 530
 531         for (;;) {
 532                 ret = read(uffd, &msg, sizeof(msg));
 533                 if (ret != sizeof(msg)) {
 534                         if (ret < 0)
 535                                 perror("blocking read error"), exit(1);
 536                         else
 537                                 fprintf(stderr, "short read\n"), exit(1);
 538                 }
 539                 if (msg.event != UFFD_EVENT_PAGEFAULT)
 540                         fprintf(stderr, "unexpected msg event %u\n",
 541                                 msg.event), exit(1);
 542                 if (bounces & BOUNCE_VERIFY &&
 543                     msg.arg.pagefault.flags & UFFD_PAGEFAULT_FLAG_WRITE)
 544                         fprintf(stderr, "unexpected write fault\n"), exit(1);
 545                 offset = (char *)(unsigned long)msg.arg.pagefault.address -
 546                          area_dst;
 547                 offset &= ~(page_size-1);
 548                 if (copy_page(uffd, offset))
 549                         (*this_cpu_userfaults)++;
 550         }
 551         return (void *)NULL;
 552 }
 553
 554 static void *background_thread(void *arg)
 555 {
 556         unsigned long cpu = (unsigned long) arg;
 557         unsigned long page_nr;
 558
 559         for (page_nr = cpu * nr_pages_per_cpu;
 560              page_nr < (cpu+1) * nr_pages_per_cpu;
 561              page_nr++)
 562                 copy_page_retry(uffd, page_nr * page_size);
 563
 564         return NULL;
 565 }
 566
 567 static int stress(unsigned long *userfaults)
 568 {
 569         unsigned long cpu;
 570         pthread_t locking_threads[nr_cpus];
 571         pthread_t uffd_threads[nr_cpus];
 572         pthread_t background_threads[nr_cpus];
 573         void **_userfaults = (void **) userfaults;
 574
 575         finished = 0;
 576         for (cpu = 0; cpu < nr_cpus; cpu++) {
 577                 if (pthread_create(&locking_threads[cpu], &attr,
 578                                    locking_thread, (void *)cpu))
 579                         return 1;
 580                 if (bounces & BOUNCE_POLL) {
 581                         if (pthread_create(&uffd_threads[cpu], &attr,
 582                                            uffd_poll_thread, (void *)cpu))
 583                                 return 1;
 584                 } else {
 585                         if (pthread_create(&uffd_threads[cpu], &attr,
 586                                            uffd_read_thread,
 587                                            &_userfaults[cpu]))
 588                                 return 1;
 589                         pthread_mutex_lock(&uffd_read_mutex);
 590                 }
 591                 if (pthread_create(&background_threads[cpu], &attr,
 592                                    background_thread, (void *)cpu))
 593                         return 1;
 594         }
 595         for (cpu = 0; cpu < nr_cpus; cpu++)
 596                 if (pthread_join(background_threads[cpu], NULL))
 597                         return 1;
 598
 599         /*
 600          * Be strict and immediately zap area_src, the whole area has
 601          * been transferred already by the background treads. The
 602          * area_src could then be faulted in in a racy way by still
 603          * running uffdio_threads reading zeropages after we zapped
 604          * area_src (but they're guaranteed to get -EEXIST from
 605          * UFFDIO_COPY without writing zero pages into area_dst
 606          * because the background threads already completed).
 607          */
 608         if (uffd_test_ops->release_pages(area_src))
 609                 return 1;
 610
 611         for (cpu = 0; cpu < nr_cpus; cpu++) {
 612                 char c;
 613                 if (bounces & BOUNCE_POLL) {
 614                         if (write(pipefd[cpu*2+1], &c, 1) != 1) {
 615                                 fprintf(stderr, "pipefd write error\n");
 616                                 return 1;
 617                         }
 618                         if (pthread_join(uffd_threads[cpu], &_userfaults[cpu]))
 619                                 return 1;
 620                 } else {
 621                         if (pthread_cancel(uffd_threads[cpu]))
 622                                 return 1;
 623                         if (pthread_join(uffd_threads[cpu], NULL))
 624                                 return 1;
 625                 }
 626         }
 627
 628         finished = 1;
 629         for (cpu = 0; cpu < nr_cpus; cpu++)
 630                 if (pthread_join(locking_threads[cpu], NULL))
 631                         return 1;
 632
 633         return 0;
 634 }
 635
 636 static int userfaultfd_open(int features)
 637 {
 638         struct uffdio_api uffdio_api;
 639
 640         uffd = syscall(__NR_userfaultfd, O_CLOEXEC | O_NONBLOCK);
 641         if (uffd < 0) {
 642                 fprintf(stderr,
 643                         "userfaultfd syscall not available in this kernel\n");
 644                 return 1;
 645         }
 646         uffd_flags = fcntl(uffd, F_GETFD, NULL);
 647
 648         uffdio_api.api = UFFD_API;
 649         uffdio_api.features = features;
 650         if (ioctl(uffd, UFFDIO_API, &uffdio_api)) {
 651                 fprintf(stderr, "UFFDIO_API\n");
 652                 return 1;
 653         }
 654         if (uffdio_api.api != UFFD_API) {
 655                 fprintf(stderr, "UFFDIO_API error %Lu\n", uffdio_api.api);
 656                 return 1;
 657         }
 658
 659         return 0;
 660 }
 661
 662 sigjmp_buf jbuf, *sigbuf;
 663
 664 static void sighndl(int sig, siginfo_t *siginfo, void *ptr)
 665 {
 666         if (sig == SIGBUS) {
 667                 if (sigbuf)
 668                         siglongjmp(*sigbuf, 1);
 669                 abort();
 670         }
 671 }
 672
 673 /*
 674  * For non-cooperative userfaultfd test we fork() a process that will
 675  * generate pagefaults, will mremap the area monitored by the
 676  * userfaultfd and at last this process will release the monitored
 677  * area.
 678  * For the anonymous and shared memory the area is divided into two
 679  * parts, the first part is accessed before mremap, and the second
 680  * part is accessed after mremap. Since hugetlbfs does not support
 681  * mremap, the entire monitored area is accessed in a single pass for
 682  * HUGETLB_TEST.
 683  * The release of the pages currently generates event for shmem and
 684  * anonymous memory (UFFD_EVENT_REMOVE), hence it is not checked
 685  * for hugetlb.
 686  * For signal test(UFFD_FEATURE_SIGBUS), signal_test = 1, we register
 687  * monitored area, generate pagefaults and test that signal is delivered.
 688  * Use UFFDIO_COPY to allocate missing page and retry. For signal_test = 2
 689  * test robustness use case - we release monitored area, fork a process
 690  * that will generate pagefaults and verify signal is generated.
 691  * This also tests UFFD_FEATURE_EVENT_FORK event along with the signal
 692  * feature. Using monitor thread, verify no userfault events are generated.
 693  */
 694 static int faulting_process(int signal_test)
 695 {
 696         unsigned long nr;
 697         unsigned long long count;
 698         unsigned long split_nr_pages;
 699         unsigned long lastnr;
 700         struct sigaction act;
 701         unsigned long signalled = 0;
 702
 703         if (test_type != TEST_HUGETLB)
 704                 split_nr_pages = (nr_pages + 1) / 2;
 705         else
 706                 split_nr_pages = nr_pages;
 707
 708         if (signal_test) {
 709                 sigbuf = &jbuf;
 710                 memset(&act, 0, sizeof(act));
 711                 act.sa_sigaction = sighndl;
 712                 act.sa_flags = SA_SIGINFO;
 713                 if (sigaction(SIGBUS, &act, 0)) {
 714                         perror("sigaction");
 715                         return 1;
 716                 }
 717                 lastnr = (unsigned long)-1;
 718         }
 719
 720         for (nr = 0; nr < split_nr_pages; nr++) {
 721                 if (signal_test) {
 722                         if (sigsetjmp(*sigbuf, 1) != 0) {
 723                                 if (nr == lastnr) {
 724                                         fprintf(stderr, "Signal repeated\n");
 725                                         return 1;
 726                                 }
 727
 728                                 lastnr = nr;
 729                                 if (signal_test == 1) {
 730                                         if (copy_page(uffd, nr * page_size))
 731                                                 signalled++;
 732                                 } else {
 733                                         signalled++;
 734                                         continue;
 735                                 }
 736                         }
 737                 }
 738
 739                 count = *area_count(area_dst, nr);
 740                 if (count != count_verify[nr]) {
 741                         fprintf(stderr,
 742                                 "nr %lu memory corruption %Lu %Lu\n",
 743                                 nr, count,
 744                                 count_verify[nr]), exit(1);
 745                 }
 746         }
 747
 748         if (signal_test)
 749                 return signalled != split_nr_pages;
 750
 751         if (test_type == TEST_HUGETLB)
 752                 return 0;
 753
 754         area_dst = mremap(area_dst, nr_pages * page_size,  nr_pages * page_size,
 755                           MREMAP_MAYMOVE | MREMAP_FIXED, area_src);
 756         if (area_dst == MAP_FAILED)
 757                 perror("mremap"), exit(1);
 758
 759         for (; nr < nr_pages; nr++) {
 760                 count = *area_count(area_dst, nr);
 761                 if (count != count_verify[nr]) {
 762                         fprintf(stderr,
 763                                 "nr %lu memory corruption %Lu %Lu\n",
 764                                 nr, count,
 765                                 count_verify[nr]), exit(1);
 766                 }
 767         }
 768
 769         if (uffd_test_ops->release_pages(area_dst))
 770                 return 1;
 771
 772         for (nr = 0; nr < nr_pages; nr++) {
 773                 if (my_bcmp(area_dst + nr * page_size, zeropage, page_size))
 774                         fprintf(stderr, "nr %lu is not zero\n", nr), exit(1);
 775         }
 776
 777         return 0;
 778 }
 779
 780 static void retry_uffdio_zeropage(int ufd,
 781                                   struct uffdio_zeropage *uffdio_zeropage,
 782                                   unsigned long offset)
 783 {
 784         uffd_test_ops->alias_mapping(&uffdio_zeropage->range.start,
 785                                      uffdio_zeropage->range.len,
 786                                      offset);
 787         if (ioctl(ufd, UFFDIO_ZEROPAGE, uffdio_zeropage)) {
 788                 if (uffdio_zeropage->zeropage != -EEXIST)
 789                         fprintf(stderr, "UFFDIO_ZEROPAGE retry error %Ld\n",
 790                                 uffdio_zeropage->zeropage), exit(1);
 791         } else {
 792                 fprintf(stderr, "UFFDIO_ZEROPAGE retry unexpected %Ld\n",
 793                         uffdio_zeropage->zeropage), exit(1);
 794         }
 795 }
 796
 797 static int __uffdio_zeropage(int ufd, unsigned long offset, bool retry)
 798 {
 799         struct uffdio_zeropage uffdio_zeropage;
 800         int ret;
 801         unsigned long has_zeropage;
 802
 803         has_zeropage = uffd_test_ops->expected_ioctls & (1 << _UFFDIO_ZEROPAGE);
 804
 805         if (offset >= nr_pages * page_size)
 806                 fprintf(stderr, "unexpected offset %lu\n",
 807                         offset), exit(1);
 808         uffdio_zeropage.range.start = (unsigned long) area_dst + offset;
 809         uffdio_zeropage.range.len = page_size;
 810         uffdio_zeropage.mode = 0;
 811         ret = ioctl(ufd, UFFDIO_ZEROPAGE, &uffdio_zeropage);
 812         if (ret) {
 813                 /* real retval in ufdio_zeropage.zeropage */
 814                 if (has_zeropage) {
 815                         if (uffdio_zeropage.zeropage == -EEXIST)
 816                                 fprintf(stderr, "UFFDIO_ZEROPAGE -EEXIST\n"),
 817                                         exit(1);
 818                         else
 819                                 fprintf(stderr, "UFFDIO_ZEROPAGE error %Ld\n",
 820                                         uffdio_zeropage.zeropage), exit(1);
 821                 } else {
 822                         if (uffdio_zeropage.zeropage != -EINVAL)
 823                                 fprintf(stderr,
 824                                         "UFFDIO_ZEROPAGE not -EINVAL %Ld\n",
 825                                         uffdio_zeropage.zeropage), exit(1);
 826                 }
 827         } else if (has_zeropage) {
 828                 if (uffdio_zeropage.zeropage != page_size) {
 829                         fprintf(stderr, "UFFDIO_ZEROPAGE unexpected %Ld\n",
 830                                 uffdio_zeropage.zeropage), exit(1);
 831                 } else {
 832                         if (test_uffdio_zeropage_eexist && retry) {
 833                                 test_uffdio_zeropage_eexist = false;
 834                                 retry_uffdio_zeropage(ufd, &uffdio_zeropage,
 835                                                       offset);
 836                         }
 837                         return 1;
 838                 }
 839         } else {
 840                 fprintf(stderr,
 841                         "UFFDIO_ZEROPAGE succeeded %Ld\n",
 842                         uffdio_zeropage.zeropage), exit(1);
 843         }
 844
 845         return 0;
 846 }
 847
 848 static int uffdio_zeropage(int ufd, unsigned long offset)
 849 {
 850         return __uffdio_zeropage(ufd, offset, false);
 851 }
 852
 853 /* exercise UFFDIO_ZEROPAGE */
 854 static int userfaultfd_zeropage_test(void)
 855 {
 856         struct uffdio_register uffdio_register;
 857         unsigned long expected_ioctls;
 858
 859         printf("testing UFFDIO_ZEROPAGE: ");
 860         fflush(stdout);
 861
 862         if (uffd_test_ops->release_pages(area_dst))
 863                 return 1;
 864
 865         if (userfaultfd_open(0) < 0)
 866                 return 1;
 867         uffdio_register.range.start = (unsigned long) area_dst;
 868         uffdio_register.range.len = nr_pages * page_size;
 869         uffdio_register.mode = UFFDIO_REGISTER_MODE_MISSING;
 870         if (ioctl(uffd, UFFDIO_REGISTER, &uffdio_register))
 871                 fprintf(stderr, "register failure\n"), exit(1);
 872
 873         expected_ioctls = uffd_test_ops->expected_ioctls;
 874         if ((uffdio_register.ioctls & expected_ioctls) !=
 875             expected_ioctls)
 876                 fprintf(stderr,
 877                         "unexpected missing ioctl for anon memory\n"),
 878                         exit(1);
 879
 880         if (uffdio_zeropage(uffd, 0)) {
 881                 if (my_bcmp(area_dst, zeropage, page_size))
 882                         fprintf(stderr, "zeropage is not zero\n"), exit(1);
 883         }
 884
 885         close(uffd);
 886         printf("done.\n");
 887         return 0;
 888 }
 889
 890 static int userfaultfd_events_test(void)
 891 {
 892         struct uffdio_register uffdio_register;
 893         unsigned long expected_ioctls;
 894         unsigned long userfaults;
 895         pthread_t uffd_mon;
 896         int err, features;
 897         pid_t pid;
 898         char c;
 899
 900         printf("testing events (fork, remap, remove): ");
 901         fflush(stdout);
 902
 903         if (uffd_test_ops->release_pages(area_dst))
 904                 return 1;
 905
 906         features = UFFD_FEATURE_EVENT_FORK | UFFD_FEATURE_EVENT_REMAP |
 907                 UFFD_FEATURE_EVENT_REMOVE;
 908         if (userfaultfd_open(features) < 0)
 909                 return 1;
 910         fcntl(uffd, F_SETFL, uffd_flags | O_NONBLOCK);
 911
 912         uffdio_register.range.start = (unsigned long) area_dst;
 913         uffdio_register.range.len = nr_pages * page_size;
 914         uffdio_register.mode = UFFDIO_REGISTER_MODE_MISSING;
 915         if (ioctl(uffd, UFFDIO_REGISTER, &uffdio_register))
 916                 fprintf(stderr, "register failure\n"), exit(1);
 917
 918         expected_ioctls = uffd_test_ops->expected_ioctls;
 919         if ((uffdio_register.ioctls & expected_ioctls) !=
 920             expected_ioctls)
 921                 fprintf(stderr,
 922                         "unexpected missing ioctl for anon memory\n"),
 923                         exit(1);
 924
 925         if (pthread_create(&uffd_mon, &attr, uffd_poll_thread, NULL))
 926                 perror("uffd_poll_thread create"), exit(1);
 927
 928         pid = fork();
 929         if (pid < 0)
 930                 perror("fork"), exit(1);
 931
 932         if (!pid)
 933                 return faulting_process(0);
 934
 935         waitpid(pid, &err, 0);
 936         if (err)
 937                 fprintf(stderr, "faulting process failed\n"), exit(1);
 938
 939         if (write(pipefd[1], &c, sizeof(c)) != sizeof(c))
 940                 perror("pipe write"), exit(1);
 941         if (pthread_join(uffd_mon, (void **)&userfaults))
 942                 return 1;
 943
 944         close(uffd);
 945         printf("userfaults: %ld\n", userfaults);
 946
 947         return userfaults != nr_pages;
 948 }
 949
 950 static int userfaultfd_sig_test(void)
 951 {
 952         struct uffdio_register uffdio_register;
 953         unsigned long expected_ioctls;
 954         unsigned long userfaults;
 955         pthread_t uffd_mon;
 956         int err, features;
 957         pid_t pid;
 958         char c;
 959
 960         printf("testing signal delivery: ");
 961         fflush(stdout);
 962
 963         if (uffd_test_ops->release_pages(area_dst))
 964                 return 1;
 965
 966         features = UFFD_FEATURE_EVENT_FORK|UFFD_FEATURE_SIGBUS;
 967         if (userfaultfd_open(features) < 0)
 968                 return 1;
 969         fcntl(uffd, F_SETFL, uffd_flags | O_NONBLOCK);
 970
 971         uffdio_register.range.start = (unsigned long) area_dst;
 972         uffdio_register.range.len = nr_pages * page_size;
 973         uffdio_register.mode = UFFDIO_REGISTER_MODE_MISSING;
 974         if (ioctl(uffd, UFFDIO_REGISTER, &uffdio_register))
 975                 fprintf(stderr, "register failure\n"), exit(1);
 976
 977         expected_ioctls = uffd_test_ops->expected_ioctls;
 978         if ((uffdio_register.ioctls & expected_ioctls) !=
 979             expected_ioctls)
 980                 fprintf(stderr,
 981                         "unexpected missing ioctl for anon memory\n"),
 982                         exit(1);
 983
 984         if (faulting_process(1))
 985                 fprintf(stderr, "faulting process failed\n"), exit(1);
 986
 987         if (uffd_test_ops->release_pages(area_dst))
 988                 return 1;
 989
 990         if (pthread_create(&uffd_mon, &attr, uffd_poll_thread, NULL))
 991                 perror("uffd_poll_thread create"), exit(1);
 992
 993         pid = fork();
 994         if (pid < 0)
 995                 perror("fork"), exit(1);
 996
 997         if (!pid)
 998                 exit(faulting_process(2));
 999
1000         waitpid(pid, &err, 0);
1001         if (err)
1002                 fprintf(stderr, "faulting process failed\n"), exit(1);
1003
1004         if (write(pipefd[1], &c, sizeof(c)) != sizeof(c))
1005                 perror("pipe write"), exit(1);
1006         if (pthread_join(uffd_mon, (void **)&userfaults))
1007                 return 1;
1008
1009         printf("done.\n");
1010         if (userfaults)
1011                 fprintf(stderr, "Signal test failed, userfaults: %ld\n",
1012                         userfaults);
1013         close(uffd);
1014         return userfaults != 0;
1015 }
1016 static int userfaultfd_stress(void)
1017 {
1018         void *area;
1019         char *tmp_area;
1020         unsigned long nr;
1021         struct uffdio_register uffdio_register;
1022         unsigned long cpu;
1023         int err;
1024         unsigned long userfaults[nr_cpus];
1025
1026         uffd_test_ops->allocate_area((void **)&area_src);
1027         if (!area_src)
1028                 return 1;
1029         uffd_test_ops->allocate_area((void **)&area_dst);
1030         if (!area_dst)
1031                 return 1;
1032
1033         if (userfaultfd_open(0) < 0)
1034                 return 1;
1035
1036         count_verify = malloc(nr_pages * sizeof(unsigned long long));
1037         if (!count_verify) {
1038                 perror("count_verify");
1039                 return 1;
1040         }
1041
1042         for (nr = 0; nr < nr_pages; nr++) {
1043                 *area_mutex(area_src, nr) = (pthread_mutex_t)
1044                         PTHREAD_MUTEX_INITIALIZER;
1045                 count_verify[nr] = *area_count(area_src, nr) = 1;
1046                 /*
1047                  * In the transition between 255 to 256, powerpc will
1048                  * read out of order in my_bcmp and see both bytes as
1049                  * zero, so leave a placeholder below always non-zero
1050                  * after the count, to avoid my_bcmp to trigger false
1051                  * positives.
1052                  */
1053                 *(area_count(area_src, nr) + 1) = 1;
1054         }
1055
1056         pipefd = malloc(sizeof(int) * nr_cpus * 2);
1057         if (!pipefd) {
1058                 perror("pipefd");
1059                 return 1;
1060         }
1061         for (cpu = 0; cpu < nr_cpus; cpu++) {
1062                 if (pipe2(&pipefd[cpu*2], O_CLOEXEC | O_NONBLOCK)) {
1063                         perror("pipe");
1064                         return 1;
1065                 }
1066         }
1067
1068         if (posix_memalign(&area, page_size, page_size)) {
1069                 fprintf(stderr, "out of memory\n");
1070                 return 1;
1071         }
1072         zeropage = area;
1073         bzero(zeropage, page_size);
1074
1075         pthread_mutex_lock(&uffd_read_mutex);
1076
1077         pthread_attr_init(&attr);
1078         pthread_attr_setstacksize(&attr, 16*1024*1024);
1079
1080         err = 0;
1081         while (bounces--) {
1082                 unsigned long expected_ioctls;
1083
1084                 printf("bounces: %d, mode:", bounces);
1085                 if (bounces & BOUNCE_RANDOM)
1086                         printf(" rnd");
1087                 if (bounces & BOUNCE_RACINGFAULTS)
1088                         printf(" racing");
1089                 if (bounces & BOUNCE_VERIFY)
1090                         printf(" ver");
1091                 if (bounces & BOUNCE_POLL)
1092                         printf(" poll");
1093                 printf(", ");
1094                 fflush(stdout);
1095
1096                 if (bounces & BOUNCE_POLL)
1097                         fcntl(uffd, F_SETFL, uffd_flags | O_NONBLOCK);
1098                 else
1099                         fcntl(uffd, F_SETFL, uffd_flags & ~O_NONBLOCK);
1100
1101                 /* register */
1102                 uffdio_register.range.start = (unsigned long) area_dst;
1103                 uffdio_register.range.len = nr_pages * page_size;
1104                 uffdio_register.mode = UFFDIO_REGISTER_MODE_MISSING;
1105                 if (ioctl(uffd, UFFDIO_REGISTER, &uffdio_register)) {
1106                         fprintf(stderr, "register failure\n");
1107                         return 1;
1108                 }
1109                 expected_ioctls = uffd_test_ops->expected_ioctls;
1110                 if ((uffdio_register.ioctls & expected_ioctls) !=
1111                     expected_ioctls) {
1112                         fprintf(stderr,
1113                                 "unexpected missing ioctl for anon memory\n");
1114                         return 1;
1115                 }
1116
1117                 if (area_dst_alias) {
1118                         uffdio_register.range.start = (unsigned long)
1119                                 area_dst_alias;
1120                         if (ioctl(uffd, UFFDIO_REGISTER, &uffdio_register)) {
1121                                 fprintf(stderr, "register failure alias\n");
1122                                 return 1;
1123                         }
1124                 }
1125
1126                 /*
1127                  * The madvise done previously isn't enough: some
1128                  * uffd_thread could have read userfaults (one of
1129                  * those already resolved by the background thread)
1130                  * and it may be in the process of calling
1131                  * UFFDIO_COPY. UFFDIO_COPY will read the zapped
1132                  * area_src and it would map a zero page in it (of
1133                  * course such a UFFDIO_COPY is perfectly safe as it'd
1134                  * return -EEXIST). The problem comes at the next
1135                  * bounce though: that racing UFFDIO_COPY would
1136                  * generate zeropages in the area_src, so invalidating
1137                  * the previous MADV_DONTNEED. Without this additional
1138                  * MADV_DONTNEED those zeropages leftovers in the
1139                  * area_src would lead to -EEXIST failure during the
1140                  * next bounce, effectively leaving a zeropage in the
1141                  * area_dst.
1142                  *
1143                  * Try to comment this out madvise to see the memory
1144                  * corruption being caught pretty quick.
1145                  *
1146                  * khugepaged is also inhibited to collapse THP after
1147                  * MADV_DONTNEED only after the UFFDIO_REGISTER, so it's
1148                  * required to MADV_DONTNEED here.
1149                  */
1150                 if (uffd_test_ops->release_pages(area_dst))
1151                         return 1;
1152
1153                 /* bounce pass */
1154                 if (stress(userfaults))
1155                         return 1;
1156
1157                 /* unregister */
1158                 if (ioctl(uffd, UFFDIO_UNREGISTER, &uffdio_register.range)) {
1159                         fprintf(stderr, "unregister failure\n");
1160                         return 1;
1161                 }
1162                 if (area_dst_alias) {
1163                         uffdio_register.range.start = (unsigned long) area_dst;
1164                         if (ioctl(uffd, UFFDIO_UNREGISTER,
1165                                   &uffdio_register.range)) {
1166                                 fprintf(stderr, "unregister failure alias\n");
1167                                 return 1;
1168                         }
1169                 }
1170
1171                 /* verification */
1172                 if (bounces & BOUNCE_VERIFY) {
1173                         for (nr = 0; nr < nr_pages; nr++) {
1174                                 if (*area_count(area_dst, nr) != count_verify[nr]) {
1175                                         fprintf(stderr,
1176                                                 "error area_count %Lu %Lu %lu\n",
1177                                                 *area_count(area_src, nr),
1178                                                 count_verify[nr],
1179                                                 nr);
1180                                         err = 1;
1181                                         bounces = 0;
1182                                 }
1183                         }
1184                 }
1185
1186                 /* prepare next bounce */
1187                 tmp_area = area_src;
1188                 area_src = area_dst;
1189                 area_dst = tmp_area;
1190
1191                 tmp_area = area_src_alias;
1192                 area_src_alias = area_dst_alias;
1193                 area_dst_alias = tmp_area;
1194
1195                 printf("userfaults:");
1196                 for (cpu = 0; cpu < nr_cpus; cpu++)
1197                         printf(" %lu", userfaults[cpu]);
1198                 printf("\n");
1199         }
1200
1201         if (err)
1202                 return err;
1203
1204         close(uffd);
1205         return userfaultfd_zeropage_test() || userfaultfd_sig_test()
1206                 || userfaultfd_events_test();
1207 }
1208
1209 /*
1210  * Copied from mlock2-tests.c
1211  */
1212 unsigned long default_huge_page_size(void)
1213 {
1214         unsigned long hps = 0;
1215         char *line = NULL;
1216         size_t linelen = 0;
1217         FILE *f = fopen("/proc/meminfo", "r");
1218
1219         if (!f)
1220                 return 0;
1221         while (getline(&line, &linelen, f) > 0) {
1222                 if (sscanf(line, "Hugepagesize:       %lu kB", &hps) == 1) {
1223                         hps <<= 10;
1224                         break;
1225                 }
1226         }
1227
1228         free(line);
1229         fclose(f);
1230         return hps;
1231 }
1232
1233 static void set_test_type(const char *type)
1234 {
1235         if (!strcmp(type, "anon")) {
1236                 test_type = TEST_ANON;
1237                 uffd_test_ops = &anon_uffd_test_ops;
1238         } else if (!strcmp(type, "hugetlb")) {
1239                 test_type = TEST_HUGETLB;
1240                 uffd_test_ops = &hugetlb_uffd_test_ops;
1241         } else if (!strcmp(type, "hugetlb_shared")) {
1242                 map_shared = true;
1243                 test_type = TEST_HUGETLB;
1244                 uffd_test_ops = &hugetlb_uffd_test_ops;
1245         } else if (!strcmp(type, "shmem")) {
1246                 map_shared = true;
1247                 test_type = TEST_SHMEM;
1248                 uffd_test_ops = &shmem_uffd_test_ops;
1249         } else {
1250                 fprintf(stderr, "Unknown test type: %s\n", type), exit(1);
1251         }
1252
1253         if (test_type == TEST_HUGETLB)
1254                 page_size = default_huge_page_size();
1255         else
1256                 page_size = sysconf(_SC_PAGE_SIZE);
1257
1258         if (!page_size)
1259                 fprintf(stderr, "Unable to determine page size\n"),
1260                                 exit(2);
1261         if ((unsigned long) area_count(NULL, 0) + sizeof(unsigned long long) * 2
1262             > page_size)
1263                 fprintf(stderr, "Impossible to run this test\n"), exit(2);
1264 }
1265
1266 static void sigalrm(int sig)
1267 {
1268         if (sig != SIGALRM)
1269                 abort();
1270         test_uffdio_copy_eexist = true;
1271         test_uffdio_zeropage_eexist = true;
1272         alarm(ALARM_INTERVAL_SECS);
1273 }
1274
1275 int main(int argc, char **argv)
1276 {
1277         if (argc < 4)
1278                 fprintf(stderr, "Usage: <test type> <MiB> <bounces> [hugetlbfs_file]\n"),
1279                                 exit(1);
1280
1281         if (signal(SIGALRM, sigalrm) == SIG_ERR)
1282                 fprintf(stderr, "failed to arm SIGALRM"), exit(1);
1283         alarm(ALARM_INTERVAL_SECS);
1284
1285         set_test_type(argv[1]);
1286
1287         nr_cpus = sysconf(_SC_NPROCESSORS_ONLN);
1288         nr_pages_per_cpu = atol(argv[2]) * 1024*1024 / page_size /
1289                 nr_cpus;
1290         if (!nr_pages_per_cpu) {
1291                 fprintf(stderr, "invalid MiB\n");
1292                 fprintf(stderr, "Usage: <MiB> <bounces>\n"), exit(1);
1293         }
1294
1295         bounces = atoi(argv[3]);
1296         if (bounces <= 0) {
1297                 fprintf(stderr, "invalid bounces\n");
1298                 fprintf(stderr, "Usage: <MiB> <bounces>\n"), exit(1);
1299         }
1300         nr_pages = nr_pages_per_cpu * nr_cpus;
1301
1302         if (test_type == TEST_HUGETLB) {
1303                 if (argc < 5)
1304                         fprintf(stderr, "Usage: hugetlb <MiB> <bounces> <hugetlbfs_file>\n"),
1305                                 exit(1);
1306                 huge_fd = open(argv[4], O_CREAT | O_RDWR, 0755);
1307                 if (huge_fd < 0) {
1308                         fprintf(stderr, "Open of %s failed", argv[3]);
1309                         perror("open");
1310                         exit(1);
1311                 }
1312                 if (ftruncate(huge_fd, 0)) {
1313                         fprintf(stderr, "ftruncate %s to size 0 failed", argv[3]);
1314                         perror("ftruncate");
1315                         exit(1);
1316                 }
1317         }
1318         printf("nr_pages: %lu, nr_pages_per_cpu: %lu\n",
1319                nr_pages, nr_pages_per_cpu);
1320         return userfaultfd_stress();
1321 }
1322
1323 #else /* __NR_userfaultfd */
1324
1325 #warning "missing __NR_userfaultfd definition"
1326
1327 int main(void)
1328 {
1329         printf("skip: Skipping userfaultfd test (missing __NR_userfaultfd)\n");
1330         return KSFT_SKIP;
1331 }
1332
1333 #endif /* __NR_userfaultfd */