tools/testing/selftests/rseq/param_test.c

   1 // SPDX-License-Identifier: LGPL-2.1
   2 #define _GNU_SOURCE
   3 #include <assert.h>
   4 #include <pthread.h>
   5 #include <sched.h>
   6 #include <stdint.h>
   7 #include <stdio.h>
   8 #include <stdlib.h>
   9 #include <string.h>
  10 #include <syscall.h>
  11 #include <unistd.h>
  12 #include <poll.h>
  13 #include <sys/types.h>
  14 #include <signal.h>
  15 #include <errno.h>
  16 #include <stddef.h>
  17
  18 static inline pid_t rseq_gettid(void)
  19 {
  20         return syscall(__NR_gettid);
  21 }
  22
  23 #define NR_INJECT       9
  24 static int loop_cnt[NR_INJECT + 1];
  25
  26 static int loop_cnt_1 asm("asm_loop_cnt_1") __attribute__((used));
  27 static int loop_cnt_2 asm("asm_loop_cnt_2") __attribute__((used));
  28 static int loop_cnt_3 asm("asm_loop_cnt_3") __attribute__((used));
  29 static int loop_cnt_4 asm("asm_loop_cnt_4") __attribute__((used));
  30 static int loop_cnt_5 asm("asm_loop_cnt_5") __attribute__((used));
  31 static int loop_cnt_6 asm("asm_loop_cnt_6") __attribute__((used));
  32
  33 static int opt_modulo, verbose;
  34
  35 static int opt_yield, opt_signal, opt_sleep,
  36                 opt_disable_rseq, opt_threads = 200,
  37                 opt_disable_mod = 0, opt_test = 's', opt_mb = 0;
  38
  39 #ifndef RSEQ_SKIP_FASTPATH
  40 static long long opt_reps = 5000;
  41 #else
  42 static long long opt_reps = 100;
  43 #endif
  44
  45 static __thread __attribute__((tls_model("initial-exec")))
  46 unsigned int signals_delivered;
  47
  48 #ifndef BENCHMARK
  49
  50 static __thread __attribute__((tls_model("initial-exec"), unused))
  51 unsigned int yield_mod_cnt, nr_abort;
  52
  53 #define printf_verbose(fmt, ...)                        \
  54         do {                                            \
  55                 if (verbose)                            \
  56                         printf(fmt, ## __VA_ARGS__);    \
  57         } while (0)
  58
  59 #ifdef __i386__
  60
  61 #define INJECT_ASM_REG  "eax"
  62
  63 #define RSEQ_INJECT_CLOBBER \
  64         , INJECT_ASM_REG
  65
  66 #define RSEQ_INJECT_ASM(n) \
  67         "mov asm_loop_cnt_" #n ", %%" INJECT_ASM_REG "\n\t" \
  68         "test %%" INJECT_ASM_REG ",%%" INJECT_ASM_REG "\n\t" \
  69         "jz 333f\n\t" \
  70         "222:\n\t" \
  71         "dec %%" INJECT_ASM_REG "\n\t" \
  72         "jnz 222b\n\t" \
  73         "333:\n\t"
  74
  75 #elif defined(__x86_64__)
  76
  77 #define INJECT_ASM_REG_P        "rax"
  78 #define INJECT_ASM_REG          "eax"
  79
  80 #define RSEQ_INJECT_CLOBBER \
  81         , INJECT_ASM_REG_P \
  82         , INJECT_ASM_REG
  83
  84 #define RSEQ_INJECT_ASM(n) \
  85         "lea asm_loop_cnt_" #n "(%%rip), %%" INJECT_ASM_REG_P "\n\t" \
  86         "mov (%%" INJECT_ASM_REG_P "), %%" INJECT_ASM_REG "\n\t" \
  87         "test %%" INJECT_ASM_REG ",%%" INJECT_ASM_REG "\n\t" \
  88         "jz 333f\n\t" \
  89         "222:\n\t" \
  90         "dec %%" INJECT_ASM_REG "\n\t" \
  91         "jnz 222b\n\t" \
  92         "333:\n\t"
  93
  94 #elif defined(__s390__)
  95
  96 #define RSEQ_INJECT_INPUT \
  97         , [loop_cnt_1]"m"(loop_cnt[1]) \
  98         , [loop_cnt_2]"m"(loop_cnt[2]) \
  99         , [loop_cnt_3]"m"(loop_cnt[3]) \
 100         , [loop_cnt_4]"m"(loop_cnt[4]) \
 101         , [loop_cnt_5]"m"(loop_cnt[5]) \
 102         , [loop_cnt_6]"m"(loop_cnt[6])
 103
 104 #define INJECT_ASM_REG  "r12"
 105
 106 #define RSEQ_INJECT_CLOBBER \
 107         , INJECT_ASM_REG
 108
 109 #define RSEQ_INJECT_ASM(n) \
 110         "l %%" INJECT_ASM_REG ", %[loop_cnt_" #n "]\n\t" \
 111         "ltr %%" INJECT_ASM_REG ", %%" INJECT_ASM_REG "\n\t" \
 112         "je 333f\n\t" \
 113         "222:\n\t" \
 114         "ahi %%" INJECT_ASM_REG ", -1\n\t" \
 115         "jnz 222b\n\t" \
 116         "333:\n\t"
 117
 118 #elif defined(__ARMEL__)
 119
 120 #define RSEQ_INJECT_INPUT \
 121         , [loop_cnt_1]"m"(loop_cnt[1]) \
 122         , [loop_cnt_2]"m"(loop_cnt[2]) \
 123         , [loop_cnt_3]"m"(loop_cnt[3]) \
 124         , [loop_cnt_4]"m"(loop_cnt[4]) \
 125         , [loop_cnt_5]"m"(loop_cnt[5]) \
 126         , [loop_cnt_6]"m"(loop_cnt[6])
 127
 128 #define INJECT_ASM_REG  "r4"
 129
 130 #define RSEQ_INJECT_CLOBBER \
 131         , INJECT_ASM_REG
 132
 133 #define RSEQ_INJECT_ASM(n) \
 134         "ldr " INJECT_ASM_REG ", %[loop_cnt_" #n "]\n\t" \
 135         "cmp " INJECT_ASM_REG ", #0\n\t" \
 136         "beq 333f\n\t" \
 137         "222:\n\t" \
 138         "subs " INJECT_ASM_REG ", #1\n\t" \
 139         "bne 222b\n\t" \
 140         "333:\n\t"
 141
 142 #elif defined(__AARCH64EL__)
 143
 144 #define RSEQ_INJECT_INPUT \
 145         , [loop_cnt_1] "Qo" (loop_cnt[1]) \
 146         , [loop_cnt_2] "Qo" (loop_cnt[2]) \
 147         , [loop_cnt_3] "Qo" (loop_cnt[3]) \
 148         , [loop_cnt_4] "Qo" (loop_cnt[4]) \
 149         , [loop_cnt_5] "Qo" (loop_cnt[5]) \
 150         , [loop_cnt_6] "Qo" (loop_cnt[6])
 151
 152 #define INJECT_ASM_REG  RSEQ_ASM_TMP_REG32
 153
 154 #define RSEQ_INJECT_ASM(n) \
 155         "       ldr     " INJECT_ASM_REG ", %[loop_cnt_" #n "]\n"       \
 156         "       cbz     " INJECT_ASM_REG ", 333f\n"                     \
 157         "222:\n"                                                        \
 158         "       sub     " INJECT_ASM_REG ", " INJECT_ASM_REG ", #1\n"   \
 159         "       cbnz    " INJECT_ASM_REG ", 222b\n"                     \
 160         "333:\n"
 161
 162 #elif __PPC__
 163
 164 #define RSEQ_INJECT_INPUT \
 165         , [loop_cnt_1]"m"(loop_cnt[1]) \
 166         , [loop_cnt_2]"m"(loop_cnt[2]) \
 167         , [loop_cnt_3]"m"(loop_cnt[3]) \
 168         , [loop_cnt_4]"m"(loop_cnt[4]) \
 169         , [loop_cnt_5]"m"(loop_cnt[5]) \
 170         , [loop_cnt_6]"m"(loop_cnt[6])
 171
 172 #define INJECT_ASM_REG  "r18"
 173
 174 #define RSEQ_INJECT_CLOBBER \
 175         , INJECT_ASM_REG
 176
 177 #define RSEQ_INJECT_ASM(n) \
 178         "lwz %%" INJECT_ASM_REG ", %[loop_cnt_" #n "]\n\t" \
 179         "cmpwi %%" INJECT_ASM_REG ", 0\n\t" \
 180         "beq 333f\n\t" \
 181         "222:\n\t" \
 182         "subic. %%" INJECT_ASM_REG ", %%" INJECT_ASM_REG ", 1\n\t" \
 183         "bne 222b\n\t" \
 184         "333:\n\t"
 185
 186 #elif defined(__mips__)
 187
 188 #define RSEQ_INJECT_INPUT \
 189         , [loop_cnt_1]"m"(loop_cnt[1]) \
 190         , [loop_cnt_2]"m"(loop_cnt[2]) \
 191         , [loop_cnt_3]"m"(loop_cnt[3]) \
 192         , [loop_cnt_4]"m"(loop_cnt[4]) \
 193         , [loop_cnt_5]"m"(loop_cnt[5]) \
 194         , [loop_cnt_6]"m"(loop_cnt[6])
 195
 196 #define INJECT_ASM_REG  "$5"
 197
 198 #define RSEQ_INJECT_CLOBBER \
 199         , INJECT_ASM_REG
 200
 201 #define RSEQ_INJECT_ASM(n) \
 202         "lw " INJECT_ASM_REG ", %[loop_cnt_" #n "]\n\t" \
 203         "beqz " INJECT_ASM_REG ", 333f\n\t" \
 204         "222:\n\t" \
 205         "addiu " INJECT_ASM_REG ", -1\n\t" \
 206         "bnez " INJECT_ASM_REG ", 222b\n\t" \
 207         "333:\n\t"
 208
 209 #else
 210 #error unsupported target
 211 #endif
 212
 213 #define RSEQ_INJECT_FAILED \
 214         nr_abort++;
 215
 216 #define RSEQ_INJECT_C(n) \
 217 { \
 218         int loc_i, loc_nr_loops = loop_cnt[n]; \
 219         \
 220         for (loc_i = 0; loc_i < loc_nr_loops; loc_i++) { \
 221                 rseq_barrier(); \
 222         } \
 223         if (loc_nr_loops == -1 && opt_modulo) { \
 224                 if (yield_mod_cnt == opt_modulo - 1) { \
 225                         if (opt_sleep > 0) \
 226                                 poll(NULL, 0, opt_sleep); \
 227                         if (opt_yield) \
 228                                 sched_yield(); \
 229                         if (opt_signal) \
 230                                 raise(SIGUSR1); \
 231                         yield_mod_cnt = 0; \
 232                 } else { \
 233                         yield_mod_cnt++; \
 234                 } \
 235         } \
 236 }
 237
 238 #else
 239
 240 #define printf_verbose(fmt, ...)
 241
 242 #endif /* BENCHMARK */
 243
 244 #include "rseq.h"
 245
 246 struct percpu_lock_entry {
 247         intptr_t v;
 248 } __attribute__((aligned(128)));
 249
 250 struct percpu_lock {
 251         struct percpu_lock_entry c[CPU_SETSIZE];
 252 };
 253
 254 struct test_data_entry {
 255         intptr_t count;
 256 } __attribute__((aligned(128)));
 257
 258 struct spinlock_test_data {
 259         struct percpu_lock lock;
 260         struct test_data_entry c[CPU_SETSIZE];
 261 };
 262
 263 struct spinlock_thread_test_data {
 264         struct spinlock_test_data *data;
 265         long long reps;
 266         int reg;
 267 };
 268
 269 struct inc_test_data {
 270         struct test_data_entry c[CPU_SETSIZE];
 271 };
 272
 273 struct inc_thread_test_data {
 274         struct inc_test_data *data;
 275         long long reps;
 276         int reg;
 277 };
 278
 279 struct percpu_list_node {
 280         intptr_t data;
 281         struct percpu_list_node *next;
 282 };
 283
 284 struct percpu_list_entry {
 285         struct percpu_list_node *head;
 286 } __attribute__((aligned(128)));
 287
 288 struct percpu_list {
 289         struct percpu_list_entry c[CPU_SETSIZE];
 290 };
 291
 292 #define BUFFER_ITEM_PER_CPU     100
 293
 294 struct percpu_buffer_node {
 295         intptr_t data;
 296 };
 297
 298 struct percpu_buffer_entry {
 299         intptr_t offset;
 300         intptr_t buflen;
 301         struct percpu_buffer_node **array;
 302 } __attribute__((aligned(128)));
 303
 304 struct percpu_buffer {
 305         struct percpu_buffer_entry c[CPU_SETSIZE];
 306 };
 307
 308 #define MEMCPY_BUFFER_ITEM_PER_CPU      100
 309
 310 struct percpu_memcpy_buffer_node {
 311         intptr_t data1;
 312         uint64_t data2;
 313 };
 314
 315 struct percpu_memcpy_buffer_entry {
 316         intptr_t offset;
 317         intptr_t buflen;
 318         struct percpu_memcpy_buffer_node *array;
 319 } __attribute__((aligned(128)));
 320
 321 struct percpu_memcpy_buffer {
 322         struct percpu_memcpy_buffer_entry c[CPU_SETSIZE];
 323 };
 324
 325 /* A simple percpu spinlock. Grabs lock on current cpu. */
 326 static int rseq_this_cpu_lock(struct percpu_lock *lock)
 327 {
 328         int cpu;
 329
 330         for (;;) {
 331                 int ret;
 332
 333                 cpu = rseq_cpu_start();
 334                 ret = rseq_cmpeqv_storev(&lock->c[cpu].v,
 335                                          0, 1, cpu);
 336                 if (rseq_likely(!ret))
 337                         break;
 338                 /* Retry if comparison fails or rseq aborts. */
 339         }
 340         /*
 341          * Acquire semantic when taking lock after control dependency.
 342          * Matches rseq_smp_store_release().
 343          */
 344         rseq_smp_acquire__after_ctrl_dep();
 345         return cpu;
 346 }
 347
 348 static void rseq_percpu_unlock(struct percpu_lock *lock, int cpu)
 349 {
 350         assert(lock->c[cpu].v == 1);
 351         /*
 352          * Release lock, with release semantic. Matches
 353          * rseq_smp_acquire__after_ctrl_dep().
 354          */
 355         rseq_smp_store_release(&lock->c[cpu].v, 0);
 356 }
 357
 358 void *test_percpu_spinlock_thread(void *arg)
 359 {
 360         struct spinlock_thread_test_data *thread_data = arg;
 361         struct spinlock_test_data *data = thread_data->data;
 362         long long i, reps;
 363
 364         if (!opt_disable_rseq && thread_data->reg &&
 365             rseq_register_current_thread())
 366                 abort();
 367         reps = thread_data->reps;
 368         for (i = 0; i < reps; i++) {
 369                 int cpu = rseq_cpu_start();
 370
 371                 cpu = rseq_this_cpu_lock(&data->lock);
 372                 data->c[cpu].count++;
 373                 rseq_percpu_unlock(&data->lock, cpu);
 374 #ifndef BENCHMARK
 375                 if (i != 0 && !(i % (reps / 10)))
 376                         printf_verbose("tid %d: count %lld\n",
 377                                        (int) rseq_gettid(), i);
 378 #endif
 379         }
 380         printf_verbose("tid %d: number of rseq abort: %d, signals delivered: %u\n",
 381                        (int) rseq_gettid(), nr_abort, signals_delivered);
 382         if (!opt_disable_rseq && thread_data->reg &&
 383             rseq_unregister_current_thread())
 384                 abort();
 385         return NULL;
 386 }
 387
 388 /*
 389  * A simple test which implements a sharded counter using a per-cpu
 390  * lock.  Obviously real applications might prefer to simply use a
 391  * per-cpu increment; however, this is reasonable for a test and the
 392  * lock can be extended to synchronize more complicated operations.
 393  */
 394 void test_percpu_spinlock(void)
 395 {
 396         const int num_threads = opt_threads;
 397         int i, ret;
 398         uint64_t sum;
 399         pthread_t test_threads[num_threads];
 400         struct spinlock_test_data data;
 401         struct spinlock_thread_test_data thread_data[num_threads];
 402
 403         memset(&data, 0, sizeof(data));
 404         for (i = 0; i < num_threads; i++) {
 405                 thread_data[i].reps = opt_reps;
 406                 if (opt_disable_mod <= 0 || (i % opt_disable_mod))
 407                         thread_data[i].reg = 1;
 408                 else
 409                         thread_data[i].reg = 0;
 410                 thread_data[i].data = &data;
 411                 ret = pthread_create(&test_threads[i], NULL,
 412                                      test_percpu_spinlock_thread,
 413                                      &thread_data[i]);
 414                 if (ret) {
 415                         errno = ret;
 416                         perror("pthread_create");
 417                         abort();
 418                 }
 419         }
 420
 421         for (i = 0; i < num_threads; i++) {
 422                 ret = pthread_join(test_threads[i], NULL);
 423                 if (ret) {
 424                         errno = ret;
 425                         perror("pthread_join");
 426                         abort();
 427                 }
 428         }
 429
 430         sum = 0;
 431         for (i = 0; i < CPU_SETSIZE; i++)
 432                 sum += data.c[i].count;
 433
 434         assert(sum == (uint64_t)opt_reps * num_threads);
 435 }
 436
 437 void *test_percpu_inc_thread(void *arg)
 438 {
 439         struct inc_thread_test_data *thread_data = arg;
 440         struct inc_test_data *data = thread_data->data;
 441         long long i, reps;
 442
 443         if (!opt_disable_rseq && thread_data->reg &&
 444             rseq_register_current_thread())
 445                 abort();
 446         reps = thread_data->reps;
 447         for (i = 0; i < reps; i++) {
 448                 int ret;
 449
 450                 do {
 451                         int cpu;
 452
 453                         cpu = rseq_cpu_start();
 454                         ret = rseq_addv(&data->c[cpu].count, 1, cpu);
 455                 } while (rseq_unlikely(ret));
 456 #ifndef BENCHMARK
 457                 if (i != 0 && !(i % (reps / 10)))
 458                         printf_verbose("tid %d: count %lld\n",
 459                                        (int) rseq_gettid(), i);
 460 #endif
 461         }
 462         printf_verbose("tid %d: number of rseq abort: %d, signals delivered: %u\n",
 463                        (int) rseq_gettid(), nr_abort, signals_delivered);
 464         if (!opt_disable_rseq && thread_data->reg &&
 465             rseq_unregister_current_thread())
 466                 abort();
 467         return NULL;
 468 }
 469
 470 void test_percpu_inc(void)
 471 {
 472         const int num_threads = opt_threads;
 473         int i, ret;
 474         uint64_t sum;
 475         pthread_t test_threads[num_threads];
 476         struct inc_test_data data;
 477         struct inc_thread_test_data thread_data[num_threads];
 478
 479         memset(&data, 0, sizeof(data));
 480         for (i = 0; i < num_threads; i++) {
 481                 thread_data[i].reps = opt_reps;
 482                 if (opt_disable_mod <= 0 || (i % opt_disable_mod))
 483                         thread_data[i].reg = 1;
 484                 else
 485                         thread_data[i].reg = 0;
 486                 thread_data[i].data = &data;
 487                 ret = pthread_create(&test_threads[i], NULL,
 488                                      test_percpu_inc_thread,
 489                                      &thread_data[i]);
 490                 if (ret) {
 491                         errno = ret;
 492                         perror("pthread_create");
 493                         abort();
 494                 }
 495         }
 496
 497         for (i = 0; i < num_threads; i++) {
 498                 ret = pthread_join(test_threads[i], NULL);
 499                 if (ret) {
 500                         errno = ret;
 501                         perror("pthread_join");
 502                         abort();
 503                 }
 504         }
 505
 506         sum = 0;
 507         for (i = 0; i < CPU_SETSIZE; i++)
 508                 sum += data.c[i].count;
 509
 510         assert(sum == (uint64_t)opt_reps * num_threads);
 511 }
 512
 513 void this_cpu_list_push(struct percpu_list *list,
 514                         struct percpu_list_node *node,
 515                         int *_cpu)
 516 {
 517         int cpu;
 518
 519         for (;;) {
 520                 intptr_t *targetptr, newval, expect;
 521                 int ret;
 522
 523                 cpu = rseq_cpu_start();
 524                 /* Load list->c[cpu].head with single-copy atomicity. */
 525                 expect = (intptr_t)RSEQ_READ_ONCE(list->c[cpu].head);
 526                 newval = (intptr_t)node;
 527                 targetptr = (intptr_t *)&list->c[cpu].head;
 528                 node->next = (struct percpu_list_node *)expect;
 529                 ret = rseq_cmpeqv_storev(targetptr, expect, newval, cpu);
 530                 if (rseq_likely(!ret))
 531                         break;
 532                 /* Retry if comparison fails or rseq aborts. */
 533         }
 534         if (_cpu)
 535                 *_cpu = cpu;
 536 }
 537
 538 /*
 539  * Unlike a traditional lock-less linked list; the availability of a
 540  * rseq primitive allows us to implement pop without concerns over
 541  * ABA-type races.
 542  */
 543 struct percpu_list_node *this_cpu_list_pop(struct percpu_list *list,
 544                                            int *_cpu)
 545 {
 546         struct percpu_list_node *node = NULL;
 547         int cpu;
 548
 549         for (;;) {
 550                 struct percpu_list_node *head;
 551                 intptr_t *targetptr, expectnot, *load;
 552                 off_t offset;
 553                 int ret;
 554
 555                 cpu = rseq_cpu_start();
 556                 targetptr = (intptr_t *)&list->c[cpu].head;
 557                 expectnot = (intptr_t)NULL;
 558                 offset = offsetof(struct percpu_list_node, next);
 559                 load = (intptr_t *)&head;
 560                 ret = rseq_cmpnev_storeoffp_load(targetptr, expectnot,
 561                                                    offset, load, cpu);
 562                 if (rseq_likely(!ret)) {
 563                         node = head;
 564                         break;
 565                 }
 566                 if (ret > 0)
 567                         break;
 568                 /* Retry if rseq aborts. */
 569         }
 570         if (_cpu)
 571                 *_cpu = cpu;
 572         return node;
 573 }
 574
 575 /*
 576  * __percpu_list_pop is not safe against concurrent accesses. Should
 577  * only be used on lists that are not concurrently modified.
 578  */
 579 struct percpu_list_node *__percpu_list_pop(struct percpu_list *list, int cpu)
 580 {
 581         struct percpu_list_node *node;
 582
 583         node = list->c[cpu].head;
 584         if (!node)
 585                 return NULL;
 586         list->c[cpu].head = node->next;
 587         return node;
 588 }
 589
 590 void *test_percpu_list_thread(void *arg)
 591 {
 592         long long i, reps;
 593         struct percpu_list *list = (struct percpu_list *)arg;
 594
 595         if (!opt_disable_rseq && rseq_register_current_thread())
 596                 abort();
 597
 598         reps = opt_reps;
 599         for (i = 0; i < reps; i++) {
 600                 struct percpu_list_node *node;
 601
 602                 node = this_cpu_list_pop(list, NULL);
 603                 if (opt_yield)
 604                         sched_yield();  /* encourage shuffling */
 605                 if (node)
 606                         this_cpu_list_push(list, node, NULL);
 607         }
 608
 609         printf_verbose("tid %d: number of rseq abort: %d, signals delivered: %u\n",
 610                        (int) rseq_gettid(), nr_abort, signals_delivered);
 611         if (!opt_disable_rseq && rseq_unregister_current_thread())
 612                 abort();
 613
 614         return NULL;
 615 }
 616
 617 /* Simultaneous modification to a per-cpu linked list from many threads.  */
 618 void test_percpu_list(void)
 619 {
 620         const int num_threads = opt_threads;
 621         int i, j, ret;
 622         uint64_t sum = 0, expected_sum = 0;
 623         struct percpu_list list;
 624         pthread_t test_threads[num_threads];
 625         cpu_set_t allowed_cpus;
 626
 627         memset(&list, 0, sizeof(list));
 628
 629         /* Generate list entries for every usable cpu. */
 630         sched_getaffinity(0, sizeof(allowed_cpus), &allowed_cpus);
 631         for (i = 0; i < CPU_SETSIZE; i++) {
 632                 if (!CPU_ISSET(i, &allowed_cpus))
 633                         continue;
 634                 for (j = 1; j <= 100; j++) {
 635                         struct percpu_list_node *node;
 636
 637                         expected_sum += j;
 638
 639                         node = malloc(sizeof(*node));
 640                         assert(node);
 641                         node->data = j;
 642                         node->next = list.c[i].head;
 643                         list.c[i].head = node;
 644                 }
 645         }
 646
 647         for (i = 0; i < num_threads; i++) {
 648                 ret = pthread_create(&test_threads[i], NULL,
 649                                      test_percpu_list_thread, &list);
 650                 if (ret) {
 651                         errno = ret;
 652                         perror("pthread_create");
 653                         abort();
 654                 }
 655         }
 656
 657         for (i = 0; i < num_threads; i++) {
 658                 ret = pthread_join(test_threads[i], NULL);
 659                 if (ret) {
 660                         errno = ret;
 661                         perror("pthread_join");
 662                         abort();
 663                 }
 664         }
 665
 666         for (i = 0; i < CPU_SETSIZE; i++) {
 667                 struct percpu_list_node *node;
 668
 669                 if (!CPU_ISSET(i, &allowed_cpus))
 670                         continue;
 671
 672                 while ((node = __percpu_list_pop(&list, i))) {
 673                         sum += node->data;
 674                         free(node);
 675                 }
 676         }
 677
 678         /*
 679          * All entries should now be accounted for (unless some external
 680          * actor is interfering with our allowed affinity while this
 681          * test is running).
 682          */
 683         assert(sum == expected_sum);
 684 }
 685
 686 bool this_cpu_buffer_push(struct percpu_buffer *buffer,
 687                           struct percpu_buffer_node *node,
 688                           int *_cpu)
 689 {
 690         bool result = false;
 691         int cpu;
 692
 693         for (;;) {
 694                 intptr_t *targetptr_spec, newval_spec;
 695                 intptr_t *targetptr_final, newval_final;
 696                 intptr_t offset;
 697                 int ret;
 698
 699                 cpu = rseq_cpu_start();
 700                 offset = RSEQ_READ_ONCE(buffer->c[cpu].offset);
 701                 if (offset == buffer->c[cpu].buflen)
 702                         break;
 703                 newval_spec = (intptr_t)node;
 704                 targetptr_spec = (intptr_t *)&buffer->c[cpu].array[offset];
 705                 newval_final = offset + 1;
 706                 targetptr_final = &buffer->c[cpu].offset;
 707                 if (opt_mb)
 708                         ret = rseq_cmpeqv_trystorev_storev_release(
 709                                 targetptr_final, offset, targetptr_spec,
 710                                 newval_spec, newval_final, cpu);
 711                 else
 712                         ret = rseq_cmpeqv_trystorev_storev(targetptr_final,
 713                                 offset, targetptr_spec, newval_spec,
 714                                 newval_final, cpu);
 715                 if (rseq_likely(!ret)) {
 716                         result = true;
 717                         break;
 718                 }
 719                 /* Retry if comparison fails or rseq aborts. */
 720         }
 721         if (_cpu)
 722                 *_cpu = cpu;
 723         return result;
 724 }
 725
 726 struct percpu_buffer_node *this_cpu_buffer_pop(struct percpu_buffer *buffer,
 727                                                int *_cpu)
 728 {
 729         struct percpu_buffer_node *head;
 730         int cpu;
 731
 732         for (;;) {
 733                 intptr_t *targetptr, newval;
 734                 intptr_t offset;
 735                 int ret;
 736
 737                 cpu = rseq_cpu_start();
 738                 /* Load offset with single-copy atomicity. */
 739                 offset = RSEQ_READ_ONCE(buffer->c[cpu].offset);
 740                 if (offset == 0) {
 741                         head = NULL;
 742                         break;
 743                 }
 744                 head = RSEQ_READ_ONCE(buffer->c[cpu].array[offset - 1]);
 745                 newval = offset - 1;
 746                 targetptr = (intptr_t *)&buffer->c[cpu].offset;
 747                 ret = rseq_cmpeqv_cmpeqv_storev(targetptr, offset,
 748                         (intptr_t *)&buffer->c[cpu].array[offset - 1],
 749                         (intptr_t)head, newval, cpu);
 750                 if (rseq_likely(!ret))
 751                         break;
 752                 /* Retry if comparison fails or rseq aborts. */
 753         }
 754         if (_cpu)
 755                 *_cpu = cpu;
 756         return head;
 757 }
 758
 759 /*
 760  * __percpu_buffer_pop is not safe against concurrent accesses. Should
 761  * only be used on buffers that are not concurrently modified.
 762  */
 763 struct percpu_buffer_node *__percpu_buffer_pop(struct percpu_buffer *buffer,
 764                                                int cpu)
 765 {
 766         struct percpu_buffer_node *head;
 767         intptr_t offset;
 768
 769         offset = buffer->c[cpu].offset;
 770         if (offset == 0)
 771                 return NULL;
 772         head = buffer->c[cpu].array[offset - 1];
 773         buffer->c[cpu].offset = offset - 1;
 774         return head;
 775 }
 776
 777 void *test_percpu_buffer_thread(void *arg)
 778 {
 779         long long i, reps;
 780         struct percpu_buffer *buffer = (struct percpu_buffer *)arg;
 781
 782         if (!opt_disable_rseq && rseq_register_current_thread())
 783                 abort();
 784
 785         reps = opt_reps;
 786         for (i = 0; i < reps; i++) {
 787                 struct percpu_buffer_node *node;
 788
 789                 node = this_cpu_buffer_pop(buffer, NULL);
 790                 if (opt_yield)
 791                         sched_yield();  /* encourage shuffling */
 792                 if (node) {
 793                         if (!this_cpu_buffer_push(buffer, node, NULL)) {
 794                                 /* Should increase buffer size. */
 795                                 abort();
 796                         }
 797                 }
 798         }
 799
 800         printf_verbose("tid %d: number of rseq abort: %d, signals delivered: %u\n",
 801                        (int) rseq_gettid(), nr_abort, signals_delivered);
 802         if (!opt_disable_rseq && rseq_unregister_current_thread())
 803                 abort();
 804
 805         return NULL;
 806 }
 807
 808 /* Simultaneous modification to a per-cpu buffer from many threads.  */
 809 void test_percpu_buffer(void)
 810 {
 811         const int num_threads = opt_threads;
 812         int i, j, ret;
 813         uint64_t sum = 0, expected_sum = 0;
 814         struct percpu_buffer buffer;
 815         pthread_t test_threads[num_threads];
 816         cpu_set_t allowed_cpus;
 817
 818         memset(&buffer, 0, sizeof(buffer));
 819
 820         /* Generate list entries for every usable cpu. */
 821         sched_getaffinity(0, sizeof(allowed_cpus), &allowed_cpus);
 822         for (i = 0; i < CPU_SETSIZE; i++) {
 823                 if (!CPU_ISSET(i, &allowed_cpus))
 824                         continue;
 825                 /* Worse-case is every item in same CPU. */
 826                 buffer.c[i].array =
 827                         malloc(sizeof(*buffer.c[i].array) * CPU_SETSIZE *
 828                                BUFFER_ITEM_PER_CPU);
 829                 assert(buffer.c[i].array);
 830                 buffer.c[i].buflen = CPU_SETSIZE * BUFFER_ITEM_PER_CPU;
 831                 for (j = 1; j <= BUFFER_ITEM_PER_CPU; j++) {
 832                         struct percpu_buffer_node *node;
 833
 834                         expected_sum += j;
 835
 836                         /*
 837                          * We could theoretically put the word-sized
 838                          * "data" directly in the buffer. However, we
 839                          * want to model objects that would not fit
 840                          * within a single word, so allocate an object
 841                          * for each node.
 842                          */
 843                         node = malloc(sizeof(*node));
 844                         assert(node);
 845                         node->data = j;
 846                         buffer.c[i].array[j - 1] = node;
 847                         buffer.c[i].offset++;
 848                 }
 849         }
 850
 851         for (i = 0; i < num_threads; i++) {
 852                 ret = pthread_create(&test_threads[i], NULL,
 853                                      test_percpu_buffer_thread, &buffer);
 854                 if (ret) {
 855                         errno = ret;
 856                         perror("pthread_create");
 857                         abort();
 858                 }
 859         }
 860
 861         for (i = 0; i < num_threads; i++) {
 862                 ret = pthread_join(test_threads[i], NULL);
 863                 if (ret) {
 864                         errno = ret;
 865                         perror("pthread_join");
 866                         abort();
 867                 }
 868         }
 869
 870         for (i = 0; i < CPU_SETSIZE; i++) {
 871                 struct percpu_buffer_node *node;
 872
 873                 if (!CPU_ISSET(i, &allowed_cpus))
 874                         continue;
 875
 876                 while ((node = __percpu_buffer_pop(&buffer, i))) {
 877                         sum += node->data;
 878                         free(node);
 879                 }
 880                 free(buffer.c[i].array);
 881         }
 882
 883         /*
 884          * All entries should now be accounted for (unless some external
 885          * actor is interfering with our allowed affinity while this
 886          * test is running).
 887          */
 888         assert(sum == expected_sum);
 889 }
 890
 891 bool this_cpu_memcpy_buffer_push(struct percpu_memcpy_buffer *buffer,
 892                                  struct percpu_memcpy_buffer_node item,
 893                                  int *_cpu)
 894 {
 895         bool result = false;
 896         int cpu;
 897
 898         for (;;) {
 899                 intptr_t *targetptr_final, newval_final, offset;
 900                 char *destptr, *srcptr;
 901                 size_t copylen;
 902                 int ret;
 903
 904                 cpu = rseq_cpu_start();
 905                 /* Load offset with single-copy atomicity. */
 906                 offset = RSEQ_READ_ONCE(buffer->c[cpu].offset);
 907                 if (offset == buffer->c[cpu].buflen)
 908                         break;
 909                 destptr = (char *)&buffer->c[cpu].array[offset];
 910                 srcptr = (char *)&item;
 911                 /* copylen must be <= 4kB. */
 912                 copylen = sizeof(item);
 913                 newval_final = offset + 1;
 914                 targetptr_final = &buffer->c[cpu].offset;
 915                 if (opt_mb)
 916                         ret = rseq_cmpeqv_trymemcpy_storev_release(
 917                                 targetptr_final, offset,
 918                                 destptr, srcptr, copylen,
 919                                 newval_final, cpu);
 920                 else
 921                         ret = rseq_cmpeqv_trymemcpy_storev(targetptr_final,
 922                                 offset, destptr, srcptr, copylen,
 923                                 newval_final, cpu);
 924                 if (rseq_likely(!ret)) {
 925                         result = true;
 926                         break;
 927                 }
 928                 /* Retry if comparison fails or rseq aborts. */
 929         }
 930         if (_cpu)
 931                 *_cpu = cpu;
 932         return result;
 933 }
 934
 935 bool this_cpu_memcpy_buffer_pop(struct percpu_memcpy_buffer *buffer,
 936                                 struct percpu_memcpy_buffer_node *item,
 937                                 int *_cpu)
 938 {
 939         bool result = false;
 940         int cpu;
 941
 942         for (;;) {
 943                 intptr_t *targetptr_final, newval_final, offset;
 944                 char *destptr, *srcptr;
 945                 size_t copylen;
 946                 int ret;
 947
 948                 cpu = rseq_cpu_start();
 949                 /* Load offset with single-copy atomicity. */
 950                 offset = RSEQ_READ_ONCE(buffer->c[cpu].offset);
 951                 if (offset == 0)
 952                         break;
 953                 destptr = (char *)item;
 954                 srcptr = (char *)&buffer->c[cpu].array[offset - 1];
 955                 /* copylen must be <= 4kB. */
 956                 copylen = sizeof(*item);
 957                 newval_final = offset - 1;
 958                 targetptr_final = &buffer->c[cpu].offset;
 959                 ret = rseq_cmpeqv_trymemcpy_storev(targetptr_final,
 960                         offset, destptr, srcptr, copylen,
 961                         newval_final, cpu);
 962                 if (rseq_likely(!ret)) {
 963                         result = true;
 964                         break;
 965                 }
 966                 /* Retry if comparison fails or rseq aborts. */
 967         }
 968         if (_cpu)
 969                 *_cpu = cpu;
 970         return result;
 971 }
 972
 973 /*
 974  * __percpu_memcpy_buffer_pop is not safe against concurrent accesses. Should
 975  * only be used on buffers that are not concurrently modified.
 976  */
 977 bool __percpu_memcpy_buffer_pop(struct percpu_memcpy_buffer *buffer,
 978                                 struct percpu_memcpy_buffer_node *item,
 979                                 int cpu)
 980 {
 981         intptr_t offset;
 982
 983         offset = buffer->c[cpu].offset;
 984         if (offset == 0)
 985                 return false;
 986         memcpy(item, &buffer->c[cpu].array[offset - 1], sizeof(*item));
 987         buffer->c[cpu].offset = offset - 1;
 988         return true;
 989 }
 990
 991 void *test_percpu_memcpy_buffer_thread(void *arg)
 992 {
 993         long long i, reps;
 994         struct percpu_memcpy_buffer *buffer = (struct percpu_memcpy_buffer *)arg;
 995
 996         if (!opt_disable_rseq && rseq_register_current_thread())
 997                 abort();
 998
 999         reps = opt_reps;
1000         for (i = 0; i < reps; i++) {
1001                 struct percpu_memcpy_buffer_node item;
1002                 bool result;
1003
1004                 result = this_cpu_memcpy_buffer_pop(buffer, &item, NULL);
1005                 if (opt_yield)
1006                         sched_yield();  /* encourage shuffling */
1007                 if (result) {
1008                         if (!this_cpu_memcpy_buffer_push(buffer, item, NULL)) {
1009                                 /* Should increase buffer size. */
1010                                 abort();
1011                         }
1012                 }
1013         }
1014
1015         printf_verbose("tid %d: number of rseq abort: %d, signals delivered: %u\n",
1016                        (int) rseq_gettid(), nr_abort, signals_delivered);
1017         if (!opt_disable_rseq && rseq_unregister_current_thread())
1018                 abort();
1019
1020         return NULL;
1021 }
1022
1023 /* Simultaneous modification to a per-cpu buffer from many threads.  */
1024 void test_percpu_memcpy_buffer(void)
1025 {
1026         const int num_threads = opt_threads;
1027         int i, j, ret;
1028         uint64_t sum = 0, expected_sum = 0;
1029         struct percpu_memcpy_buffer buffer;
1030         pthread_t test_threads[num_threads];
1031         cpu_set_t allowed_cpus;
1032
1033         memset(&buffer, 0, sizeof(buffer));
1034
1035         /* Generate list entries for every usable cpu. */
1036         sched_getaffinity(0, sizeof(allowed_cpus), &allowed_cpus);
1037         for (i = 0; i < CPU_SETSIZE; i++) {
1038                 if (!CPU_ISSET(i, &allowed_cpus))
1039                         continue;
1040                 /* Worse-case is every item in same CPU. */
1041                 buffer.c[i].array =
1042                         malloc(sizeof(*buffer.c[i].array) * CPU_SETSIZE *
1043                                MEMCPY_BUFFER_ITEM_PER_CPU);
1044                 assert(buffer.c[i].array);
1045                 buffer.c[i].buflen = CPU_SETSIZE * MEMCPY_BUFFER_ITEM_PER_CPU;
1046                 for (j = 1; j <= MEMCPY_BUFFER_ITEM_PER_CPU; j++) {
1047                         expected_sum += 2 * j + 1;
1048
1049                         /*
1050                          * We could theoretically put the word-sized
1051                          * "data" directly in the buffer. However, we
1052                          * want to model objects that would not fit
1053                          * within a single word, so allocate an object
1054                          * for each node.
1055                          */
1056                         buffer.c[i].array[j - 1].data1 = j;
1057                         buffer.c[i].array[j - 1].data2 = j + 1;
1058                         buffer.c[i].offset++;
1059                 }
1060         }
1061
1062         for (i = 0; i < num_threads; i++) {
1063                 ret = pthread_create(&test_threads[i], NULL,
1064                                      test_percpu_memcpy_buffer_thread,
1065                                      &buffer);
1066                 if (ret) {
1067                         errno = ret;
1068                         perror("pthread_create");
1069                         abort();
1070                 }
1071         }
1072
1073         for (i = 0; i < num_threads; i++) {
1074                 ret = pthread_join(test_threads[i], NULL);
1075                 if (ret) {
1076                         errno = ret;
1077                         perror("pthread_join");
1078                         abort();
1079                 }
1080         }
1081
1082         for (i = 0; i < CPU_SETSIZE; i++) {
1083                 struct percpu_memcpy_buffer_node item;
1084
1085                 if (!CPU_ISSET(i, &allowed_cpus))
1086                         continue;
1087
1088                 while (__percpu_memcpy_buffer_pop(&buffer, &item, i)) {
1089                         sum += item.data1;
1090                         sum += item.data2;
1091                 }
1092                 free(buffer.c[i].array);
1093         }
1094
1095         /*
1096          * All entries should now be accounted for (unless some external
1097          * actor is interfering with our allowed affinity while this
1098          * test is running).
1099          */
1100         assert(sum == expected_sum);
1101 }
1102
1103 static void test_signal_interrupt_handler(int signo)
1104 {
1105         signals_delivered++;
1106 }
1107
1108 static int set_signal_handler(void)
1109 {
1110         int ret = 0;
1111         struct sigaction sa;
1112         sigset_t sigset;
1113
1114         ret = sigemptyset(&sigset);
1115         if (ret < 0) {
1116                 perror("sigemptyset");
1117                 return ret;
1118         }
1119
1120         sa.sa_handler = test_signal_interrupt_handler;
1121         sa.sa_mask = sigset;
1122         sa.sa_flags = 0;
1123         ret = sigaction(SIGUSR1, &sa, NULL);
1124         if (ret < 0) {
1125                 perror("sigaction");
1126                 return ret;
1127         }
1128
1129         printf_verbose("Signal handler set for SIGUSR1\n");
1130
1131         return ret;
1132 }
1133
1134 static void show_usage(int argc, char **argv)
1135 {
1136         printf("Usage : %s <OPTIONS>\n",
1137                 argv[0]);
1138         printf("OPTIONS:\n");
1139         printf("        [-1 loops] Number of loops for delay injection 1\n");
1140         printf("        [-2 loops] Number of loops for delay injection 2\n");
1141         printf("        [-3 loops] Number of loops for delay injection 3\n");
1142         printf("        [-4 loops] Number of loops for delay injection 4\n");
1143         printf("        [-5 loops] Number of loops for delay injection 5\n");
1144         printf("        [-6 loops] Number of loops for delay injection 6\n");
1145         printf("        [-7 loops] Number of loops for delay injection 7 (-1 to enable -m)\n");
1146         printf("        [-8 loops] Number of loops for delay injection 8 (-1 to enable -m)\n");
1147         printf("        [-9 loops] Number of loops for delay injection 9 (-1 to enable -m)\n");
1148         printf("        [-m N] Yield/sleep/kill every modulo N (default 0: disabled) (>= 0)\n");
1149         printf("        [-y] Yield\n");
1150         printf("        [-k] Kill thread with signal\n");
1151         printf("        [-s S] S: =0: disabled (default), >0: sleep time (ms)\n");
1152         printf("        [-t N] Number of threads (default 200)\n");
1153         printf("        [-r N] Number of repetitions per thread (default 5000)\n");
1154         printf("        [-d] Disable rseq system call (no initialization)\n");
1155         printf("        [-D M] Disable rseq for each M threads\n");
1156         printf("        [-T test] Choose test: (s)pinlock, (l)ist, (b)uffer, (m)emcpy, (i)ncrement\n");
1157         printf("        [-M] Push into buffer and memcpy buffer with memory barriers.\n");
1158         printf("        [-v] Verbose output.\n");
1159         printf("        [-h] Show this help.\n");
1160         printf("\n");
1161 }
1162
1163 int main(int argc, char **argv)
1164 {
1165         int i;
1166
1167         for (i = 1; i < argc; i++) {
1168                 if (argv[i][0] != '-')
1169                         continue;
1170                 switch (argv[i][1]) {
1171                 case '1':
1172                 case '2':
1173                 case '3':
1174                 case '4':
1175                 case '5':
1176                 case '6':
1177                 case '7':
1178                 case '8':
1179                 case '9':
1180                         if (argc < i + 2) {
1181                                 show_usage(argc, argv);
1182                                 goto error;
1183                         }
1184                         loop_cnt[argv[i][1] - '0'] = atol(argv[i + 1]);
1185                         i++;
1186                         break;
1187                 case 'm':
1188                         if (argc < i + 2) {
1189                                 show_usage(argc, argv);
1190                                 goto error;
1191                         }
1192                         opt_modulo = atol(argv[i + 1]);
1193                         if (opt_modulo < 0) {
1194                                 show_usage(argc, argv);
1195                                 goto error;
1196                         }
1197                         i++;
1198                         break;
1199                 case 's':
1200                         if (argc < i + 2) {
1201                                 show_usage(argc, argv);
1202                                 goto error;
1203                         }
1204                         opt_sleep = atol(argv[i + 1]);
1205                         if (opt_sleep < 0) {
1206                                 show_usage(argc, argv);
1207                                 goto error;
1208                         }
1209                         i++;
1210                         break;
1211                 case 'y':
1212                         opt_yield = 1;
1213                         break;
1214                 case 'k':
1215                         opt_signal = 1;
1216                         break;
1217                 case 'd':
1218                         opt_disable_rseq = 1;
1219                         break;
1220                 case 'D':
1221                         if (argc < i + 2) {
1222                                 show_usage(argc, argv);
1223                                 goto error;
1224                         }
1225                         opt_disable_mod = atol(argv[i + 1]);
1226                         if (opt_disable_mod < 0) {
1227                                 show_usage(argc, argv);
1228                                 goto error;
1229                         }
1230                         i++;
1231                         break;
1232                 case 't':
1233                         if (argc < i + 2) {
1234                                 show_usage(argc, argv);
1235                                 goto error;
1236                         }
1237                         opt_threads = atol(argv[i + 1]);
1238                         if (opt_threads < 0) {
1239                                 show_usage(argc, argv);
1240                                 goto error;
1241                         }
1242                         i++;
1243                         break;
1244                 case 'r':
1245                         if (argc < i + 2) {
1246                                 show_usage(argc, argv);
1247                                 goto error;
1248                         }
1249                         opt_reps = atoll(argv[i + 1]);
1250                         if (opt_reps < 0) {
1251                                 show_usage(argc, argv);
1252                                 goto error;
1253                         }
1254                         i++;
1255                         break;
1256                 case 'h':
1257                         show_usage(argc, argv);
1258                         goto end;
1259                 case 'T':
1260                         if (argc < i + 2) {
1261                                 show_usage(argc, argv);
1262                                 goto error;
1263                         }
1264                         opt_test = *argv[i + 1];
1265                         switch (opt_test) {
1266                         case 's':
1267                         case 'l':
1268                         case 'i':
1269                         case 'b':
1270                         case 'm':
1271                                 break;
1272                         default:
1273                                 show_usage(argc, argv);
1274                                 goto error;
1275                         }
1276                         i++;
1277                         break;
1278                 case 'v':
1279                         verbose = 1;
1280                         break;
1281                 case 'M':
1282                         opt_mb = 1;
1283                         break;
1284                 default:
1285                         show_usage(argc, argv);
1286                         goto error;
1287                 }
1288         }
1289
1290         loop_cnt_1 = loop_cnt[1];
1291         loop_cnt_2 = loop_cnt[2];
1292         loop_cnt_3 = loop_cnt[3];
1293         loop_cnt_4 = loop_cnt[4];
1294         loop_cnt_5 = loop_cnt[5];
1295         loop_cnt_6 = loop_cnt[6];
1296
1297         if (set_signal_handler())
1298                 goto error;
1299
1300         if (!opt_disable_rseq && rseq_register_current_thread())
1301                 goto error;
1302         switch (opt_test) {
1303         case 's':
1304                 printf_verbose("spinlock\n");
1305                 test_percpu_spinlock();
1306                 break;
1307         case 'l':
1308                 printf_verbose("linked list\n");
1309                 test_percpu_list();
1310                 break;
1311         case 'b':
1312                 printf_verbose("buffer\n");
1313                 test_percpu_buffer();
1314                 break;
1315         case 'm':
1316                 printf_verbose("memcpy buffer\n");
1317                 test_percpu_memcpy_buffer();
1318                 break;
1319         case 'i':
1320                 printf_verbose("counter increment\n");
1321                 test_percpu_inc();
1322                 break;
1323         }
1324         if (!opt_disable_rseq && rseq_unregister_current_thread())
1325                 abort();
1326 end:
1327         return 0;
1328
1329 error:
1330         return -1;
1331 }