tools/testing/selftests/rseq/rseq-x86.h

   1 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */
   2 /*
   3  * rseq-x86.h
   4  *
   5  * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
   6  */
   7
   8 #include <stdint.h>
   9
  10 #define RSEQ_SIG        0x53053053
  11
  12 #ifdef __x86_64__
  13
  14 #define rseq_smp_mb()   \
  15         __asm__ __volatile__ ("lock; addl $0,-128(%%rsp)" ::: "memory", "cc")
  16 #define rseq_smp_rmb()  rseq_barrier()
  17 #define rseq_smp_wmb()  rseq_barrier()
  18
  19 #define rseq_smp_load_acquire(p)                                        \
  20 __extension__ ({                                                        \
  21         __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p);                       \
  22         rseq_barrier();                                                 \
  23         ____p1;                                                         \
  24 })
  25
  26 #define rseq_smp_acquire__after_ctrl_dep()      rseq_smp_rmb()
  27
  28 #define rseq_smp_store_release(p, v)                                    \
  29 do {                                                                    \
  30         rseq_barrier();                                                 \
  31         RSEQ_WRITE_ONCE(*p, v);                                         \
  32 } while (0)
  33
  34 #ifdef RSEQ_SKIP_FASTPATH
  35 #include "rseq-skip.h"
  36 #else /* !RSEQ_SKIP_FASTPATH */
  37
  38 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags,                  \
  39                                 start_ip, post_commit_offset, abort_ip) \
  40                 ".pushsection __rseq_table, \"aw\"\n\t"                 \
  41                 ".balign 32\n\t"                                        \
  42                 __rseq_str(label) ":\n\t"                               \
  43                 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
  44                 ".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \
  45                 ".popsection\n\t"
  46
  47 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
  48         __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip,              \
  49                                 (post_commit_ip - start_ip), abort_ip)
  50
  51 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs)                \
  52                 RSEQ_INJECT_ASM(1)                                      \
  53                 "leaq " __rseq_str(cs_label) "(%%rip), %%rax\n\t"       \
  54                 "movq %%rax, %[" __rseq_str(rseq_cs) "]\n\t"            \
  55                 __rseq_str(label) ":\n\t"
  56
  57 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label)              \
  58                 RSEQ_INJECT_ASM(2)                                      \
  59                 "cmpl %[" __rseq_str(cpu_id) "], %[" __rseq_str(current_cpu_id) "]\n\t" \
  60                 "jnz " __rseq_str(label) "\n\t"
  61
  62 #define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label)             \
  63                 ".pushsection __rseq_failure, \"ax\"\n\t"               \
  64                 /* Disassembler-friendly signature: nopl <sig>(%rip). */\
  65                 ".byte 0x0f, 0x1f, 0x05\n\t"                            \
  66                 ".long " __rseq_str(RSEQ_SIG) "\n\t"                    \
  67                 __rseq_str(label) ":\n\t"                               \
  68                 teardown                                                \
  69                 "jmp %l[" __rseq_str(abort_label) "]\n\t"               \
  70                 ".popsection\n\t"
  71
  72 #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label)         \
  73                 ".pushsection __rseq_failure, \"ax\"\n\t"               \
  74                 __rseq_str(label) ":\n\t"                               \
  75                 teardown                                                \
  76                 "jmp %l[" __rseq_str(cmpfail_label) "]\n\t"             \
  77                 ".popsection\n\t"
  78
  79 static inline __attribute__((always_inline))
  80 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
  81 {
  82         RSEQ_INJECT_C(9)
  83
  84         __asm__ __volatile__ goto (
  85                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
  86                 /* Start rseq by storing table entry pointer into rseq_cs. */
  87                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
  88                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
  89                 RSEQ_INJECT_ASM(3)
  90                 "cmpq %[v], %[expect]\n\t"
  91                 "jnz %l[cmpfail]\n\t"
  92                 RSEQ_INJECT_ASM(4)
  93 #ifdef RSEQ_COMPARE_TWICE
  94                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
  95                 "cmpq %[v], %[expect]\n\t"
  96                 "jnz %l[error2]\n\t"
  97 #endif
  98                 /* final store */
  99                 "movq %[newv], %[v]\n\t"
 100                 "2:\n\t"
 101                 RSEQ_INJECT_ASM(5)
 102                 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 103                 : /* gcc asm goto does not allow outputs */
 104                 : [cpu_id]              "r" (cpu),
 105                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
 106                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 107                   [v]                   "m" (*v),
 108                   [expect]              "r" (expect),
 109                   [newv]                "r" (newv)
 110                 : "memory", "cc", "rax"
 111                   RSEQ_INJECT_CLOBBER
 112                 : abort, cmpfail
 113 #ifdef RSEQ_COMPARE_TWICE
 114                   , error1, error2
 115 #endif
 116         );
 117         return 0;
 118 abort:
 119         RSEQ_INJECT_FAILED
 120         return -1;
 121 cmpfail:
 122         return 1;
 123 #ifdef RSEQ_COMPARE_TWICE
 124 error1:
 125         rseq_bug("cpu_id comparison failed");
 126 error2:
 127         rseq_bug("expected value comparison failed");
 128 #endif
 129 }
 130
 131 /*
 132  * Compare @v against @expectnot. When it does _not_ match, load @v
 133  * into @load, and store the content of *@v + voffp into @v.
 134  */
 135 static inline __attribute__((always_inline))
 136 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
 137                                off_t voffp, intptr_t *load, int cpu)
 138 {
 139         RSEQ_INJECT_C(9)
 140
 141         __asm__ __volatile__ goto (
 142                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 143                 /* Start rseq by storing table entry pointer into rseq_cs. */
 144                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 145                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 146                 RSEQ_INJECT_ASM(3)
 147                 "movq %[v], %%rbx\n\t"
 148                 "cmpq %%rbx, %[expectnot]\n\t"
 149                 "je %l[cmpfail]\n\t"
 150                 RSEQ_INJECT_ASM(4)
 151 #ifdef RSEQ_COMPARE_TWICE
 152                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 153                 "movq %[v], %%rbx\n\t"
 154                 "cmpq %%rbx, %[expectnot]\n\t"
 155                 "je %l[error2]\n\t"
 156 #endif
 157                 "movq %%rbx, %[load]\n\t"
 158                 "addq %[voffp], %%rbx\n\t"
 159                 "movq (%%rbx), %%rbx\n\t"
 160                 /* final store */
 161                 "movq %%rbx, %[v]\n\t"
 162                 "2:\n\t"
 163                 RSEQ_INJECT_ASM(5)
 164                 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 165                 : /* gcc asm goto does not allow outputs */
 166                 : [cpu_id]              "r" (cpu),
 167                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
 168                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 169                   /* final store input */
 170                   [v]                   "m" (*v),
 171                   [expectnot]           "r" (expectnot),
 172                   [voffp]               "er" (voffp),
 173                   [load]                "m" (*load)
 174                 : "memory", "cc", "rax", "rbx"
 175                   RSEQ_INJECT_CLOBBER
 176                 : abort, cmpfail
 177 #ifdef RSEQ_COMPARE_TWICE
 178                   , error1, error2
 179 #endif
 180         );
 181         return 0;
 182 abort:
 183         RSEQ_INJECT_FAILED
 184         return -1;
 185 cmpfail:
 186         return 1;
 187 #ifdef RSEQ_COMPARE_TWICE
 188 error1:
 189         rseq_bug("cpu_id comparison failed");
 190 error2:
 191         rseq_bug("expected value comparison failed");
 192 #endif
 193 }
 194
 195 static inline __attribute__((always_inline))
 196 int rseq_addv(intptr_t *v, intptr_t count, int cpu)
 197 {
 198         RSEQ_INJECT_C(9)
 199
 200         __asm__ __volatile__ goto (
 201                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 202                 /* Start rseq by storing table entry pointer into rseq_cs. */
 203                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 204                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 205                 RSEQ_INJECT_ASM(3)
 206 #ifdef RSEQ_COMPARE_TWICE
 207                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 208 #endif
 209                 /* final store */
 210                 "addq %[count], %[v]\n\t"
 211                 "2:\n\t"
 212                 RSEQ_INJECT_ASM(4)
 213                 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 214                 : /* gcc asm goto does not allow outputs */
 215                 : [cpu_id]              "r" (cpu),
 216                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
 217                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 218                   /* final store input */
 219                   [v]                   "m" (*v),
 220                   [count]               "er" (count)
 221                 : "memory", "cc", "rax"
 222                   RSEQ_INJECT_CLOBBER
 223                 : abort
 224 #ifdef RSEQ_COMPARE_TWICE
 225                   , error1
 226 #endif
 227         );
 228         return 0;
 229 abort:
 230         RSEQ_INJECT_FAILED
 231         return -1;
 232 #ifdef RSEQ_COMPARE_TWICE
 233 error1:
 234         rseq_bug("cpu_id comparison failed");
 235 #endif
 236 }
 237
 238 static inline __attribute__((always_inline))
 239 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
 240                                  intptr_t *v2, intptr_t newv2,
 241                                  intptr_t newv, int cpu)
 242 {
 243         RSEQ_INJECT_C(9)
 244
 245         __asm__ __volatile__ goto (
 246                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 247                 /* Start rseq by storing table entry pointer into rseq_cs. */
 248                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 249                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 250                 RSEQ_INJECT_ASM(3)
 251                 "cmpq %[v], %[expect]\n\t"
 252                 "jnz %l[cmpfail]\n\t"
 253                 RSEQ_INJECT_ASM(4)
 254 #ifdef RSEQ_COMPARE_TWICE
 255                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 256                 "cmpq %[v], %[expect]\n\t"
 257                 "jnz %l[error2]\n\t"
 258 #endif
 259                 /* try store */
 260                 "movq %[newv2], %[v2]\n\t"
 261                 RSEQ_INJECT_ASM(5)
 262                 /* final store */
 263                 "movq %[newv], %[v]\n\t"
 264                 "2:\n\t"
 265                 RSEQ_INJECT_ASM(6)
 266                 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 267                 : /* gcc asm goto does not allow outputs */
 268                 : [cpu_id]              "r" (cpu),
 269                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
 270                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 271                   /* try store input */
 272                   [v2]                  "m" (*v2),
 273                   [newv2]               "r" (newv2),
 274                   /* final store input */
 275                   [v]                   "m" (*v),
 276                   [expect]              "r" (expect),
 277                   [newv]                "r" (newv)
 278                 : "memory", "cc", "rax"
 279                   RSEQ_INJECT_CLOBBER
 280                 : abort, cmpfail
 281 #ifdef RSEQ_COMPARE_TWICE
 282                   , error1, error2
 283 #endif
 284         );
 285         return 0;
 286 abort:
 287         RSEQ_INJECT_FAILED
 288         return -1;
 289 cmpfail:
 290         return 1;
 291 #ifdef RSEQ_COMPARE_TWICE
 292 error1:
 293         rseq_bug("cpu_id comparison failed");
 294 error2:
 295         rseq_bug("expected value comparison failed");
 296 #endif
 297 }
 298
 299 /* x86-64 is TSO. */
 300 static inline __attribute__((always_inline))
 301 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
 302                                          intptr_t *v2, intptr_t newv2,
 303                                          intptr_t newv, int cpu)
 304 {
 305         return rseq_cmpeqv_trystorev_storev(v, expect, v2, newv2, newv, cpu);
 306 }
 307
 308 static inline __attribute__((always_inline))
 309 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
 310                               intptr_t *v2, intptr_t expect2,
 311                               intptr_t newv, int cpu)
 312 {
 313         RSEQ_INJECT_C(9)
 314
 315         __asm__ __volatile__ goto (
 316                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 317                 /* Start rseq by storing table entry pointer into rseq_cs. */
 318                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 319                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 320                 RSEQ_INJECT_ASM(3)
 321                 "cmpq %[v], %[expect]\n\t"
 322                 "jnz %l[cmpfail]\n\t"
 323                 RSEQ_INJECT_ASM(4)
 324                 "cmpq %[v2], %[expect2]\n\t"
 325                 "jnz %l[cmpfail]\n\t"
 326                 RSEQ_INJECT_ASM(5)
 327 #ifdef RSEQ_COMPARE_TWICE
 328                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 329                 "cmpq %[v], %[expect]\n\t"
 330                 "jnz %l[error2]\n\t"
 331                 "cmpq %[v2], %[expect2]\n\t"
 332                 "jnz %l[error3]\n\t"
 333 #endif
 334                 /* final store */
 335                 "movq %[newv], %[v]\n\t"
 336                 "2:\n\t"
 337                 RSEQ_INJECT_ASM(6)
 338                 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 339                 : /* gcc asm goto does not allow outputs */
 340                 : [cpu_id]              "r" (cpu),
 341                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
 342                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 343                   /* cmp2 input */
 344                   [v2]                  "m" (*v2),
 345                   [expect2]             "r" (expect2),
 346                   /* final store input */
 347                   [v]                   "m" (*v),
 348                   [expect]              "r" (expect),
 349                   [newv]                "r" (newv)
 350                 : "memory", "cc", "rax"
 351                   RSEQ_INJECT_CLOBBER
 352                 : abort, cmpfail
 353 #ifdef RSEQ_COMPARE_TWICE
 354                   , error1, error2, error3
 355 #endif
 356         );
 357         return 0;
 358 abort:
 359         RSEQ_INJECT_FAILED
 360         return -1;
 361 cmpfail:
 362         return 1;
 363 #ifdef RSEQ_COMPARE_TWICE
 364 error1:
 365         rseq_bug("cpu_id comparison failed");
 366 error2:
 367         rseq_bug("1st expected value comparison failed");
 368 error3:
 369         rseq_bug("2nd expected value comparison failed");
 370 #endif
 371 }
 372
 373 static inline __attribute__((always_inline))
 374 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
 375                                  void *dst, void *src, size_t len,
 376                                  intptr_t newv, int cpu)
 377 {
 378         uint64_t rseq_scratch[3];
 379
 380         RSEQ_INJECT_C(9)
 381
 382         __asm__ __volatile__ goto (
 383                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 384                 "movq %[src], %[rseq_scratch0]\n\t"
 385                 "movq %[dst], %[rseq_scratch1]\n\t"
 386                 "movq %[len], %[rseq_scratch2]\n\t"
 387                 /* Start rseq by storing table entry pointer into rseq_cs. */
 388                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 389                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 390                 RSEQ_INJECT_ASM(3)
 391                 "cmpq %[v], %[expect]\n\t"
 392                 "jnz 5f\n\t"
 393                 RSEQ_INJECT_ASM(4)
 394 #ifdef RSEQ_COMPARE_TWICE
 395                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f)
 396                 "cmpq %[v], %[expect]\n\t"
 397                 "jnz 7f\n\t"
 398 #endif
 399                 /* try memcpy */
 400                 "test %[len], %[len]\n\t" \
 401                 "jz 333f\n\t" \
 402                 "222:\n\t" \
 403                 "movb (%[src]), %%al\n\t" \
 404                 "movb %%al, (%[dst])\n\t" \
 405                 "inc %[src]\n\t" \
 406                 "inc %[dst]\n\t" \
 407                 "dec %[len]\n\t" \
 408                 "jnz 222b\n\t" \
 409                 "333:\n\t" \
 410                 RSEQ_INJECT_ASM(5)
 411                 /* final store */
 412                 "movq %[newv], %[v]\n\t"
 413                 "2:\n\t"
 414                 RSEQ_INJECT_ASM(6)
 415                 /* teardown */
 416                 "movq %[rseq_scratch2], %[len]\n\t"
 417                 "movq %[rseq_scratch1], %[dst]\n\t"
 418                 "movq %[rseq_scratch0], %[src]\n\t"
 419                 RSEQ_ASM_DEFINE_ABORT(4,
 420                         "movq %[rseq_scratch2], %[len]\n\t"
 421                         "movq %[rseq_scratch1], %[dst]\n\t"
 422                         "movq %[rseq_scratch0], %[src]\n\t",
 423                         abort)
 424                 RSEQ_ASM_DEFINE_CMPFAIL(5,
 425                         "movq %[rseq_scratch2], %[len]\n\t"
 426                         "movq %[rseq_scratch1], %[dst]\n\t"
 427                         "movq %[rseq_scratch0], %[src]\n\t",
 428                         cmpfail)
 429 #ifdef RSEQ_COMPARE_TWICE
 430                 RSEQ_ASM_DEFINE_CMPFAIL(6,
 431                         "movq %[rseq_scratch2], %[len]\n\t"
 432                         "movq %[rseq_scratch1], %[dst]\n\t"
 433                         "movq %[rseq_scratch0], %[src]\n\t",
 434                         error1)
 435                 RSEQ_ASM_DEFINE_CMPFAIL(7,
 436                         "movq %[rseq_scratch2], %[len]\n\t"
 437                         "movq %[rseq_scratch1], %[dst]\n\t"
 438                         "movq %[rseq_scratch0], %[src]\n\t",
 439                         error2)
 440 #endif
 441                 : /* gcc asm goto does not allow outputs */
 442                 : [cpu_id]              "r" (cpu),
 443                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
 444                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 445                   /* final store input */
 446                   [v]                   "m" (*v),
 447                   [expect]              "r" (expect),
 448                   [newv]                "r" (newv),
 449                   /* try memcpy input */
 450                   [dst]                 "r" (dst),
 451                   [src]                 "r" (src),
 452                   [len]                 "r" (len),
 453                   [rseq_scratch0]       "m" (rseq_scratch[0]),
 454                   [rseq_scratch1]       "m" (rseq_scratch[1]),
 455                   [rseq_scratch2]       "m" (rseq_scratch[2])
 456                 : "memory", "cc", "rax"
 457                   RSEQ_INJECT_CLOBBER
 458                 : abort, cmpfail
 459 #ifdef RSEQ_COMPARE_TWICE
 460                   , error1, error2
 461 #endif
 462         );
 463         return 0;
 464 abort:
 465         RSEQ_INJECT_FAILED
 466         return -1;
 467 cmpfail:
 468         return 1;
 469 #ifdef RSEQ_COMPARE_TWICE
 470 error1:
 471         rseq_bug("cpu_id comparison failed");
 472 error2:
 473         rseq_bug("expected value comparison failed");
 474 #endif
 475 }
 476
 477 /* x86-64 is TSO. */
 478 static inline __attribute__((always_inline))
 479 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
 480                                          void *dst, void *src, size_t len,
 481                                          intptr_t newv, int cpu)
 482 {
 483         return rseq_cmpeqv_trymemcpy_storev(v, expect, dst, src, len,
 484                                             newv, cpu);
 485 }
 486
 487 #endif /* !RSEQ_SKIP_FASTPATH */
 488
 489 #elif __i386__
 490
 491 #define rseq_smp_mb()   \
 492         __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc")
 493 #define rseq_smp_rmb()  \
 494         __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc")
 495 #define rseq_smp_wmb()  \
 496         __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc")
 497
 498 #define rseq_smp_load_acquire(p)                                        \
 499 __extension__ ({                                                        \
 500         __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p);                       \
 501         rseq_smp_mb();                                                  \
 502         ____p1;                                                         \
 503 })
 504
 505 #define rseq_smp_acquire__after_ctrl_dep()      rseq_smp_rmb()
 506
 507 #define rseq_smp_store_release(p, v)                                    \
 508 do {                                                                    \
 509         rseq_smp_mb();                                                  \
 510         RSEQ_WRITE_ONCE(*p, v);                                         \
 511 } while (0)
 512
 513 #ifdef RSEQ_SKIP_FASTPATH
 514 #include "rseq-skip.h"
 515 #else /* !RSEQ_SKIP_FASTPATH */
 516
 517 /*
 518  * Use eax as scratch register and take memory operands as input to
 519  * lessen register pressure. Especially needed when compiling in O0.
 520  */
 521 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags,                  \
 522                                 start_ip, post_commit_offset, abort_ip) \
 523                 ".pushsection __rseq_table, \"aw\"\n\t"                 \
 524                 ".balign 32\n\t"                                        \
 525                 __rseq_str(label) ":\n\t"                               \
 526                 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
 527                 ".long " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \
 528                 ".popsection\n\t"
 529
 530 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
 531         __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip,              \
 532                                 (post_commit_ip - start_ip), abort_ip)
 533
 534 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs)                \
 535                 RSEQ_INJECT_ASM(1)                                      \
 536                 "movl $" __rseq_str(cs_label) ", %[rseq_cs]\n\t"        \
 537                 __rseq_str(label) ":\n\t"
 538
 539 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label)              \
 540                 RSEQ_INJECT_ASM(2)                                      \
 541                 "cmpl %[" __rseq_str(cpu_id) "], %[" __rseq_str(current_cpu_id) "]\n\t" \
 542                 "jnz " __rseq_str(label) "\n\t"
 543
 544 #define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label)             \
 545                 ".pushsection __rseq_failure, \"ax\"\n\t"               \
 546                 /* Disassembler-friendly signature: nopl <sig>. */      \
 547                 ".byte 0x0f, 0x1f, 0x05\n\t"                            \
 548                 ".long " __rseq_str(RSEQ_SIG) "\n\t"                    \
 549                 __rseq_str(label) ":\n\t"                               \
 550                 teardown                                                \
 551                 "jmp %l[" __rseq_str(abort_label) "]\n\t"               \
 552                 ".popsection\n\t"
 553
 554 #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label)         \
 555                 ".pushsection __rseq_failure, \"ax\"\n\t"               \
 556                 __rseq_str(label) ":\n\t"                               \
 557                 teardown                                                \
 558                 "jmp %l[" __rseq_str(cmpfail_label) "]\n\t"             \
 559                 ".popsection\n\t"
 560
 561 static inline __attribute__((always_inline))
 562 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
 563 {
 564         RSEQ_INJECT_C(9)
 565
 566         __asm__ __volatile__ goto (
 567                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 568                 /* Start rseq by storing table entry pointer into rseq_cs. */
 569                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 570                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 571                 RSEQ_INJECT_ASM(3)
 572                 "cmpl %[v], %[expect]\n\t"
 573                 "jnz %l[cmpfail]\n\t"
 574                 RSEQ_INJECT_ASM(4)
 575 #ifdef RSEQ_COMPARE_TWICE
 576                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 577                 "cmpl %[v], %[expect]\n\t"
 578                 "jnz %l[error2]\n\t"
 579 #endif
 580                 /* final store */
 581                 "movl %[newv], %[v]\n\t"
 582                 "2:\n\t"
 583                 RSEQ_INJECT_ASM(5)
 584                 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 585                 : /* gcc asm goto does not allow outputs */
 586                 : [cpu_id]              "r" (cpu),
 587                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
 588                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 589                   [v]                   "m" (*v),
 590                   [expect]              "r" (expect),
 591                   [newv]                "r" (newv)
 592                 : "memory", "cc", "eax"
 593                   RSEQ_INJECT_CLOBBER
 594                 : abort, cmpfail
 595 #ifdef RSEQ_COMPARE_TWICE
 596                   , error1, error2
 597 #endif
 598         );
 599         return 0;
 600 abort:
 601         RSEQ_INJECT_FAILED
 602         return -1;
 603 cmpfail:
 604         return 1;
 605 #ifdef RSEQ_COMPARE_TWICE
 606 error1:
 607         rseq_bug("cpu_id comparison failed");
 608 error2:
 609         rseq_bug("expected value comparison failed");
 610 #endif
 611 }
 612
 613 /*
 614  * Compare @v against @expectnot. When it does _not_ match, load @v
 615  * into @load, and store the content of *@v + voffp into @v.
 616  */
 617 static inline __attribute__((always_inline))
 618 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
 619                                off_t voffp, intptr_t *load, int cpu)
 620 {
 621         RSEQ_INJECT_C(9)
 622
 623         __asm__ __volatile__ goto (
 624                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 625                 /* Start rseq by storing table entry pointer into rseq_cs. */
 626                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 627                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 628                 RSEQ_INJECT_ASM(3)
 629                 "movl %[v], %%ebx\n\t"
 630                 "cmpl %%ebx, %[expectnot]\n\t"
 631                 "je %l[cmpfail]\n\t"
 632                 RSEQ_INJECT_ASM(4)
 633 #ifdef RSEQ_COMPARE_TWICE
 634                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 635                 "movl %[v], %%ebx\n\t"
 636                 "cmpl %%ebx, %[expectnot]\n\t"
 637                 "je %l[error2]\n\t"
 638 #endif
 639                 "movl %%ebx, %[load]\n\t"
 640                 "addl %[voffp], %%ebx\n\t"
 641                 "movl (%%ebx), %%ebx\n\t"
 642                 /* final store */
 643                 "movl %%ebx, %[v]\n\t"
 644                 "2:\n\t"
 645                 RSEQ_INJECT_ASM(5)
 646                 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 647                 : /* gcc asm goto does not allow outputs */
 648                 : [cpu_id]              "r" (cpu),
 649                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
 650                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 651                   /* final store input */
 652                   [v]                   "m" (*v),
 653                   [expectnot]           "r" (expectnot),
 654                   [voffp]               "ir" (voffp),
 655                   [load]                "m" (*load)
 656                 : "memory", "cc", "eax", "ebx"
 657                   RSEQ_INJECT_CLOBBER
 658                 : abort, cmpfail
 659 #ifdef RSEQ_COMPARE_TWICE
 660                   , error1, error2
 661 #endif
 662         );
 663         return 0;
 664 abort:
 665         RSEQ_INJECT_FAILED
 666         return -1;
 667 cmpfail:
 668         return 1;
 669 #ifdef RSEQ_COMPARE_TWICE
 670 error1:
 671         rseq_bug("cpu_id comparison failed");
 672 error2:
 673         rseq_bug("expected value comparison failed");
 674 #endif
 675 }
 676
 677 static inline __attribute__((always_inline))
 678 int rseq_addv(intptr_t *v, intptr_t count, int cpu)
 679 {
 680         RSEQ_INJECT_C(9)
 681
 682         __asm__ __volatile__ goto (
 683                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 684                 /* Start rseq by storing table entry pointer into rseq_cs. */
 685                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 686                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 687                 RSEQ_INJECT_ASM(3)
 688 #ifdef RSEQ_COMPARE_TWICE
 689                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 690 #endif
 691                 /* final store */
 692                 "addl %[count], %[v]\n\t"
 693                 "2:\n\t"
 694                 RSEQ_INJECT_ASM(4)
 695                 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 696                 : /* gcc asm goto does not allow outputs */
 697                 : [cpu_id]              "r" (cpu),
 698                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
 699                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 700                   /* final store input */
 701                   [v]                   "m" (*v),
 702                   [count]               "ir" (count)
 703                 : "memory", "cc", "eax"
 704                   RSEQ_INJECT_CLOBBER
 705                 : abort
 706 #ifdef RSEQ_COMPARE_TWICE
 707                   , error1
 708 #endif
 709         );
 710         return 0;
 711 abort:
 712         RSEQ_INJECT_FAILED
 713         return -1;
 714 #ifdef RSEQ_COMPARE_TWICE
 715 error1:
 716         rseq_bug("cpu_id comparison failed");
 717 #endif
 718 }
 719
 720 static inline __attribute__((always_inline))
 721 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
 722                                  intptr_t *v2, intptr_t newv2,
 723                                  intptr_t newv, int cpu)
 724 {
 725         RSEQ_INJECT_C(9)
 726
 727         __asm__ __volatile__ goto (
 728                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 729                 /* Start rseq by storing table entry pointer into rseq_cs. */
 730                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 731                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 732                 RSEQ_INJECT_ASM(3)
 733                 "cmpl %[v], %[expect]\n\t"
 734                 "jnz %l[cmpfail]\n\t"
 735                 RSEQ_INJECT_ASM(4)
 736 #ifdef RSEQ_COMPARE_TWICE
 737                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 738                 "cmpl %[v], %[expect]\n\t"
 739                 "jnz %l[error2]\n\t"
 740 #endif
 741                 /* try store */
 742                 "movl %[newv2], %%eax\n\t"
 743                 "movl %%eax, %[v2]\n\t"
 744                 RSEQ_INJECT_ASM(5)
 745                 /* final store */
 746                 "movl %[newv], %[v]\n\t"
 747                 "2:\n\t"
 748                 RSEQ_INJECT_ASM(6)
 749                 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 750                 : /* gcc asm goto does not allow outputs */
 751                 : [cpu_id]              "r" (cpu),
 752                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
 753                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 754                   /* try store input */
 755                   [v2]                  "m" (*v2),
 756                   [newv2]               "m" (newv2),
 757                   /* final store input */
 758                   [v]                   "m" (*v),
 759                   [expect]              "r" (expect),
 760                   [newv]                "r" (newv)
 761                 : "memory", "cc", "eax"
 762                   RSEQ_INJECT_CLOBBER
 763                 : abort, cmpfail
 764 #ifdef RSEQ_COMPARE_TWICE
 765                   , error1, error2
 766 #endif
 767         );
 768         return 0;
 769 abort:
 770         RSEQ_INJECT_FAILED
 771         return -1;
 772 cmpfail:
 773         return 1;
 774 #ifdef RSEQ_COMPARE_TWICE
 775 error1:
 776         rseq_bug("cpu_id comparison failed");
 777 error2:
 778         rseq_bug("expected value comparison failed");
 779 #endif
 780 }
 781
 782 static inline __attribute__((always_inline))
 783 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
 784                                          intptr_t *v2, intptr_t newv2,
 785                                          intptr_t newv, int cpu)
 786 {
 787         RSEQ_INJECT_C(9)
 788
 789         __asm__ __volatile__ goto (
 790                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 791                 /* Start rseq by storing table entry pointer into rseq_cs. */
 792                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 793                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 794                 RSEQ_INJECT_ASM(3)
 795                 "movl %[expect], %%eax\n\t"
 796                 "cmpl %[v], %%eax\n\t"
 797                 "jnz %l[cmpfail]\n\t"
 798                 RSEQ_INJECT_ASM(4)
 799 #ifdef RSEQ_COMPARE_TWICE
 800                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 801                 "movl %[expect], %%eax\n\t"
 802                 "cmpl %[v], %%eax\n\t"
 803                 "jnz %l[error2]\n\t"
 804 #endif
 805                 /* try store */
 806                 "movl %[newv2], %[v2]\n\t"
 807                 RSEQ_INJECT_ASM(5)
 808                 "lock; addl $0,-128(%%esp)\n\t"
 809                 /* final store */
 810                 "movl %[newv], %[v]\n\t"
 811                 "2:\n\t"
 812                 RSEQ_INJECT_ASM(6)
 813                 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 814                 : /* gcc asm goto does not allow outputs */
 815                 : [cpu_id]              "r" (cpu),
 816                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
 817                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 818                   /* try store input */
 819                   [v2]                  "m" (*v2),
 820                   [newv2]               "r" (newv2),
 821                   /* final store input */
 822                   [v]                   "m" (*v),
 823                   [expect]              "m" (expect),
 824                   [newv]                "r" (newv)
 825                 : "memory", "cc", "eax"
 826                   RSEQ_INJECT_CLOBBER
 827                 : abort, cmpfail
 828 #ifdef RSEQ_COMPARE_TWICE
 829                   , error1, error2
 830 #endif
 831         );
 832         return 0;
 833 abort:
 834         RSEQ_INJECT_FAILED
 835         return -1;
 836 cmpfail:
 837         return 1;
 838 #ifdef RSEQ_COMPARE_TWICE
 839 error1:
 840         rseq_bug("cpu_id comparison failed");
 841 error2:
 842         rseq_bug("expected value comparison failed");
 843 #endif
 844
 845 }
 846
 847 static inline __attribute__((always_inline))
 848 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
 849                               intptr_t *v2, intptr_t expect2,
 850                               intptr_t newv, int cpu)
 851 {
 852         RSEQ_INJECT_C(9)
 853
 854         __asm__ __volatile__ goto (
 855                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 856                 /* Start rseq by storing table entry pointer into rseq_cs. */
 857                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 858                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 859                 RSEQ_INJECT_ASM(3)
 860                 "cmpl %[v], %[expect]\n\t"
 861                 "jnz %l[cmpfail]\n\t"
 862                 RSEQ_INJECT_ASM(4)
 863                 "cmpl %[expect2], %[v2]\n\t"
 864                 "jnz %l[cmpfail]\n\t"
 865                 RSEQ_INJECT_ASM(5)
 866 #ifdef RSEQ_COMPARE_TWICE
 867                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 868                 "cmpl %[v], %[expect]\n\t"
 869                 "jnz %l[error2]\n\t"
 870                 "cmpl %[expect2], %[v2]\n\t"
 871                 "jnz %l[error3]\n\t"
 872 #endif
 873                 "movl %[newv], %%eax\n\t"
 874                 /* final store */
 875                 "movl %%eax, %[v]\n\t"
 876                 "2:\n\t"
 877                 RSEQ_INJECT_ASM(6)
 878                 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 879                 : /* gcc asm goto does not allow outputs */
 880                 : [cpu_id]              "r" (cpu),
 881                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
 882                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 883                   /* cmp2 input */
 884                   [v2]                  "m" (*v2),
 885                   [expect2]             "r" (expect2),
 886                   /* final store input */
 887                   [v]                   "m" (*v),
 888                   [expect]              "r" (expect),
 889                   [newv]                "m" (newv)
 890                 : "memory", "cc", "eax"
 891                   RSEQ_INJECT_CLOBBER
 892                 : abort, cmpfail
 893 #ifdef RSEQ_COMPARE_TWICE
 894                   , error1, error2, error3
 895 #endif
 896         );
 897         return 0;
 898 abort:
 899         RSEQ_INJECT_FAILED
 900         return -1;
 901 cmpfail:
 902         return 1;
 903 #ifdef RSEQ_COMPARE_TWICE
 904 error1:
 905         rseq_bug("cpu_id comparison failed");
 906 error2:
 907         rseq_bug("1st expected value comparison failed");
 908 error3:
 909         rseq_bug("2nd expected value comparison failed");
 910 #endif
 911 }
 912
 913 /* TODO: implement a faster memcpy. */
 914 static inline __attribute__((always_inline))
 915 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
 916                                  void *dst, void *src, size_t len,
 917                                  intptr_t newv, int cpu)
 918 {
 919         uint32_t rseq_scratch[3];
 920
 921         RSEQ_INJECT_C(9)
 922
 923         __asm__ __volatile__ goto (
 924                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 925                 "movl %[src], %[rseq_scratch0]\n\t"
 926                 "movl %[dst], %[rseq_scratch1]\n\t"
 927                 "movl %[len], %[rseq_scratch2]\n\t"
 928                 /* Start rseq by storing table entry pointer into rseq_cs. */
 929                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 930                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 931                 RSEQ_INJECT_ASM(3)
 932                 "movl %[expect], %%eax\n\t"
 933                 "cmpl %%eax, %[v]\n\t"
 934                 "jnz 5f\n\t"
 935                 RSEQ_INJECT_ASM(4)
 936 #ifdef RSEQ_COMPARE_TWICE
 937                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f)
 938                 "movl %[expect], %%eax\n\t"
 939                 "cmpl %%eax, %[v]\n\t"
 940                 "jnz 7f\n\t"
 941 #endif
 942                 /* try memcpy */
 943                 "test %[len], %[len]\n\t" \
 944                 "jz 333f\n\t" \
 945                 "222:\n\t" \
 946                 "movb (%[src]), %%al\n\t" \
 947                 "movb %%al, (%[dst])\n\t" \
 948                 "inc %[src]\n\t" \
 949                 "inc %[dst]\n\t" \
 950                 "dec %[len]\n\t" \
 951                 "jnz 222b\n\t" \
 952                 "333:\n\t" \
 953                 RSEQ_INJECT_ASM(5)
 954                 "movl %[newv], %%eax\n\t"
 955                 /* final store */
 956                 "movl %%eax, %[v]\n\t"
 957                 "2:\n\t"
 958                 RSEQ_INJECT_ASM(6)
 959                 /* teardown */
 960                 "movl %[rseq_scratch2], %[len]\n\t"
 961                 "movl %[rseq_scratch1], %[dst]\n\t"
 962                 "movl %[rseq_scratch0], %[src]\n\t"
 963                 RSEQ_ASM_DEFINE_ABORT(4,
 964                         "movl %[rseq_scratch2], %[len]\n\t"
 965                         "movl %[rseq_scratch1], %[dst]\n\t"
 966                         "movl %[rseq_scratch0], %[src]\n\t",
 967                         abort)
 968                 RSEQ_ASM_DEFINE_CMPFAIL(5,
 969                         "movl %[rseq_scratch2], %[len]\n\t"
 970                         "movl %[rseq_scratch1], %[dst]\n\t"
 971                         "movl %[rseq_scratch0], %[src]\n\t",
 972                         cmpfail)
 973 #ifdef RSEQ_COMPARE_TWICE
 974                 RSEQ_ASM_DEFINE_CMPFAIL(6,
 975                         "movl %[rseq_scratch2], %[len]\n\t"
 976                         "movl %[rseq_scratch1], %[dst]\n\t"
 977                         "movl %[rseq_scratch0], %[src]\n\t",
 978                         error1)
 979                 RSEQ_ASM_DEFINE_CMPFAIL(7,
 980                         "movl %[rseq_scratch2], %[len]\n\t"
 981                         "movl %[rseq_scratch1], %[dst]\n\t"
 982                         "movl %[rseq_scratch0], %[src]\n\t",
 983                         error2)
 984 #endif
 985                 : /* gcc asm goto does not allow outputs */
 986                 : [cpu_id]              "r" (cpu),
 987                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
 988                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 989                   /* final store input */
 990                   [v]                   "m" (*v),
 991                   [expect]              "m" (expect),
 992                   [newv]                "m" (newv),
 993                   /* try memcpy input */
 994                   [dst]                 "r" (dst),
 995                   [src]                 "r" (src),
 996                   [len]                 "r" (len),
 997                   [rseq_scratch0]       "m" (rseq_scratch[0]),
 998                   [rseq_scratch1]       "m" (rseq_scratch[1]),
 999                   [rseq_scratch2]       "m" (rseq_scratch[2])
1000                 : "memory", "cc", "eax"
1001                   RSEQ_INJECT_CLOBBER
1002                 : abort, cmpfail
1003 #ifdef RSEQ_COMPARE_TWICE
1004                   , error1, error2
1005 #endif
1006         );
1007         return 0;
1008 abort:
1009         RSEQ_INJECT_FAILED
1010         return -1;
1011 cmpfail:
1012         return 1;
1013 #ifdef RSEQ_COMPARE_TWICE
1014 error1:
1015         rseq_bug("cpu_id comparison failed");
1016 error2:
1017         rseq_bug("expected value comparison failed");
1018 #endif
1019 }
1020
1021 /* TODO: implement a faster memcpy. */
1022 static inline __attribute__((always_inline))
1023 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
1024                                          void *dst, void *src, size_t len,
1025                                          intptr_t newv, int cpu)
1026 {
1027         uint32_t rseq_scratch[3];
1028
1029         RSEQ_INJECT_C(9)
1030
1031         __asm__ __volatile__ goto (
1032                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
1033                 "movl %[src], %[rseq_scratch0]\n\t"
1034                 "movl %[dst], %[rseq_scratch1]\n\t"
1035                 "movl %[len], %[rseq_scratch2]\n\t"
1036                 /* Start rseq by storing table entry pointer into rseq_cs. */
1037                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
1038                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
1039                 RSEQ_INJECT_ASM(3)
1040                 "movl %[expect], %%eax\n\t"
1041                 "cmpl %%eax, %[v]\n\t"
1042                 "jnz 5f\n\t"
1043                 RSEQ_INJECT_ASM(4)
1044 #ifdef RSEQ_COMPARE_TWICE
1045                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f)
1046                 "movl %[expect], %%eax\n\t"
1047                 "cmpl %%eax, %[v]\n\t"
1048                 "jnz 7f\n\t"
1049 #endif
1050                 /* try memcpy */
1051                 "test %[len], %[len]\n\t" \
1052                 "jz 333f\n\t" \
1053                 "222:\n\t" \
1054                 "movb (%[src]), %%al\n\t" \
1055                 "movb %%al, (%[dst])\n\t" \
1056                 "inc %[src]\n\t" \
1057                 "inc %[dst]\n\t" \
1058                 "dec %[len]\n\t" \
1059                 "jnz 222b\n\t" \
1060                 "333:\n\t" \
1061                 RSEQ_INJECT_ASM(5)
1062                 "lock; addl $0,-128(%%esp)\n\t"
1063                 "movl %[newv], %%eax\n\t"
1064                 /* final store */
1065                 "movl %%eax, %[v]\n\t"
1066                 "2:\n\t"
1067                 RSEQ_INJECT_ASM(6)
1068                 /* teardown */
1069                 "movl %[rseq_scratch2], %[len]\n\t"
1070                 "movl %[rseq_scratch1], %[dst]\n\t"
1071                 "movl %[rseq_scratch0], %[src]\n\t"
1072                 RSEQ_ASM_DEFINE_ABORT(4,
1073                         "movl %[rseq_scratch2], %[len]\n\t"
1074                         "movl %[rseq_scratch1], %[dst]\n\t"
1075                         "movl %[rseq_scratch0], %[src]\n\t",
1076                         abort)
1077                 RSEQ_ASM_DEFINE_CMPFAIL(5,
1078                         "movl %[rseq_scratch2], %[len]\n\t"
1079                         "movl %[rseq_scratch1], %[dst]\n\t"
1080                         "movl %[rseq_scratch0], %[src]\n\t",
1081                         cmpfail)
1082 #ifdef RSEQ_COMPARE_TWICE
1083                 RSEQ_ASM_DEFINE_CMPFAIL(6,
1084                         "movl %[rseq_scratch2], %[len]\n\t"
1085                         "movl %[rseq_scratch1], %[dst]\n\t"
1086                         "movl %[rseq_scratch0], %[src]\n\t",
1087                         error1)
1088                 RSEQ_ASM_DEFINE_CMPFAIL(7,
1089                         "movl %[rseq_scratch2], %[len]\n\t"
1090                         "movl %[rseq_scratch1], %[dst]\n\t"
1091                         "movl %[rseq_scratch0], %[src]\n\t",
1092                         error2)
1093 #endif
1094                 : /* gcc asm goto does not allow outputs */
1095                 : [cpu_id]              "r" (cpu),
1096                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
1097                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
1098                   /* final store input */
1099                   [v]                   "m" (*v),
1100                   [expect]              "m" (expect),
1101                   [newv]                "m" (newv),
1102                   /* try memcpy input */
1103                   [dst]                 "r" (dst),
1104                   [src]                 "r" (src),
1105                   [len]                 "r" (len),
1106                   [rseq_scratch0]       "m" (rseq_scratch[0]),
1107                   [rseq_scratch1]       "m" (rseq_scratch[1]),
1108                   [rseq_scratch2]       "m" (rseq_scratch[2])
1109                 : "memory", "cc", "eax"
1110                   RSEQ_INJECT_CLOBBER
1111                 : abort, cmpfail
1112 #ifdef RSEQ_COMPARE_TWICE
1113                   , error1, error2
1114 #endif
1115         );
1116         return 0;
1117 abort:
1118         RSEQ_INJECT_FAILED
1119         return -1;
1120 cmpfail:
1121         return 1;
1122 #ifdef RSEQ_COMPARE_TWICE
1123 error1:
1124         rseq_bug("cpu_id comparison failed");
1125 error2:
1126         rseq_bug("expected value comparison failed");
1127 #endif
1128 }
1129
1130 #endif /* !RSEQ_SKIP_FASTPATH */
1131
1132 #endif