kernel/locking/locktorture.c

   1 /*
   2  * Module-based torture test facility for locking
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License as published by
   6  * the Free Software Foundation; either version 2 of the License, or
   7  * (at your option) any later version.
   8  *
   9  * This program is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  * GNU General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License
  15  * along with this program; if not, you can access it online at
  16  * http://www.gnu.org/licenses/gpl-2.0.html.
  17  *
  18  * Copyright (C) IBM Corporation, 2014
  19  *
  20  * Authors: Paul E. McKenney <paulmck@us.ibm.com>
  21  *          Davidlohr Bueso <dave@stgolabs.net>
  22  *      Based on kernel/rcu/torture.c.
  23  */
  24 #include <linux/kernel.h>
  25 #include <linux/module.h>
  26 #include <linux/kthread.h>
  27 #include <linux/sched/rt.h>
  28 #include <linux/spinlock.h>
  29 #include <linux/rwlock.h>
  30 #include <linux/mutex.h>
  31 #include <linux/rwsem.h>
  32 #include <linux/smp.h>
  33 #include <linux/interrupt.h>
  34 #include <linux/sched.h>
  35 #include <linux/atomic.h>
  36 #include <linux/moduleparam.h>
  37 #include <linux/delay.h>
  38 #include <linux/slab.h>
  39 #include <linux/percpu-rwsem.h>
  40 #include <linux/torture.h>
  41
  42 MODULE_LICENSE("GPL");
  43 MODULE_AUTHOR("Paul E. McKenney <paulmck@us.ibm.com>");
  44
  45 torture_param(int, nwriters_stress, -1,
  46              "Number of write-locking stress-test threads");
  47 torture_param(int, nreaders_stress, -1,
  48              "Number of read-locking stress-test threads");
  49 torture_param(int, onoff_holdoff, 0, "Time after boot before CPU hotplugs (s)");
  50 torture_param(int, onoff_interval, 0,
  51              "Time between CPU hotplugs (s), 0=disable");
  52 torture_param(int, shuffle_interval, 3,
  53              "Number of jiffies between shuffles, 0=disable");
  54 torture_param(int, shutdown_secs, 0, "Shutdown time (j), <= zero to disable.");
  55 torture_param(int, stat_interval, 60,
  56              "Number of seconds between stats printk()s");
  57 torture_param(int, stutter, 5, "Number of jiffies to run/halt test, 0=disable");
  58 torture_param(bool, verbose, true,
  59              "Enable verbose debugging printk()s");
  60
  61 static char *torture_type = "spin_lock";
  62 module_param(torture_type, charp, 0444);
  63 MODULE_PARM_DESC(torture_type,
  64                  "Type of lock to torture (spin_lock, spin_lock_irq, mutex_lock, ...)");
  65
  66 static struct task_struct *stats_task;
  67 static struct task_struct **writer_tasks;
  68 static struct task_struct **reader_tasks;
  69
  70 static bool lock_is_write_held;
  71 static bool lock_is_read_held;
  72
  73 struct lock_stress_stats {
  74         long n_lock_fail;
  75         long n_lock_acquired;
  76 };
  77
  78 #if defined(MODULE)
  79 #define LOCKTORTURE_RUNNABLE_INIT 1
  80 #else
  81 #define LOCKTORTURE_RUNNABLE_INIT 0
  82 #endif
  83 int torture_runnable = LOCKTORTURE_RUNNABLE_INIT;
  84 module_param(torture_runnable, int, 0444);
  85 MODULE_PARM_DESC(torture_runnable, "Start locktorture at module init");
  86
  87 /* Forward reference. */
  88 static void lock_torture_cleanup(void);
  89
  90 /*
  91  * Operations vector for selecting different types of tests.
  92  */
  93 struct lock_torture_ops {
  94         void (*init)(void);
  95         int (*writelock)(void);
  96         void (*write_delay)(struct torture_random_state *trsp);
  97         void (*task_boost)(struct torture_random_state *trsp);
  98         void (*writeunlock)(void);
  99         int (*readlock)(void);
 100         void (*read_delay)(struct torture_random_state *trsp);
 101         void (*readunlock)(void);
 102
 103         unsigned long flags; /* for irq spinlocks */
 104         const char *name;
 105 };
 106
 107 struct lock_torture_cxt {
 108         int nrealwriters_stress;
 109         int nrealreaders_stress;
 110         bool debug_lock;
 111         atomic_t n_lock_torture_errors;
 112         struct lock_torture_ops *cur_ops;
 113         struct lock_stress_stats *lwsa; /* writer statistics */
 114         struct lock_stress_stats *lrsa; /* reader statistics */
 115 };
 116 static struct lock_torture_cxt cxt = { 0, 0, false,
 117                                        ATOMIC_INIT(0),
 118                                        NULL, NULL};
 119 /*
 120  * Definitions for lock torture testing.
 121  */
 122
 123 static int torture_lock_busted_write_lock(void)
 124 {
 125         return 0;  /* BUGGY, do not use in real life!!! */
 126 }
 127
 128 static void torture_lock_busted_write_delay(struct torture_random_state *trsp)
 129 {
 130         const unsigned long longdelay_ms = 100;
 131
 132         /* We want a long delay occasionally to force massive contention.  */
 133         if (!(torture_random(trsp) %
 134               (cxt.nrealwriters_stress * 2000 * longdelay_ms)))
 135                 mdelay(longdelay_ms);
 136 #ifdef CONFIG_PREEMPT
 137         if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000)))
 138                 preempt_schedule();  /* Allow test to be preempted. */
 139 #endif
 140 }
 141
 142 static void torture_lock_busted_write_unlock(void)
 143 {
 144           /* BUGGY, do not use in real life!!! */
 145 }
 146
 147 static void torture_boost_dummy(struct torture_random_state *trsp)
 148 {
 149         /* Only rtmutexes care about priority */
 150 }
 151
 152 static struct lock_torture_ops lock_busted_ops = {
 153         .writelock      = torture_lock_busted_write_lock,
 154         .write_delay    = torture_lock_busted_write_delay,
 155         .task_boost     = torture_boost_dummy,
 156         .writeunlock    = torture_lock_busted_write_unlock,
 157         .readlock       = NULL,
 158         .read_delay     = NULL,
 159         .readunlock     = NULL,
 160         .name           = "lock_busted"
 161 };
 162
 163 static DEFINE_SPINLOCK(torture_spinlock);
 164
 165 static int torture_spin_lock_write_lock(void) __acquires(torture_spinlock)
 166 {
 167         spin_lock(&torture_spinlock);
 168         return 0;
 169 }
 170
 171 static void torture_spin_lock_write_delay(struct torture_random_state *trsp)
 172 {
 173         const unsigned long shortdelay_us = 2;
 174         const unsigned long longdelay_ms = 100;
 175
 176         /* We want a short delay mostly to emulate likely code, and
 177          * we want a long delay occasionally to force massive contention.
 178          */
 179         if (!(torture_random(trsp) %
 180               (cxt.nrealwriters_stress * 2000 * longdelay_ms)))
 181                 mdelay(longdelay_ms);
 182         if (!(torture_random(trsp) %
 183               (cxt.nrealwriters_stress * 2 * shortdelay_us)))
 184                 udelay(shortdelay_us);
 185 #ifdef CONFIG_PREEMPT
 186         if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000)))
 187                 preempt_schedule();  /* Allow test to be preempted. */
 188 #endif
 189 }
 190
 191 static void torture_spin_lock_write_unlock(void) __releases(torture_spinlock)
 192 {
 193         spin_unlock(&torture_spinlock);
 194 }
 195
 196 static struct lock_torture_ops spin_lock_ops = {
 197         .writelock      = torture_spin_lock_write_lock,
 198         .write_delay    = torture_spin_lock_write_delay,
 199         .task_boost     = torture_boost_dummy,
 200         .writeunlock    = torture_spin_lock_write_unlock,
 201         .readlock       = NULL,
 202         .read_delay     = NULL,
 203         .readunlock     = NULL,
 204         .name           = "spin_lock"
 205 };
 206
 207 static int torture_spin_lock_write_lock_irq(void)
 208 __acquires(torture_spinlock)
 209 {
 210         unsigned long flags;
 211
 212         spin_lock_irqsave(&torture_spinlock, flags);
 213         cxt.cur_ops->flags = flags;
 214         return 0;
 215 }
 216
 217 static void torture_lock_spin_write_unlock_irq(void)
 218 __releases(torture_spinlock)
 219 {
 220         spin_unlock_irqrestore(&torture_spinlock, cxt.cur_ops->flags);
 221 }
 222
 223 static struct lock_torture_ops spin_lock_irq_ops = {
 224         .writelock      = torture_spin_lock_write_lock_irq,
 225         .write_delay    = torture_spin_lock_write_delay,
 226         .task_boost     = torture_boost_dummy,
 227         .writeunlock    = torture_lock_spin_write_unlock_irq,
 228         .readlock       = NULL,
 229         .read_delay     = NULL,
 230         .readunlock     = NULL,
 231         .name           = "spin_lock_irq"
 232 };
 233
 234 static DEFINE_RWLOCK(torture_rwlock);
 235
 236 static int torture_rwlock_write_lock(void) __acquires(torture_rwlock)
 237 {
 238         write_lock(&torture_rwlock);
 239         return 0;
 240 }
 241
 242 static void torture_rwlock_write_delay(struct torture_random_state *trsp)
 243 {
 244         const unsigned long shortdelay_us = 2;
 245         const unsigned long longdelay_ms = 100;
 246
 247         /* We want a short delay mostly to emulate likely code, and
 248          * we want a long delay occasionally to force massive contention.
 249          */
 250         if (!(torture_random(trsp) %
 251               (cxt.nrealwriters_stress * 2000 * longdelay_ms)))
 252                 mdelay(longdelay_ms);
 253         else
 254                 udelay(shortdelay_us);
 255 }
 256
 257 static void torture_rwlock_write_unlock(void) __releases(torture_rwlock)
 258 {
 259         write_unlock(&torture_rwlock);
 260 }
 261
 262 static int torture_rwlock_read_lock(void) __acquires(torture_rwlock)
 263 {
 264         read_lock(&torture_rwlock);
 265         return 0;
 266 }
 267
 268 static void torture_rwlock_read_delay(struct torture_random_state *trsp)
 269 {
 270         const unsigned long shortdelay_us = 10;
 271         const unsigned long longdelay_ms = 100;
 272
 273         /* We want a short delay mostly to emulate likely code, and
 274          * we want a long delay occasionally to force massive contention.
 275          */
 276         if (!(torture_random(trsp) %
 277               (cxt.nrealreaders_stress * 2000 * longdelay_ms)))
 278                 mdelay(longdelay_ms);
 279         else
 280                 udelay(shortdelay_us);
 281 }
 282
 283 static void torture_rwlock_read_unlock(void) __releases(torture_rwlock)
 284 {
 285         read_unlock(&torture_rwlock);
 286 }
 287
 288 static struct lock_torture_ops rw_lock_ops = {
 289         .writelock      = torture_rwlock_write_lock,
 290         .write_delay    = torture_rwlock_write_delay,
 291         .task_boost     = torture_boost_dummy,
 292         .writeunlock    = torture_rwlock_write_unlock,
 293         .readlock       = torture_rwlock_read_lock,
 294         .read_delay     = torture_rwlock_read_delay,
 295         .readunlock     = torture_rwlock_read_unlock,
 296         .name           = "rw_lock"
 297 };
 298
 299 static int torture_rwlock_write_lock_irq(void) __acquires(torture_rwlock)
 300 {
 301         unsigned long flags;
 302
 303         write_lock_irqsave(&torture_rwlock, flags);
 304         cxt.cur_ops->flags = flags;
 305         return 0;
 306 }
 307
 308 static void torture_rwlock_write_unlock_irq(void)
 309 __releases(torture_rwlock)
 310 {
 311         write_unlock_irqrestore(&torture_rwlock, cxt.cur_ops->flags);
 312 }
 313
 314 static int torture_rwlock_read_lock_irq(void) __acquires(torture_rwlock)
 315 {
 316         unsigned long flags;
 317
 318         read_lock_irqsave(&torture_rwlock, flags);
 319         cxt.cur_ops->flags = flags;
 320         return 0;
 321 }
 322
 323 static void torture_rwlock_read_unlock_irq(void)
 324 __releases(torture_rwlock)
 325 {
 326         read_unlock_irqrestore(&torture_rwlock, cxt.cur_ops->flags);
 327 }
 328
 329 static struct lock_torture_ops rw_lock_irq_ops = {
 330         .writelock      = torture_rwlock_write_lock_irq,
 331         .write_delay    = torture_rwlock_write_delay,
 332         .task_boost     = torture_boost_dummy,
 333         .writeunlock    = torture_rwlock_write_unlock_irq,
 334         .readlock       = torture_rwlock_read_lock_irq,
 335         .read_delay     = torture_rwlock_read_delay,
 336         .readunlock     = torture_rwlock_read_unlock_irq,
 337         .name           = "rw_lock_irq"
 338 };
 339
 340 static DEFINE_MUTEX(torture_mutex);
 341
 342 static int torture_mutex_lock(void) __acquires(torture_mutex)
 343 {
 344         mutex_lock(&torture_mutex);
 345         return 0;
 346 }
 347
 348 static void torture_mutex_delay(struct torture_random_state *trsp)
 349 {
 350         const unsigned long longdelay_ms = 100;
 351
 352         /* We want a long delay occasionally to force massive contention.  */
 353         if (!(torture_random(trsp) %
 354               (cxt.nrealwriters_stress * 2000 * longdelay_ms)))
 355                 mdelay(longdelay_ms * 5);
 356         else
 357                 mdelay(longdelay_ms / 5);
 358 #ifdef CONFIG_PREEMPT
 359         if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000)))
 360                 preempt_schedule();  /* Allow test to be preempted. */
 361 #endif
 362 }
 363
 364 static void torture_mutex_unlock(void) __releases(torture_mutex)
 365 {
 366         mutex_unlock(&torture_mutex);
 367 }
 368
 369 static struct lock_torture_ops mutex_lock_ops = {
 370         .writelock      = torture_mutex_lock,
 371         .write_delay    = torture_mutex_delay,
 372         .task_boost     = torture_boost_dummy,
 373         .writeunlock    = torture_mutex_unlock,
 374         .readlock       = NULL,
 375         .read_delay     = NULL,
 376         .readunlock     = NULL,
 377         .name           = "mutex_lock"
 378 };
 379
 380 #ifdef CONFIG_RT_MUTEXES
 381 static DEFINE_RT_MUTEX(torture_rtmutex);
 382
 383 static int torture_rtmutex_lock(void) __acquires(torture_rtmutex)
 384 {
 385         rt_mutex_lock(&torture_rtmutex);
 386         return 0;
 387 }
 388
 389 static void torture_rtmutex_boost(struct torture_random_state *trsp)
 390 {
 391         int policy;
 392         struct sched_param param;
 393         const unsigned int factor = 50000; /* yes, quite arbitrary */
 394
 395         if (!rt_task(current)) {
 396                 /*
 397                  * (1) Boost priority once every ~50k operations. When the
 398                  * task tries to take the lock, the rtmutex it will account
 399                  * for the new priority, and do any corresponding pi-dance.
 400                  */
 401                 if (!(torture_random(trsp) %
 402                       (cxt.nrealwriters_stress * factor))) {
 403                         policy = SCHED_FIFO;
 404                         param.sched_priority = MAX_RT_PRIO - 1;
 405                 } else /* common case, do nothing */
 406                         return;
 407         } else {
 408                 /*
 409                  * The task will remain boosted for another ~500k operations,
 410                  * then restored back to its original prio, and so forth.
 411                  *
 412                  * When @trsp is nil, we want to force-reset the task for
 413                  * stopping the kthread.
 414                  */
 415                 if (!trsp || !(torture_random(trsp) %
 416                                (cxt.nrealwriters_stress * factor * 2))) {
 417                         policy = SCHED_NORMAL;
 418                         param.sched_priority = 0;
 419                 } else /* common case, do nothing */
 420                         return;
 421         }
 422
 423         sched_setscheduler_nocheck(current, policy, &param);
 424 }
 425
 426 static void torture_rtmutex_delay(struct torture_random_state *trsp)
 427 {
 428         const unsigned long shortdelay_us = 2;
 429         const unsigned long longdelay_ms = 100;
 430
 431         /*
 432          * We want a short delay mostly to emulate likely code, and
 433          * we want a long delay occasionally to force massive contention.
 434          */
 435         if (!(torture_random(trsp) %
 436               (cxt.nrealwriters_stress * 2000 * longdelay_ms)))
 437                 mdelay(longdelay_ms);
 438         if (!(torture_random(trsp) %
 439               (cxt.nrealwriters_stress * 2 * shortdelay_us)))
 440                 udelay(shortdelay_us);
 441 #ifdef CONFIG_PREEMPT
 442         if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000)))
 443                 preempt_schedule();  /* Allow test to be preempted. */
 444 #endif
 445 }
 446
 447 static void torture_rtmutex_unlock(void) __releases(torture_rtmutex)
 448 {
 449         rt_mutex_unlock(&torture_rtmutex);
 450 }
 451
 452 static struct lock_torture_ops rtmutex_lock_ops = {
 453         .writelock      = torture_rtmutex_lock,
 454         .write_delay    = torture_rtmutex_delay,
 455         .task_boost     = torture_rtmutex_boost,
 456         .writeunlock    = torture_rtmutex_unlock,
 457         .readlock       = NULL,
 458         .read_delay     = NULL,
 459         .readunlock     = NULL,
 460         .name           = "rtmutex_lock"
 461 };
 462 #endif
 463
 464 static DECLARE_RWSEM(torture_rwsem);
 465 static int torture_rwsem_down_write(void) __acquires(torture_rwsem)
 466 {
 467         down_write(&torture_rwsem);
 468         return 0;
 469 }
 470
 471 static void torture_rwsem_write_delay(struct torture_random_state *trsp)
 472 {
 473         const unsigned long longdelay_ms = 100;
 474
 475         /* We want a long delay occasionally to force massive contention.  */
 476         if (!(torture_random(trsp) %
 477               (cxt.nrealwriters_stress * 2000 * longdelay_ms)))
 478                 mdelay(longdelay_ms * 10);
 479         else
 480                 mdelay(longdelay_ms / 10);
 481 #ifdef CONFIG_PREEMPT
 482         if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000)))
 483                 preempt_schedule();  /* Allow test to be preempted. */
 484 #endif
 485 }
 486
 487 static void torture_rwsem_up_write(void) __releases(torture_rwsem)
 488 {
 489         up_write(&torture_rwsem);
 490 }
 491
 492 static int torture_rwsem_down_read(void) __acquires(torture_rwsem)
 493 {
 494         down_read(&torture_rwsem);
 495         return 0;
 496 }
 497
 498 static void torture_rwsem_read_delay(struct torture_random_state *trsp)
 499 {
 500         const unsigned long longdelay_ms = 100;
 501
 502         /* We want a long delay occasionally to force massive contention.  */
 503         if (!(torture_random(trsp) %
 504               (cxt.nrealwriters_stress * 2000 * longdelay_ms)))
 505                 mdelay(longdelay_ms * 2);
 506         else
 507                 mdelay(longdelay_ms / 2);
 508 #ifdef CONFIG_PREEMPT
 509         if (!(torture_random(trsp) % (cxt.nrealreaders_stress * 20000)))
 510                 preempt_schedule();  /* Allow test to be preempted. */
 511 #endif
 512 }
 513
 514 static void torture_rwsem_up_read(void) __releases(torture_rwsem)
 515 {
 516         up_read(&torture_rwsem);
 517 }
 518
 519 static struct lock_torture_ops rwsem_lock_ops = {
 520         .writelock      = torture_rwsem_down_write,
 521         .write_delay    = torture_rwsem_write_delay,
 522         .task_boost     = torture_boost_dummy,
 523         .writeunlock    = torture_rwsem_up_write,
 524         .readlock       = torture_rwsem_down_read,
 525         .read_delay     = torture_rwsem_read_delay,
 526         .readunlock     = torture_rwsem_up_read,
 527         .name           = "rwsem_lock"
 528 };
 529
 530 #include <linux/percpu-rwsem.h>
 531 static struct percpu_rw_semaphore pcpu_rwsem;
 532
 533 void torture_percpu_rwsem_init(void)
 534 {
 535         BUG_ON(percpu_init_rwsem(&pcpu_rwsem));
 536 }
 537
 538 static int torture_percpu_rwsem_down_write(void) __acquires(pcpu_rwsem)
 539 {
 540         percpu_down_write(&pcpu_rwsem);
 541         return 0;
 542 }
 543
 544 static void torture_percpu_rwsem_up_write(void) __releases(pcpu_rwsem)
 545 {
 546         percpu_up_write(&pcpu_rwsem);
 547 }
 548
 549 static int torture_percpu_rwsem_down_read(void) __acquires(pcpu_rwsem)
 550 {
 551         percpu_down_read(&pcpu_rwsem);
 552         return 0;
 553 }
 554
 555 static void torture_percpu_rwsem_up_read(void) __releases(pcpu_rwsem)
 556 {
 557         percpu_up_read(&pcpu_rwsem);
 558 }
 559
 560 static struct lock_torture_ops percpu_rwsem_lock_ops = {
 561         .init           = torture_percpu_rwsem_init,
 562         .writelock      = torture_percpu_rwsem_down_write,
 563         .write_delay    = torture_rwsem_write_delay,
 564         .task_boost     = torture_boost_dummy,
 565         .writeunlock    = torture_percpu_rwsem_up_write,
 566         .readlock       = torture_percpu_rwsem_down_read,
 567         .read_delay     = torture_rwsem_read_delay,
 568         .readunlock     = torture_percpu_rwsem_up_read,
 569         .name           = "percpu_rwsem_lock"
 570 };
 571
 572 /*
 573  * Lock torture writer kthread.  Repeatedly acquires and releases
 574  * the lock, checking for duplicate acquisitions.
 575  */
 576 static int lock_torture_writer(void *arg)
 577 {
 578         struct lock_stress_stats *lwsp = arg;
 579         static DEFINE_TORTURE_RANDOM(rand);
 580
 581         VERBOSE_TOROUT_STRING("lock_torture_writer task started");
 582         set_user_nice(current, MAX_NICE);
 583
 584         do {
 585                 if ((torture_random(&rand) & 0xfffff) == 0)
 586                         schedule_timeout_uninterruptible(1);
 587
 588                 cxt.cur_ops->task_boost(&rand);
 589                 cxt.cur_ops->writelock();
 590                 if (WARN_ON_ONCE(lock_is_write_held))
 591                         lwsp->n_lock_fail++;
 592                 lock_is_write_held = 1;
 593                 if (WARN_ON_ONCE(lock_is_read_held))
 594                         lwsp->n_lock_fail++; /* rare, but... */
 595
 596                 lwsp->n_lock_acquired++;
 597                 cxt.cur_ops->write_delay(&rand);
 598                 lock_is_write_held = 0;
 599                 cxt.cur_ops->writeunlock();
 600
 601                 stutter_wait("lock_torture_writer");
 602         } while (!torture_must_stop());
 603
 604         cxt.cur_ops->task_boost(NULL); /* reset prio */
 605         torture_kthread_stopping("lock_torture_writer");
 606         return 0;
 607 }
 608
 609 /*
 610  * Lock torture reader kthread.  Repeatedly acquires and releases
 611  * the reader lock.
 612  */
 613 static int lock_torture_reader(void *arg)
 614 {
 615         struct lock_stress_stats *lrsp = arg;
 616         static DEFINE_TORTURE_RANDOM(rand);
 617
 618         VERBOSE_TOROUT_STRING("lock_torture_reader task started");
 619         set_user_nice(current, MAX_NICE);
 620
 621         do {
 622                 if ((torture_random(&rand) & 0xfffff) == 0)
 623                         schedule_timeout_uninterruptible(1);
 624
 625                 cxt.cur_ops->readlock();
 626                 lock_is_read_held = 1;
 627                 if (WARN_ON_ONCE(lock_is_write_held))
 628                         lrsp->n_lock_fail++; /* rare, but... */
 629
 630                 lrsp->n_lock_acquired++;
 631                 cxt.cur_ops->read_delay(&rand);
 632                 lock_is_read_held = 0;
 633                 cxt.cur_ops->readunlock();
 634
 635                 stutter_wait("lock_torture_reader");
 636         } while (!torture_must_stop());
 637         torture_kthread_stopping("lock_torture_reader");
 638         return 0;
 639 }
 640
 641 /*
 642  * Create an lock-torture-statistics message in the specified buffer.
 643  */
 644 static void __torture_print_stats(char *page,
 645                                   struct lock_stress_stats *statp, bool write)
 646 {
 647         bool fail = 0;
 648         int i, n_stress;
 649         long max = 0;
 650         long min = statp[0].n_lock_acquired;
 651         long long sum = 0;
 652
 653         n_stress = write ? cxt.nrealwriters_stress : cxt.nrealreaders_stress;
 654         for (i = 0; i < n_stress; i++) {
 655                 if (statp[i].n_lock_fail)
 656                         fail = true;
 657                 sum += statp[i].n_lock_acquired;
 658                 if (max < statp[i].n_lock_acquired)
 659                         max = statp[i].n_lock_acquired;
 660                 if (min > statp[i].n_lock_acquired)
 661                         min = statp[i].n_lock_acquired;
 662         }
 663         page += sprintf(page,
 664                         "%s:  Total: %lld  Max/Min: %ld/%ld %s  Fail: %d %s\n",
 665                         write ? "Writes" : "Reads ",
 666                         sum, max, min, max / 2 > min ? "???" : "",
 667                         fail, fail ? "!!!" : "");
 668         if (fail)
 669                 atomic_inc(&cxt.n_lock_torture_errors);
 670 }
 671
 672 /*
 673  * Print torture statistics.  Caller must ensure that there is only one
 674  * call to this function at a given time!!!  This is normally accomplished
 675  * by relying on the module system to only have one copy of the module
 676  * loaded, and then by giving the lock_torture_stats kthread full control
 677  * (or the init/cleanup functions when lock_torture_stats thread is not
 678  * running).
 679  */
 680 static void lock_torture_stats_print(void)
 681 {
 682         int size = cxt.nrealwriters_stress * 200 + 8192;
 683         char *buf;
 684
 685         if (cxt.cur_ops->readlock)
 686                 size += cxt.nrealreaders_stress * 200 + 8192;
 687
 688         buf = kmalloc(size, GFP_KERNEL);
 689         if (!buf) {
 690                 pr_err("lock_torture_stats_print: Out of memory, need: %d",
 691                        size);
 692                 return;
 693         }
 694
 695         __torture_print_stats(buf, cxt.lwsa, true);
 696         pr_alert("%s", buf);
 697         kfree(buf);
 698
 699         if (cxt.cur_ops->readlock) {
 700                 buf = kmalloc(size, GFP_KERNEL);
 701                 if (!buf) {
 702                         pr_err("lock_torture_stats_print: Out of memory, need: %d",
 703                                size);
 704                         return;
 705                 }
 706
 707                 __torture_print_stats(buf, cxt.lrsa, false);
 708                 pr_alert("%s", buf);
 709                 kfree(buf);
 710         }
 711 }
 712
 713 /*
 714  * Periodically prints torture statistics, if periodic statistics printing
 715  * was specified via the stat_interval module parameter.
 716  *
 717  * No need to worry about fullstop here, since this one doesn't reference
 718  * volatile state or register callbacks.
 719  */
 720 static int lock_torture_stats(void *arg)
 721 {
 722         VERBOSE_TOROUT_STRING("lock_torture_stats task started");
 723         do {
 724                 schedule_timeout_interruptible(stat_interval * HZ);
 725                 lock_torture_stats_print();
 726                 torture_shutdown_absorb("lock_torture_stats");
 727         } while (!torture_must_stop());
 728         torture_kthread_stopping("lock_torture_stats");
 729         return 0;
 730 }
 731
 732 static inline void
 733 lock_torture_print_module_parms(struct lock_torture_ops *cur_ops,
 734                                 const char *tag)
 735 {
 736         pr_alert("%s" TORTURE_FLAG
 737                  "--- %s%s: nwriters_stress=%d nreaders_stress=%d stat_interval=%d verbose=%d shuffle_interval=%d stutter=%d shutdown_secs=%d onoff_interval=%d onoff_holdoff=%d\n",
 738                  torture_type, tag, cxt.debug_lock ? " [debug]": "",
 739                  cxt.nrealwriters_stress, cxt.nrealreaders_stress, stat_interval,
 740                  verbose, shuffle_interval, stutter, shutdown_secs,
 741                  onoff_interval, onoff_holdoff);
 742 }
 743
 744 static void lock_torture_cleanup(void)
 745 {
 746         int i;
 747
 748         if (torture_cleanup_begin())
 749                 return;
 750
 751         if (writer_tasks) {
 752                 for (i = 0; i < cxt.nrealwriters_stress; i++)
 753                         torture_stop_kthread(lock_torture_writer,
 754                                              writer_tasks[i]);
 755                 kfree(writer_tasks);
 756                 writer_tasks = NULL;
 757         }
 758
 759         if (reader_tasks) {
 760                 for (i = 0; i < cxt.nrealreaders_stress; i++)
 761                         torture_stop_kthread(lock_torture_reader,
 762                                              reader_tasks[i]);
 763                 kfree(reader_tasks);
 764                 reader_tasks = NULL;
 765         }
 766
 767         torture_stop_kthread(lock_torture_stats, stats_task);
 768         lock_torture_stats_print();  /* -After- the stats thread is stopped! */
 769
 770         if (atomic_read(&cxt.n_lock_torture_errors))
 771                 lock_torture_print_module_parms(cxt.cur_ops,
 772                                                 "End of test: FAILURE");
 773         else if (torture_onoff_failures())
 774                 lock_torture_print_module_parms(cxt.cur_ops,
 775                                                 "End of test: LOCK_HOTPLUG");
 776         else
 777                 lock_torture_print_module_parms(cxt.cur_ops,
 778                                                 "End of test: SUCCESS");
 779         kfree(cxt.lwsa);
 780         kfree(cxt.lrsa);
 781         torture_cleanup_end();
 782 }
 783
 784 static int __init lock_torture_init(void)
 785 {
 786         int i, j;
 787         int firsterr = 0;
 788         static struct lock_torture_ops *torture_ops[] = {
 789                 &lock_busted_ops,
 790                 &spin_lock_ops, &spin_lock_irq_ops,
 791                 &rw_lock_ops, &rw_lock_irq_ops,
 792                 &mutex_lock_ops,
 793 #ifdef CONFIG_RT_MUTEXES
 794                 &rtmutex_lock_ops,
 795 #endif
 796                 &rwsem_lock_ops,
 797                 &percpu_rwsem_lock_ops,
 798         };
 799
 800         if (!torture_init_begin(torture_type, verbose, &torture_runnable))
 801                 return -EBUSY;
 802
 803         /* Process args and tell the world that the torturer is on the job. */
 804         for (i = 0; i < ARRAY_SIZE(torture_ops); i++) {
 805                 cxt.cur_ops = torture_ops[i];
 806                 if (strcmp(torture_type, cxt.cur_ops->name) == 0)
 807                         break;
 808         }
 809         if (i == ARRAY_SIZE(torture_ops)) {
 810                 pr_alert("lock-torture: invalid torture type: \"%s\"\n",
 811                          torture_type);
 812                 pr_alert("lock-torture types:");
 813                 for (i = 0; i < ARRAY_SIZE(torture_ops); i++)
 814                         pr_alert(" %s", torture_ops[i]->name);
 815                 pr_alert("\n");
 816                 firsterr = -EINVAL;
 817                 goto unwind;
 818         }
 819         if (cxt.cur_ops->init)
 820                 cxt.cur_ops->init();
 821
 822         if (nwriters_stress >= 0)
 823                 cxt.nrealwriters_stress = nwriters_stress;
 824         else
 825                 cxt.nrealwriters_stress = 2 * num_online_cpus();
 826
 827 #ifdef CONFIG_DEBUG_MUTEXES
 828         if (strncmp(torture_type, "mutex", 5) == 0)
 829                 cxt.debug_lock = true;
 830 #endif
 831 #ifdef CONFIG_DEBUG_RT_MUTEXES
 832         if (strncmp(torture_type, "rtmutex", 7) == 0)
 833                 cxt.debug_lock = true;
 834 #endif
 835 #ifdef CONFIG_DEBUG_SPINLOCK
 836         if ((strncmp(torture_type, "spin", 4) == 0) ||
 837             (strncmp(torture_type, "rw_lock", 7) == 0))
 838                 cxt.debug_lock = true;
 839 #endif
 840
 841         /* Initialize the statistics so that each run gets its own numbers. */
 842
 843         lock_is_write_held = 0;
 844         cxt.lwsa = kmalloc(sizeof(*cxt.lwsa) * cxt.nrealwriters_stress, GFP_KERNEL);
 845         if (cxt.lwsa == NULL) {
 846                 VERBOSE_TOROUT_STRING("cxt.lwsa: Out of memory");
 847                 firsterr = -ENOMEM;
 848                 goto unwind;
 849         }
 850         for (i = 0; i < cxt.nrealwriters_stress; i++) {
 851                 cxt.lwsa[i].n_lock_fail = 0;
 852                 cxt.lwsa[i].n_lock_acquired = 0;
 853         }
 854
 855         if (cxt.cur_ops->readlock) {
 856                 if (nreaders_stress >= 0)
 857                         cxt.nrealreaders_stress = nreaders_stress;
 858                 else {
 859                         /*
 860                          * By default distribute evenly the number of
 861                          * readers and writers. We still run the same number
 862                          * of threads as the writer-only locks default.
 863                          */
 864                         if (nwriters_stress < 0) /* user doesn't care */
 865                                 cxt.nrealwriters_stress = num_online_cpus();
 866                         cxt.nrealreaders_stress = cxt.nrealwriters_stress;
 867                 }
 868
 869                 lock_is_read_held = 0;
 870                 cxt.lrsa = kmalloc(sizeof(*cxt.lrsa) * cxt.nrealreaders_stress, GFP_KERNEL);
 871                 if (cxt.lrsa == NULL) {
 872                         VERBOSE_TOROUT_STRING("cxt.lrsa: Out of memory");
 873                         firsterr = -ENOMEM;
 874                         kfree(cxt.lwsa);
 875                         goto unwind;
 876                 }
 877
 878                 for (i = 0; i < cxt.nrealreaders_stress; i++) {
 879                         cxt.lrsa[i].n_lock_fail = 0;
 880                         cxt.lrsa[i].n_lock_acquired = 0;
 881                 }
 882         }
 883         lock_torture_print_module_parms(cxt.cur_ops, "Start of test");
 884
 885         /* Prepare torture context. */
 886         if (onoff_interval > 0) {
 887                 firsterr = torture_onoff_init(onoff_holdoff * HZ,
 888                                               onoff_interval * HZ);
 889                 if (firsterr)
 890                         goto unwind;
 891         }
 892         if (shuffle_interval > 0) {
 893                 firsterr = torture_shuffle_init(shuffle_interval);
 894                 if (firsterr)
 895                         goto unwind;
 896         }
 897         if (shutdown_secs > 0) {
 898                 firsterr = torture_shutdown_init(shutdown_secs,
 899                                                  lock_torture_cleanup);
 900                 if (firsterr)
 901                         goto unwind;
 902         }
 903         if (stutter > 0) {
 904                 firsterr = torture_stutter_init(stutter);
 905                 if (firsterr)
 906                         goto unwind;
 907         }
 908
 909         writer_tasks = kzalloc(cxt.nrealwriters_stress * sizeof(writer_tasks[0]),
 910                                GFP_KERNEL);
 911         if (writer_tasks == NULL) {
 912                 VERBOSE_TOROUT_ERRSTRING("writer_tasks: Out of memory");
 913                 firsterr = -ENOMEM;
 914                 goto unwind;
 915         }
 916
 917         if (cxt.cur_ops->readlock) {
 918                 reader_tasks = kzalloc(cxt.nrealreaders_stress * sizeof(reader_tasks[0]),
 919                                        GFP_KERNEL);
 920                 if (reader_tasks == NULL) {
 921                         VERBOSE_TOROUT_ERRSTRING("reader_tasks: Out of memory");
 922                         kfree(writer_tasks);
 923                         writer_tasks = NULL;
 924                         firsterr = -ENOMEM;
 925                         goto unwind;
 926                 }
 927         }
 928
 929         /*
 930          * Create the kthreads and start torturing (oh, those poor little locks).
 931          *
 932          * TODO: Note that we interleave writers with readers, giving writers a
 933          * slight advantage, by creating its kthread first. This can be modified
 934          * for very specific needs, or even let the user choose the policy, if
 935          * ever wanted.
 936          */
 937         for (i = 0, j = 0; i < cxt.nrealwriters_stress ||
 938                     j < cxt.nrealreaders_stress; i++, j++) {
 939                 if (i >= cxt.nrealwriters_stress)
 940                         goto create_reader;
 941
 942                 /* Create writer. */
 943                 firsterr = torture_create_kthread(lock_torture_writer, &cxt.lwsa[i],
 944                                                   writer_tasks[i]);
 945                 if (firsterr)
 946                         goto unwind;
 947
 948         create_reader:
 949                 if (cxt.cur_ops->readlock == NULL || (j >= cxt.nrealreaders_stress))
 950                         continue;
 951                 /* Create reader. */
 952                 firsterr = torture_create_kthread(lock_torture_reader, &cxt.lrsa[j],
 953                                                   reader_tasks[j]);
 954                 if (firsterr)
 955                         goto unwind;
 956         }
 957         if (stat_interval > 0) {
 958                 firsterr = torture_create_kthread(lock_torture_stats, NULL,
 959                                                   stats_task);
 960                 if (firsterr)
 961                         goto unwind;
 962         }
 963         torture_init_end();
 964         return 0;
 965
 966 unwind:
 967         torture_init_end();
 968         lock_torture_cleanup();
 969         return firsterr;
 970 }
 971
 972 module_init(lock_torture_init);
 973 module_exit(lock_torture_cleanup);