kernel/locking/locktorture.c

   1 /*
   2  * Module-based torture test facility for locking
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License as published by
   6  * the Free Software Foundation; either version 2 of the License, or
   7  * (at your option) any later version.
   8  *
   9  * This program is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  * GNU General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License
  15  * along with this program; if not, you can access it online at
  16  * http://www.gnu.org/licenses/gpl-2.0.html.
  17  *
  18  * Copyright (C) IBM Corporation, 2014
  19  *
  20  * Authors: Paul E. McKenney <paulmck@us.ibm.com>
  21  *          Davidlohr Bueso <dave@stgolabs.net>
  22  *      Based on kernel/rcu/torture.c.
  23  */
  24 #include <linux/kernel.h>
  25 #include <linux/module.h>
  26 #include <linux/kthread.h>
  27 #include <linux/sched/rt.h>
  28 #include <linux/spinlock.h>
  29 #include <linux/rwlock.h>
  30 #include <linux/mutex.h>
  31 #include <linux/rwsem.h>
  32 #include <linux/smp.h>
  33 #include <linux/interrupt.h>
  34 #include <linux/sched.h>
  35 #include <linux/atomic.h>
  36 #include <linux/moduleparam.h>
  37 #include <linux/delay.h>
  38 #include <linux/slab.h>
  39 #include <linux/percpu-rwsem.h>
  40 #include <linux/torture.h>
  41
  42 MODULE_LICENSE("GPL");
  43 MODULE_AUTHOR("Paul E. McKenney <paulmck@us.ibm.com>");
  44
  45 torture_param(int, nwriters_stress, -1,
  46              "Number of write-locking stress-test threads");
  47 torture_param(int, nreaders_stress, -1,
  48              "Number of read-locking stress-test threads");
  49 torture_param(int, onoff_holdoff, 0, "Time after boot before CPU hotplugs (s)");
  50 torture_param(int, onoff_interval, 0,
  51              "Time between CPU hotplugs (s), 0=disable");
  52 torture_param(int, shuffle_interval, 3,
  53              "Number of jiffies between shuffles, 0=disable");
  54 torture_param(int, shutdown_secs, 0, "Shutdown time (j), <= zero to disable.");
  55 torture_param(int, stat_interval, 60,
  56              "Number of seconds between stats printk()s");
  57 torture_param(int, stutter, 5, "Number of jiffies to run/halt test, 0=disable");
  58 torture_param(bool, verbose, true,
  59              "Enable verbose debugging printk()s");
  60
  61 static char *torture_type = "spin_lock";
  62 module_param(torture_type, charp, 0444);
  63 MODULE_PARM_DESC(torture_type,
  64                  "Type of lock to torture (spin_lock, spin_lock_irq, mutex_lock, ...)");
  65
  66 static struct task_struct *stats_task;
  67 static struct task_struct **writer_tasks;
  68 static struct task_struct **reader_tasks;
  69
  70 static bool lock_is_write_held;
  71 static bool lock_is_read_held;
  72
  73 struct lock_stress_stats {
  74         long n_lock_fail;
  75         long n_lock_acquired;
  76 };
  77
  78 int torture_runnable = IS_ENABLED(MODULE);
  79 module_param(torture_runnable, int, 0444);
  80 MODULE_PARM_DESC(torture_runnable, "Start locktorture at module init");
  81
  82 /* Forward reference. */
  83 static void lock_torture_cleanup(void);
  84
  85 /*
  86  * Operations vector for selecting different types of tests.
  87  */
  88 struct lock_torture_ops {
  89         void (*init)(void);
  90         int (*writelock)(void);
  91         void (*write_delay)(struct torture_random_state *trsp);
  92         void (*task_boost)(struct torture_random_state *trsp);
  93         void (*writeunlock)(void);
  94         int (*readlock)(void);
  95         void (*read_delay)(struct torture_random_state *trsp);
  96         void (*readunlock)(void);
  97
  98         unsigned long flags; /* for irq spinlocks */
  99         const char *name;
 100 };
 101
 102 struct lock_torture_cxt {
 103         int nrealwriters_stress;
 104         int nrealreaders_stress;
 105         bool debug_lock;
 106         atomic_t n_lock_torture_errors;
 107         struct lock_torture_ops *cur_ops;
 108         struct lock_stress_stats *lwsa; /* writer statistics */
 109         struct lock_stress_stats *lrsa; /* reader statistics */
 110 };
 111 static struct lock_torture_cxt cxt = { 0, 0, false,
 112                                        ATOMIC_INIT(0),
 113                                        NULL, NULL};
 114 /*
 115  * Definitions for lock torture testing.
 116  */
 117
 118 static int torture_lock_busted_write_lock(void)
 119 {
 120         return 0;  /* BUGGY, do not use in real life!!! */
 121 }
 122
 123 static void torture_lock_busted_write_delay(struct torture_random_state *trsp)
 124 {
 125         const unsigned long longdelay_ms = 100;
 126
 127         /* We want a long delay occasionally to force massive contention.  */
 128         if (!(torture_random(trsp) %
 129               (cxt.nrealwriters_stress * 2000 * longdelay_ms)))
 130                 mdelay(longdelay_ms);
 131 #ifdef CONFIG_PREEMPT
 132         if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000)))
 133                 preempt_schedule();  /* Allow test to be preempted. */
 134 #endif
 135 }
 136
 137 static void torture_lock_busted_write_unlock(void)
 138 {
 139           /* BUGGY, do not use in real life!!! */
 140 }
 141
 142 static void torture_boost_dummy(struct torture_random_state *trsp)
 143 {
 144         /* Only rtmutexes care about priority */
 145 }
 146
 147 static struct lock_torture_ops lock_busted_ops = {
 148         .writelock      = torture_lock_busted_write_lock,
 149         .write_delay    = torture_lock_busted_write_delay,
 150         .task_boost     = torture_boost_dummy,
 151         .writeunlock    = torture_lock_busted_write_unlock,
 152         .readlock       = NULL,
 153         .read_delay     = NULL,
 154         .readunlock     = NULL,
 155         .name           = "lock_busted"
 156 };
 157
 158 static DEFINE_SPINLOCK(torture_spinlock);
 159
 160 static int torture_spin_lock_write_lock(void) __acquires(torture_spinlock)
 161 {
 162         spin_lock(&torture_spinlock);
 163         return 0;
 164 }
 165
 166 static void torture_spin_lock_write_delay(struct torture_random_state *trsp)
 167 {
 168         const unsigned long shortdelay_us = 2;
 169         const unsigned long longdelay_ms = 100;
 170
 171         /* We want a short delay mostly to emulate likely code, and
 172          * we want a long delay occasionally to force massive contention.
 173          */
 174         if (!(torture_random(trsp) %
 175               (cxt.nrealwriters_stress * 2000 * longdelay_ms)))
 176                 mdelay(longdelay_ms);
 177         if (!(torture_random(trsp) %
 178               (cxt.nrealwriters_stress * 2 * shortdelay_us)))
 179                 udelay(shortdelay_us);
 180 #ifdef CONFIG_PREEMPT
 181         if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000)))
 182                 preempt_schedule();  /* Allow test to be preempted. */
 183 #endif
 184 }
 185
 186 static void torture_spin_lock_write_unlock(void) __releases(torture_spinlock)
 187 {
 188         spin_unlock(&torture_spinlock);
 189 }
 190
 191 static struct lock_torture_ops spin_lock_ops = {
 192         .writelock      = torture_spin_lock_write_lock,
 193         .write_delay    = torture_spin_lock_write_delay,
 194         .task_boost     = torture_boost_dummy,
 195         .writeunlock    = torture_spin_lock_write_unlock,
 196         .readlock       = NULL,
 197         .read_delay     = NULL,
 198         .readunlock     = NULL,
 199         .name           = "spin_lock"
 200 };
 201
 202 static int torture_spin_lock_write_lock_irq(void)
 203 __acquires(torture_spinlock)
 204 {
 205         unsigned long flags;
 206
 207         spin_lock_irqsave(&torture_spinlock, flags);
 208         cxt.cur_ops->flags = flags;
 209         return 0;
 210 }
 211
 212 static void torture_lock_spin_write_unlock_irq(void)
 213 __releases(torture_spinlock)
 214 {
 215         spin_unlock_irqrestore(&torture_spinlock, cxt.cur_ops->flags);
 216 }
 217
 218 static struct lock_torture_ops spin_lock_irq_ops = {
 219         .writelock      = torture_spin_lock_write_lock_irq,
 220         .write_delay    = torture_spin_lock_write_delay,
 221         .task_boost     = torture_boost_dummy,
 222         .writeunlock    = torture_lock_spin_write_unlock_irq,
 223         .readlock       = NULL,
 224         .read_delay     = NULL,
 225         .readunlock     = NULL,
 226         .name           = "spin_lock_irq"
 227 };
 228
 229 static DEFINE_RWLOCK(torture_rwlock);
 230
 231 static int torture_rwlock_write_lock(void) __acquires(torture_rwlock)
 232 {
 233         write_lock(&torture_rwlock);
 234         return 0;
 235 }
 236
 237 static void torture_rwlock_write_delay(struct torture_random_state *trsp)
 238 {
 239         const unsigned long shortdelay_us = 2;
 240         const unsigned long longdelay_ms = 100;
 241
 242         /* We want a short delay mostly to emulate likely code, and
 243          * we want a long delay occasionally to force massive contention.
 244          */
 245         if (!(torture_random(trsp) %
 246               (cxt.nrealwriters_stress * 2000 * longdelay_ms)))
 247                 mdelay(longdelay_ms);
 248         else
 249                 udelay(shortdelay_us);
 250 }
 251
 252 static void torture_rwlock_write_unlock(void) __releases(torture_rwlock)
 253 {
 254         write_unlock(&torture_rwlock);
 255 }
 256
 257 static int torture_rwlock_read_lock(void) __acquires(torture_rwlock)
 258 {
 259         read_lock(&torture_rwlock);
 260         return 0;
 261 }
 262
 263 static void torture_rwlock_read_delay(struct torture_random_state *trsp)
 264 {
 265         const unsigned long shortdelay_us = 10;
 266         const unsigned long longdelay_ms = 100;
 267
 268         /* We want a short delay mostly to emulate likely code, and
 269          * we want a long delay occasionally to force massive contention.
 270          */
 271         if (!(torture_random(trsp) %
 272               (cxt.nrealreaders_stress * 2000 * longdelay_ms)))
 273                 mdelay(longdelay_ms);
 274         else
 275                 udelay(shortdelay_us);
 276 }
 277
 278 static void torture_rwlock_read_unlock(void) __releases(torture_rwlock)
 279 {
 280         read_unlock(&torture_rwlock);
 281 }
 282
 283 static struct lock_torture_ops rw_lock_ops = {
 284         .writelock      = torture_rwlock_write_lock,
 285         .write_delay    = torture_rwlock_write_delay,
 286         .task_boost     = torture_boost_dummy,
 287         .writeunlock    = torture_rwlock_write_unlock,
 288         .readlock       = torture_rwlock_read_lock,
 289         .read_delay     = torture_rwlock_read_delay,
 290         .readunlock     = torture_rwlock_read_unlock,
 291         .name           = "rw_lock"
 292 };
 293
 294 static int torture_rwlock_write_lock_irq(void) __acquires(torture_rwlock)
 295 {
 296         unsigned long flags;
 297
 298         write_lock_irqsave(&torture_rwlock, flags);
 299         cxt.cur_ops->flags = flags;
 300         return 0;
 301 }
 302
 303 static void torture_rwlock_write_unlock_irq(void)
 304 __releases(torture_rwlock)
 305 {
 306         write_unlock_irqrestore(&torture_rwlock, cxt.cur_ops->flags);
 307 }
 308
 309 static int torture_rwlock_read_lock_irq(void) __acquires(torture_rwlock)
 310 {
 311         unsigned long flags;
 312
 313         read_lock_irqsave(&torture_rwlock, flags);
 314         cxt.cur_ops->flags = flags;
 315         return 0;
 316 }
 317
 318 static void torture_rwlock_read_unlock_irq(void)
 319 __releases(torture_rwlock)
 320 {
 321         read_unlock_irqrestore(&torture_rwlock, cxt.cur_ops->flags);
 322 }
 323
 324 static struct lock_torture_ops rw_lock_irq_ops = {
 325         .writelock      = torture_rwlock_write_lock_irq,
 326         .write_delay    = torture_rwlock_write_delay,
 327         .task_boost     = torture_boost_dummy,
 328         .writeunlock    = torture_rwlock_write_unlock_irq,
 329         .readlock       = torture_rwlock_read_lock_irq,
 330         .read_delay     = torture_rwlock_read_delay,
 331         .readunlock     = torture_rwlock_read_unlock_irq,
 332         .name           = "rw_lock_irq"
 333 };
 334
 335 static DEFINE_MUTEX(torture_mutex);
 336
 337 static int torture_mutex_lock(void) __acquires(torture_mutex)
 338 {
 339         mutex_lock(&torture_mutex);
 340         return 0;
 341 }
 342
 343 static void torture_mutex_delay(struct torture_random_state *trsp)
 344 {
 345         const unsigned long longdelay_ms = 100;
 346
 347         /* We want a long delay occasionally to force massive contention.  */
 348         if (!(torture_random(trsp) %
 349               (cxt.nrealwriters_stress * 2000 * longdelay_ms)))
 350                 mdelay(longdelay_ms * 5);
 351         else
 352                 mdelay(longdelay_ms / 5);
 353 #ifdef CONFIG_PREEMPT
 354         if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000)))
 355                 preempt_schedule();  /* Allow test to be preempted. */
 356 #endif
 357 }
 358
 359 static void torture_mutex_unlock(void) __releases(torture_mutex)
 360 {
 361         mutex_unlock(&torture_mutex);
 362 }
 363
 364 static struct lock_torture_ops mutex_lock_ops = {
 365         .writelock      = torture_mutex_lock,
 366         .write_delay    = torture_mutex_delay,
 367         .task_boost     = torture_boost_dummy,
 368         .writeunlock    = torture_mutex_unlock,
 369         .readlock       = NULL,
 370         .read_delay     = NULL,
 371         .readunlock     = NULL,
 372         .name           = "mutex_lock"
 373 };
 374
 375 #ifdef CONFIG_RT_MUTEXES
 376 static DEFINE_RT_MUTEX(torture_rtmutex);
 377
 378 static int torture_rtmutex_lock(void) __acquires(torture_rtmutex)
 379 {
 380         rt_mutex_lock(&torture_rtmutex);
 381         return 0;
 382 }
 383
 384 static void torture_rtmutex_boost(struct torture_random_state *trsp)
 385 {
 386         int policy;
 387         struct sched_param param;
 388         const unsigned int factor = 50000; /* yes, quite arbitrary */
 389
 390         if (!rt_task(current)) {
 391                 /*
 392                  * Boost priority once every ~50k operations. When the
 393                  * task tries to take the lock, the rtmutex it will account
 394                  * for the new priority, and do any corresponding pi-dance.
 395                  */
 396                 if (trsp && !(torture_random(trsp) %
 397                               (cxt.nrealwriters_stress * factor))) {
 398                         policy = SCHED_FIFO;
 399                         param.sched_priority = MAX_RT_PRIO - 1;
 400                 } else /* common case, do nothing */
 401                         return;
 402         } else {
 403                 /*
 404                  * The task will remain boosted for another ~500k operations,
 405                  * then restored back to its original prio, and so forth.
 406                  *
 407                  * When @trsp is nil, we want to force-reset the task for
 408                  * stopping the kthread.
 409                  */
 410                 if (!trsp || !(torture_random(trsp) %
 411                                (cxt.nrealwriters_stress * factor * 2))) {
 412                         policy = SCHED_NORMAL;
 413                         param.sched_priority = 0;
 414                 } else /* common case, do nothing */
 415                         return;
 416         }
 417
 418         sched_setscheduler_nocheck(current, policy, &param);
 419 }
 420
 421 static void torture_rtmutex_delay(struct torture_random_state *trsp)
 422 {
 423         const unsigned long shortdelay_us = 2;
 424         const unsigned long longdelay_ms = 100;
 425
 426         /*
 427          * We want a short delay mostly to emulate likely code, and
 428          * we want a long delay occasionally to force massive contention.
 429          */
 430         if (!(torture_random(trsp) %
 431               (cxt.nrealwriters_stress * 2000 * longdelay_ms)))
 432                 mdelay(longdelay_ms);
 433         if (!(torture_random(trsp) %
 434               (cxt.nrealwriters_stress * 2 * shortdelay_us)))
 435                 udelay(shortdelay_us);
 436 #ifdef CONFIG_PREEMPT
 437         if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000)))
 438                 preempt_schedule();  /* Allow test to be preempted. */
 439 #endif
 440 }
 441
 442 static void torture_rtmutex_unlock(void) __releases(torture_rtmutex)
 443 {
 444         rt_mutex_unlock(&torture_rtmutex);
 445 }
 446
 447 static struct lock_torture_ops rtmutex_lock_ops = {
 448         .writelock      = torture_rtmutex_lock,
 449         .write_delay    = torture_rtmutex_delay,
 450         .task_boost     = torture_rtmutex_boost,
 451         .writeunlock    = torture_rtmutex_unlock,
 452         .readlock       = NULL,
 453         .read_delay     = NULL,
 454         .readunlock     = NULL,
 455         .name           = "rtmutex_lock"
 456 };
 457 #endif
 458
 459 static DECLARE_RWSEM(torture_rwsem);
 460 static int torture_rwsem_down_write(void) __acquires(torture_rwsem)
 461 {
 462         down_write(&torture_rwsem);
 463         return 0;
 464 }
 465
 466 static void torture_rwsem_write_delay(struct torture_random_state *trsp)
 467 {
 468         const unsigned long longdelay_ms = 100;
 469
 470         /* We want a long delay occasionally to force massive contention.  */
 471         if (!(torture_random(trsp) %
 472               (cxt.nrealwriters_stress * 2000 * longdelay_ms)))
 473                 mdelay(longdelay_ms * 10);
 474         else
 475                 mdelay(longdelay_ms / 10);
 476 #ifdef CONFIG_PREEMPT
 477         if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000)))
 478                 preempt_schedule();  /* Allow test to be preempted. */
 479 #endif
 480 }
 481
 482 static void torture_rwsem_up_write(void) __releases(torture_rwsem)
 483 {
 484         up_write(&torture_rwsem);
 485 }
 486
 487 static int torture_rwsem_down_read(void) __acquires(torture_rwsem)
 488 {
 489         down_read(&torture_rwsem);
 490         return 0;
 491 }
 492
 493 static void torture_rwsem_read_delay(struct torture_random_state *trsp)
 494 {
 495         const unsigned long longdelay_ms = 100;
 496
 497         /* We want a long delay occasionally to force massive contention.  */
 498         if (!(torture_random(trsp) %
 499               (cxt.nrealwriters_stress * 2000 * longdelay_ms)))
 500                 mdelay(longdelay_ms * 2);
 501         else
 502                 mdelay(longdelay_ms / 2);
 503 #ifdef CONFIG_PREEMPT
 504         if (!(torture_random(trsp) % (cxt.nrealreaders_stress * 20000)))
 505                 preempt_schedule();  /* Allow test to be preempted. */
 506 #endif
 507 }
 508
 509 static void torture_rwsem_up_read(void) __releases(torture_rwsem)
 510 {
 511         up_read(&torture_rwsem);
 512 }
 513
 514 static struct lock_torture_ops rwsem_lock_ops = {
 515         .writelock      = torture_rwsem_down_write,
 516         .write_delay    = torture_rwsem_write_delay,
 517         .task_boost     = torture_boost_dummy,
 518         .writeunlock    = torture_rwsem_up_write,
 519         .readlock       = torture_rwsem_down_read,
 520         .read_delay     = torture_rwsem_read_delay,
 521         .readunlock     = torture_rwsem_up_read,
 522         .name           = "rwsem_lock"
 523 };
 524
 525 #include <linux/percpu-rwsem.h>
 526 static struct percpu_rw_semaphore pcpu_rwsem;
 527
 528 void torture_percpu_rwsem_init(void)
 529 {
 530         BUG_ON(percpu_init_rwsem(&pcpu_rwsem));
 531 }
 532
 533 static int torture_percpu_rwsem_down_write(void) __acquires(pcpu_rwsem)
 534 {
 535         percpu_down_write(&pcpu_rwsem);
 536         return 0;
 537 }
 538
 539 static void torture_percpu_rwsem_up_write(void) __releases(pcpu_rwsem)
 540 {
 541         percpu_up_write(&pcpu_rwsem);
 542 }
 543
 544 static int torture_percpu_rwsem_down_read(void) __acquires(pcpu_rwsem)
 545 {
 546         percpu_down_read(&pcpu_rwsem);
 547         return 0;
 548 }
 549
 550 static void torture_percpu_rwsem_up_read(void) __releases(pcpu_rwsem)
 551 {
 552         percpu_up_read(&pcpu_rwsem);
 553 }
 554
 555 static struct lock_torture_ops percpu_rwsem_lock_ops = {
 556         .init           = torture_percpu_rwsem_init,
 557         .writelock      = torture_percpu_rwsem_down_write,
 558         .write_delay    = torture_rwsem_write_delay,
 559         .task_boost     = torture_boost_dummy,
 560         .writeunlock    = torture_percpu_rwsem_up_write,
 561         .readlock       = torture_percpu_rwsem_down_read,
 562         .read_delay     = torture_rwsem_read_delay,
 563         .readunlock     = torture_percpu_rwsem_up_read,
 564         .name           = "percpu_rwsem_lock"
 565 };
 566
 567 /*
 568  * Lock torture writer kthread.  Repeatedly acquires and releases
 569  * the lock, checking for duplicate acquisitions.
 570  */
 571 static int lock_torture_writer(void *arg)
 572 {
 573         struct lock_stress_stats *lwsp = arg;
 574         static DEFINE_TORTURE_RANDOM(rand);
 575
 576         VERBOSE_TOROUT_STRING("lock_torture_writer task started");
 577         set_user_nice(current, MAX_NICE);
 578
 579         do {
 580                 if ((torture_random(&rand) & 0xfffff) == 0)
 581                         schedule_timeout_uninterruptible(1);
 582
 583                 cxt.cur_ops->task_boost(&rand);
 584                 cxt.cur_ops->writelock();
 585                 if (WARN_ON_ONCE(lock_is_write_held))
 586                         lwsp->n_lock_fail++;
 587                 lock_is_write_held = 1;
 588                 if (WARN_ON_ONCE(lock_is_read_held))
 589                         lwsp->n_lock_fail++; /* rare, but... */
 590
 591                 lwsp->n_lock_acquired++;
 592                 cxt.cur_ops->write_delay(&rand);
 593                 lock_is_write_held = 0;
 594                 cxt.cur_ops->writeunlock();
 595
 596                 stutter_wait("lock_torture_writer");
 597         } while (!torture_must_stop());
 598
 599         cxt.cur_ops->task_boost(NULL); /* reset prio */
 600         torture_kthread_stopping("lock_torture_writer");
 601         return 0;
 602 }
 603
 604 /*
 605  * Lock torture reader kthread.  Repeatedly acquires and releases
 606  * the reader lock.
 607  */
 608 static int lock_torture_reader(void *arg)
 609 {
 610         struct lock_stress_stats *lrsp = arg;
 611         static DEFINE_TORTURE_RANDOM(rand);
 612
 613         VERBOSE_TOROUT_STRING("lock_torture_reader task started");
 614         set_user_nice(current, MAX_NICE);
 615
 616         do {
 617                 if ((torture_random(&rand) & 0xfffff) == 0)
 618                         schedule_timeout_uninterruptible(1);
 619
 620                 cxt.cur_ops->readlock();
 621                 lock_is_read_held = 1;
 622                 if (WARN_ON_ONCE(lock_is_write_held))
 623                         lrsp->n_lock_fail++; /* rare, but... */
 624
 625                 lrsp->n_lock_acquired++;
 626                 cxt.cur_ops->read_delay(&rand);
 627                 lock_is_read_held = 0;
 628                 cxt.cur_ops->readunlock();
 629
 630                 stutter_wait("lock_torture_reader");
 631         } while (!torture_must_stop());
 632         torture_kthread_stopping("lock_torture_reader");
 633         return 0;
 634 }
 635
 636 /*
 637  * Create an lock-torture-statistics message in the specified buffer.
 638  */
 639 static void __torture_print_stats(char *page,
 640                                   struct lock_stress_stats *statp, bool write)
 641 {
 642         bool fail = 0;
 643         int i, n_stress;
 644         long max = 0, min = statp ? statp[0].n_lock_acquired : 0;
 645         long long sum = 0;
 646
 647         n_stress = write ? cxt.nrealwriters_stress : cxt.nrealreaders_stress;
 648         for (i = 0; i < n_stress; i++) {
 649                 if (statp[i].n_lock_fail)
 650                         fail = true;
 651                 sum += statp[i].n_lock_acquired;
 652                 if (max < statp[i].n_lock_acquired)
 653                         max = statp[i].n_lock_acquired;
 654                 if (min > statp[i].n_lock_acquired)
 655                         min = statp[i].n_lock_acquired;
 656         }
 657         page += sprintf(page,
 658                         "%s:  Total: %lld  Max/Min: %ld/%ld %s  Fail: %d %s\n",
 659                         write ? "Writes" : "Reads ",
 660                         sum, max, min, max / 2 > min ? "???" : "",
 661                         fail, fail ? "!!!" : "");
 662         if (fail)
 663                 atomic_inc(&cxt.n_lock_torture_errors);
 664 }
 665
 666 /*
 667  * Print torture statistics.  Caller must ensure that there is only one
 668  * call to this function at a given time!!!  This is normally accomplished
 669  * by relying on the module system to only have one copy of the module
 670  * loaded, and then by giving the lock_torture_stats kthread full control
 671  * (or the init/cleanup functions when lock_torture_stats thread is not
 672  * running).
 673  */
 674 static void lock_torture_stats_print(void)
 675 {
 676         int size = cxt.nrealwriters_stress * 200 + 8192;
 677         char *buf;
 678
 679         if (cxt.cur_ops->readlock)
 680                 size += cxt.nrealreaders_stress * 200 + 8192;
 681
 682         buf = kmalloc(size, GFP_KERNEL);
 683         if (!buf) {
 684                 pr_err("lock_torture_stats_print: Out of memory, need: %d",
 685                        size);
 686                 return;
 687         }
 688
 689         __torture_print_stats(buf, cxt.lwsa, true);
 690         pr_alert("%s", buf);
 691         kfree(buf);
 692
 693         if (cxt.cur_ops->readlock) {
 694                 buf = kmalloc(size, GFP_KERNEL);
 695                 if (!buf) {
 696                         pr_err("lock_torture_stats_print: Out of memory, need: %d",
 697                                size);
 698                         return;
 699                 }
 700
 701                 __torture_print_stats(buf, cxt.lrsa, false);
 702                 pr_alert("%s", buf);
 703                 kfree(buf);
 704         }
 705 }
 706
 707 /*
 708  * Periodically prints torture statistics, if periodic statistics printing
 709  * was specified via the stat_interval module parameter.
 710  *
 711  * No need to worry about fullstop here, since this one doesn't reference
 712  * volatile state or register callbacks.
 713  */
 714 static int lock_torture_stats(void *arg)
 715 {
 716         VERBOSE_TOROUT_STRING("lock_torture_stats task started");
 717         do {
 718                 schedule_timeout_interruptible(stat_interval * HZ);
 719                 lock_torture_stats_print();
 720                 torture_shutdown_absorb("lock_torture_stats");
 721         } while (!torture_must_stop());
 722         torture_kthread_stopping("lock_torture_stats");
 723         return 0;
 724 }
 725
 726 static inline void
 727 lock_torture_print_module_parms(struct lock_torture_ops *cur_ops,
 728                                 const char *tag)
 729 {
 730         pr_alert("%s" TORTURE_FLAG
 731                  "--- %s%s: nwriters_stress=%d nreaders_stress=%d stat_interval=%d verbose=%d shuffle_interval=%d stutter=%d shutdown_secs=%d onoff_interval=%d onoff_holdoff=%d\n",
 732                  torture_type, tag, cxt.debug_lock ? " [debug]": "",
 733                  cxt.nrealwriters_stress, cxt.nrealreaders_stress, stat_interval,
 734                  verbose, shuffle_interval, stutter, shutdown_secs,
 735                  onoff_interval, onoff_holdoff);
 736 }
 737
 738 static void lock_torture_cleanup(void)
 739 {
 740         int i;
 741
 742         if (torture_cleanup_begin())
 743                 return;
 744
 745         /*
 746          * Indicates early cleanup, meaning that the test has not run,
 747          * such as when passing bogus args when loading the module. As
 748          * such, only perform the underlying torture-specific cleanups,
 749          * and avoid anything related to locktorture.
 750          */
 751         if (!cxt.lwsa && !cxt.lrsa)
 752                 goto end;
 753
 754         if (writer_tasks) {
 755                 for (i = 0; i < cxt.nrealwriters_stress; i++)
 756                         torture_stop_kthread(lock_torture_writer,
 757                                              writer_tasks[i]);
 758                 kfree(writer_tasks);
 759                 writer_tasks = NULL;
 760         }
 761
 762         if (reader_tasks) {
 763                 for (i = 0; i < cxt.nrealreaders_stress; i++)
 764                         torture_stop_kthread(lock_torture_reader,
 765                                              reader_tasks[i]);
 766                 kfree(reader_tasks);
 767                 reader_tasks = NULL;
 768         }
 769
 770         torture_stop_kthread(lock_torture_stats, stats_task);
 771         lock_torture_stats_print();  /* -After- the stats thread is stopped! */
 772
 773         if (atomic_read(&cxt.n_lock_torture_errors))
 774                 lock_torture_print_module_parms(cxt.cur_ops,
 775                                                 "End of test: FAILURE");
 776         else if (torture_onoff_failures())
 777                 lock_torture_print_module_parms(cxt.cur_ops,
 778                                                 "End of test: LOCK_HOTPLUG");
 779         else
 780                 lock_torture_print_module_parms(cxt.cur_ops,
 781                                                 "End of test: SUCCESS");
 782
 783         kfree(cxt.lwsa);
 784         kfree(cxt.lrsa);
 785
 786 end:
 787         torture_cleanup_end();
 788 }
 789
 790 static int __init lock_torture_init(void)
 791 {
 792         int i, j;
 793         int firsterr = 0;
 794         static struct lock_torture_ops *torture_ops[] = {
 795                 &lock_busted_ops,
 796                 &spin_lock_ops, &spin_lock_irq_ops,
 797                 &rw_lock_ops, &rw_lock_irq_ops,
 798                 &mutex_lock_ops,
 799 #ifdef CONFIG_RT_MUTEXES
 800                 &rtmutex_lock_ops,
 801 #endif
 802                 &rwsem_lock_ops,
 803                 &percpu_rwsem_lock_ops,
 804         };
 805
 806         if (!torture_init_begin(torture_type, verbose, &torture_runnable))
 807                 return -EBUSY;
 808
 809         /* Process args and tell the world that the torturer is on the job. */
 810         for (i = 0; i < ARRAY_SIZE(torture_ops); i++) {
 811                 cxt.cur_ops = torture_ops[i];
 812                 if (strcmp(torture_type, cxt.cur_ops->name) == 0)
 813                         break;
 814         }
 815         if (i == ARRAY_SIZE(torture_ops)) {
 816                 pr_alert("lock-torture: invalid torture type: \"%s\"\n",
 817                          torture_type);
 818                 pr_alert("lock-torture types:");
 819                 for (i = 0; i < ARRAY_SIZE(torture_ops); i++)
 820                         pr_alert(" %s", torture_ops[i]->name);
 821                 pr_alert("\n");
 822                 firsterr = -EINVAL;
 823                 goto unwind;
 824         }
 825
 826         if (nwriters_stress == 0 && nreaders_stress == 0) {
 827                 pr_alert("lock-torture: must run at least one locking thread\n");
 828                 firsterr = -EINVAL;
 829                 goto unwind;
 830         }
 831
 832         if (cxt.cur_ops->init)
 833                 cxt.cur_ops->init();
 834
 835         if (nwriters_stress >= 0)
 836                 cxt.nrealwriters_stress = nwriters_stress;
 837         else
 838                 cxt.nrealwriters_stress = 2 * num_online_cpus();
 839
 840 #ifdef CONFIG_DEBUG_MUTEXES
 841         if (strncmp(torture_type, "mutex", 5) == 0)
 842                 cxt.debug_lock = true;
 843 #endif
 844 #ifdef CONFIG_DEBUG_RT_MUTEXES
 845         if (strncmp(torture_type, "rtmutex", 7) == 0)
 846                 cxt.debug_lock = true;
 847 #endif
 848 #ifdef CONFIG_DEBUG_SPINLOCK
 849         if ((strncmp(torture_type, "spin", 4) == 0) ||
 850             (strncmp(torture_type, "rw_lock", 7) == 0))
 851                 cxt.debug_lock = true;
 852 #endif
 853
 854         /* Initialize the statistics so that each run gets its own numbers. */
 855         if (nwriters_stress) {
 856                 lock_is_write_held = 0;
 857                 cxt.lwsa = kmalloc(sizeof(*cxt.lwsa) * cxt.nrealwriters_stress, GFP_KERNEL);
 858                 if (cxt.lwsa == NULL) {
 859                         VERBOSE_TOROUT_STRING("cxt.lwsa: Out of memory");
 860                         firsterr = -ENOMEM;
 861                         goto unwind;
 862                 }
 863
 864                 for (i = 0; i < cxt.nrealwriters_stress; i++) {
 865                         cxt.lwsa[i].n_lock_fail = 0;
 866                         cxt.lwsa[i].n_lock_acquired = 0;
 867                 }
 868         }
 869
 870         if (cxt.cur_ops->readlock) {
 871                 if (nreaders_stress >= 0)
 872                         cxt.nrealreaders_stress = nreaders_stress;
 873                 else {
 874                         /*
 875                          * By default distribute evenly the number of
 876                          * readers and writers. We still run the same number
 877                          * of threads as the writer-only locks default.
 878                          */
 879                         if (nwriters_stress < 0) /* user doesn't care */
 880                                 cxt.nrealwriters_stress = num_online_cpus();
 881                         cxt.nrealreaders_stress = cxt.nrealwriters_stress;
 882                 }
 883
 884                 if (nreaders_stress) {
 885                         lock_is_read_held = 0;
 886                         cxt.lrsa = kmalloc(sizeof(*cxt.lrsa) * cxt.nrealreaders_stress, GFP_KERNEL);
 887                         if (cxt.lrsa == NULL) {
 888                                 VERBOSE_TOROUT_STRING("cxt.lrsa: Out of memory");
 889                                 firsterr = -ENOMEM;
 890                                 kfree(cxt.lwsa);
 891                                 cxt.lwsa = NULL;
 892                                 goto unwind;
 893                         }
 894
 895                         for (i = 0; i < cxt.nrealreaders_stress; i++) {
 896                                 cxt.lrsa[i].n_lock_fail = 0;
 897                                 cxt.lrsa[i].n_lock_acquired = 0;
 898                         }
 899                 }
 900         }
 901
 902         lock_torture_print_module_parms(cxt.cur_ops, "Start of test");
 903
 904         /* Prepare torture context. */
 905         if (onoff_interval > 0) {
 906                 firsterr = torture_onoff_init(onoff_holdoff * HZ,
 907                                               onoff_interval * HZ);
 908                 if (firsterr)
 909                         goto unwind;
 910         }
 911         if (shuffle_interval > 0) {
 912                 firsterr = torture_shuffle_init(shuffle_interval);
 913                 if (firsterr)
 914                         goto unwind;
 915         }
 916         if (shutdown_secs > 0) {
 917                 firsterr = torture_shutdown_init(shutdown_secs,
 918                                                  lock_torture_cleanup);
 919                 if (firsterr)
 920                         goto unwind;
 921         }
 922         if (stutter > 0) {
 923                 firsterr = torture_stutter_init(stutter);
 924                 if (firsterr)
 925                         goto unwind;
 926         }
 927
 928         if (nwriters_stress) {
 929                 writer_tasks = kzalloc(cxt.nrealwriters_stress * sizeof(writer_tasks[0]),
 930                                        GFP_KERNEL);
 931                 if (writer_tasks == NULL) {
 932                         VERBOSE_TOROUT_ERRSTRING("writer_tasks: Out of memory");
 933                         firsterr = -ENOMEM;
 934                         goto unwind;
 935                 }
 936         }
 937
 938         if (cxt.cur_ops->readlock) {
 939                 reader_tasks = kzalloc(cxt.nrealreaders_stress * sizeof(reader_tasks[0]),
 940                                        GFP_KERNEL);
 941                 if (reader_tasks == NULL) {
 942                         VERBOSE_TOROUT_ERRSTRING("reader_tasks: Out of memory");
 943                         kfree(writer_tasks);
 944                         writer_tasks = NULL;
 945                         firsterr = -ENOMEM;
 946                         goto unwind;
 947                 }
 948         }
 949
 950         /*
 951          * Create the kthreads and start torturing (oh, those poor little locks).
 952          *
 953          * TODO: Note that we interleave writers with readers, giving writers a
 954          * slight advantage, by creating its kthread first. This can be modified
 955          * for very specific needs, or even let the user choose the policy, if
 956          * ever wanted.
 957          */
 958         for (i = 0, j = 0; i < cxt.nrealwriters_stress ||
 959                     j < cxt.nrealreaders_stress; i++, j++) {
 960                 if (i >= cxt.nrealwriters_stress)
 961                         goto create_reader;
 962
 963                 /* Create writer. */
 964                 firsterr = torture_create_kthread(lock_torture_writer, &cxt.lwsa[i],
 965                                                   writer_tasks[i]);
 966                 if (firsterr)
 967                         goto unwind;
 968
 969         create_reader:
 970                 if (cxt.cur_ops->readlock == NULL || (j >= cxt.nrealreaders_stress))
 971                         continue;
 972                 /* Create reader. */
 973                 firsterr = torture_create_kthread(lock_torture_reader, &cxt.lrsa[j],
 974                                                   reader_tasks[j]);
 975                 if (firsterr)
 976                         goto unwind;
 977         }
 978         if (stat_interval > 0) {
 979                 firsterr = torture_create_kthread(lock_torture_stats, NULL,
 980                                                   stats_task);
 981                 if (firsterr)
 982                         goto unwind;
 983         }
 984         torture_init_end();
 985         return 0;
 986
 987 unwind:
 988         torture_init_end();
 989         lock_torture_cleanup();
 990         return firsterr;
 991 }
 992
 993 module_init(lock_torture_init);
 994 module_exit(lock_torture_cleanup);