drivers/md/dm-mpath.c

   1 /*
   2  * Copyright (C) 2003 Sistina Software Limited.
   3  * Copyright (C) 2004-2005 Red Hat, Inc. All rights reserved.
   4  *
   5  * This file is released under the GPL.
   6  */
   7
   8 #include <linux/device-mapper.h>
   9
  10 #include "dm-rq.h"
  11 #include "dm-bio-record.h"
  12 #include "dm-path-selector.h"
  13 #include "dm-uevent.h"
  14
  15 #include <linux/blkdev.h>
  16 #include <linux/ctype.h>
  17 #include <linux/init.h>
  18 #include <linux/mempool.h>
  19 #include <linux/module.h>
  20 #include <linux/pagemap.h>
  21 #include <linux/slab.h>
  22 #include <linux/time.h>
  23 #include <linux/workqueue.h>
  24 #include <linux/delay.h>
  25 #include <scsi/scsi_dh.h>
  26 #include <linux/atomic.h>
  27 #include <linux/blk-mq.h>
  28
  29 #define DM_MSG_PREFIX "multipath"
  30 #define DM_PG_INIT_DELAY_MSECS 2000
  31 #define DM_PG_INIT_DELAY_DEFAULT ((unsigned) -1)
  32
  33 /* Path properties */
  34 struct pgpath {
  35         struct list_head list;
  36
  37         struct priority_group *pg;      /* Owning PG */
  38         unsigned fail_count;            /* Cumulative failure count */
  39
  40         struct dm_path path;
  41         struct delayed_work activate_path;
  42
  43         bool is_active:1;               /* Path status */
  44 };
  45
  46 #define path_to_pgpath(__pgp) container_of((__pgp), struct pgpath, path)
  47
  48 /*
  49  * Paths are grouped into Priority Groups and numbered from 1 upwards.
  50  * Each has a path selector which controls which path gets used.
  51  */
  52 struct priority_group {
  53         struct list_head list;
  54
  55         struct multipath *m;            /* Owning multipath instance */
  56         struct path_selector ps;
  57
  58         unsigned pg_num;                /* Reference number */
  59         unsigned nr_pgpaths;            /* Number of paths in PG */
  60         struct list_head pgpaths;
  61
  62         bool bypassed:1;                /* Temporarily bypass this PG? */
  63 };
  64
  65 /* Multipath context */
  66 struct multipath {
  67         unsigned long flags;            /* Multipath state flags */
  68
  69         spinlock_t lock;
  70         enum dm_queue_mode queue_mode;
  71
  72         struct pgpath *current_pgpath;
  73         struct priority_group *current_pg;
  74         struct priority_group *next_pg; /* Switch to this PG if set */
  75
  76         atomic_t nr_valid_paths;        /* Total number of usable paths */
  77         unsigned nr_priority_groups;
  78         struct list_head priority_groups;
  79
  80         const char *hw_handler_name;
  81         char *hw_handler_params;
  82         wait_queue_head_t pg_init_wait; /* Wait for pg_init completion */
  83         unsigned pg_init_retries;       /* Number of times to retry pg_init */
  84         unsigned pg_init_delay_msecs;   /* Number of msecs before pg_init retry */
  85         atomic_t pg_init_in_progress;   /* Only one pg_init allowed at once */
  86         atomic_t pg_init_count;         /* Number of times pg_init called */
  87
  88         struct mutex work_mutex;
  89         struct work_struct trigger_event;
  90         struct dm_target *ti;
  91
  92         struct work_struct process_queued_bios;
  93         struct bio_list queued_bios;
  94 };
  95
  96 /*
  97  * Context information attached to each io we process.
  98  */
  99 struct dm_mpath_io {
 100         struct pgpath *pgpath;
 101         size_t nr_bytes;
 102 };
 103
 104 typedef int (*action_fn) (struct pgpath *pgpath);
 105
 106 static struct workqueue_struct *kmultipathd, *kmpath_handlerd;
 107 static void trigger_event(struct work_struct *work);
 108 static void activate_or_offline_path(struct pgpath *pgpath);
 109 static void activate_path_work(struct work_struct *work);
 110 static void process_queued_bios(struct work_struct *work);
 111
 112 /*-----------------------------------------------
 113  * Multipath state flags.
 114  *-----------------------------------------------*/
 115
 116 #define MPATHF_QUEUE_IO 0                       /* Must we queue all I/O? */
 117 #define MPATHF_QUEUE_IF_NO_PATH 1               /* Queue I/O if last path fails? */
 118 #define MPATHF_SAVED_QUEUE_IF_NO_PATH 2         /* Saved state during suspension */
 119 #define MPATHF_RETAIN_ATTACHED_HW_HANDLER 3     /* If there's already a hw_handler present, don't change it. */
 120 #define MPATHF_PG_INIT_DISABLED 4               /* pg_init is not currently allowed */
 121 #define MPATHF_PG_INIT_REQUIRED 5               /* pg_init needs calling? */
 122 #define MPATHF_PG_INIT_DELAY_RETRY 6            /* Delay pg_init retry? */
 123
 124 /*-----------------------------------------------
 125  * Allocation routines
 126  *-----------------------------------------------*/
 127
 128 static struct pgpath *alloc_pgpath(void)
 129 {
 130         struct pgpath *pgpath = kzalloc(sizeof(*pgpath), GFP_KERNEL);
 131
 132         if (!pgpath)
 133                 return NULL;
 134
 135         pgpath->is_active = true;
 136
 137         return pgpath;
 138 }
 139
 140 static void free_pgpath(struct pgpath *pgpath)
 141 {
 142         kfree(pgpath);
 143 }
 144
 145 static struct priority_group *alloc_priority_group(void)
 146 {
 147         struct priority_group *pg;
 148
 149         pg = kzalloc(sizeof(*pg), GFP_KERNEL);
 150
 151         if (pg)
 152                 INIT_LIST_HEAD(&pg->pgpaths);
 153
 154         return pg;
 155 }
 156
 157 static void free_pgpaths(struct list_head *pgpaths, struct dm_target *ti)
 158 {
 159         struct pgpath *pgpath, *tmp;
 160
 161         list_for_each_entry_safe(pgpath, tmp, pgpaths, list) {
 162                 list_del(&pgpath->list);
 163                 dm_put_device(ti, pgpath->path.dev);
 164                 free_pgpath(pgpath);
 165         }
 166 }
 167
 168 static void free_priority_group(struct priority_group *pg,
 169                                 struct dm_target *ti)
 170 {
 171         struct path_selector *ps = &pg->ps;
 172
 173         if (ps->type) {
 174                 ps->type->destroy(ps);
 175                 dm_put_path_selector(ps->type);
 176         }
 177
 178         free_pgpaths(&pg->pgpaths, ti);
 179         kfree(pg);
 180 }
 181
 182 static struct multipath *alloc_multipath(struct dm_target *ti)
 183 {
 184         struct multipath *m;
 185
 186         m = kzalloc(sizeof(*m), GFP_KERNEL);
 187         if (m) {
 188                 INIT_LIST_HEAD(&m->priority_groups);
 189                 spin_lock_init(&m->lock);
 190                 atomic_set(&m->nr_valid_paths, 0);
 191                 INIT_WORK(&m->trigger_event, trigger_event);
 192                 mutex_init(&m->work_mutex);
 193
 194                 m->queue_mode = DM_TYPE_NONE;
 195
 196                 m->ti = ti;
 197                 ti->private = m;
 198         }
 199
 200         return m;
 201 }
 202
 203 static int alloc_multipath_stage2(struct dm_target *ti, struct multipath *m)
 204 {
 205         if (m->queue_mode == DM_TYPE_NONE) {
 206                 /*
 207                  * Default to request-based.
 208                  */
 209                 if (dm_use_blk_mq(dm_table_get_md(ti->table)))
 210                         m->queue_mode = DM_TYPE_MQ_REQUEST_BASED;
 211                 else
 212                         m->queue_mode = DM_TYPE_REQUEST_BASED;
 213
 214         } else if (m->queue_mode == DM_TYPE_BIO_BASED) {
 215                 INIT_WORK(&m->process_queued_bios, process_queued_bios);
 216                 /*
 217                  * bio-based doesn't support any direct scsi_dh management;
 218                  * it just discovers if a scsi_dh is attached.
 219                  */
 220                 set_bit(MPATHF_RETAIN_ATTACHED_HW_HANDLER, &m->flags);
 221         }
 222
 223         dm_table_set_type(ti->table, m->queue_mode);
 224
 225         /*
 226          * Init fields that are only used when a scsi_dh is attached
 227          * - must do this unconditionally (really doesn't hurt non-SCSI uses)
 228          */
 229         set_bit(MPATHF_QUEUE_IO, &m->flags);
 230         atomic_set(&m->pg_init_in_progress, 0);
 231         atomic_set(&m->pg_init_count, 0);
 232         m->pg_init_delay_msecs = DM_PG_INIT_DELAY_DEFAULT;
 233         init_waitqueue_head(&m->pg_init_wait);
 234
 235         return 0;
 236 }
 237
 238 static void free_multipath(struct multipath *m)
 239 {
 240         struct priority_group *pg, *tmp;
 241
 242         list_for_each_entry_safe(pg, tmp, &m->priority_groups, list) {
 243                 list_del(&pg->list);
 244                 free_priority_group(pg, m->ti);
 245         }
 246
 247         kfree(m->hw_handler_name);
 248         kfree(m->hw_handler_params);
 249         mutex_destroy(&m->work_mutex);
 250         kfree(m);
 251 }
 252
 253 static struct dm_mpath_io *get_mpio(union map_info *info)
 254 {
 255         return info->ptr;
 256 }
 257
 258 static size_t multipath_per_bio_data_size(void)
 259 {
 260         return sizeof(struct dm_mpath_io) + sizeof(struct dm_bio_details);
 261 }
 262
 263 static struct dm_mpath_io *get_mpio_from_bio(struct bio *bio)
 264 {
 265         return dm_per_bio_data(bio, multipath_per_bio_data_size());
 266 }
 267
 268 static struct dm_bio_details *get_bio_details_from_mpio(struct dm_mpath_io *mpio)
 269 {
 270         /* dm_bio_details is immediately after the dm_mpath_io in bio's per-bio-data */
 271         void *bio_details = mpio + 1;
 272         return bio_details;
 273 }
 274
 275 static void multipath_init_per_bio_data(struct bio *bio, struct dm_mpath_io **mpio_p)
 276 {
 277         struct dm_mpath_io *mpio = get_mpio_from_bio(bio);
 278         struct dm_bio_details *bio_details = get_bio_details_from_mpio(mpio);
 279
 280         mpio->nr_bytes = bio->bi_iter.bi_size;
 281         mpio->pgpath = NULL;
 282         *mpio_p = mpio;
 283
 284         dm_bio_record(bio_details, bio);
 285 }
 286
 287 /*-----------------------------------------------
 288  * Path selection
 289  *-----------------------------------------------*/
 290
 291 static int __pg_init_all_paths(struct multipath *m)
 292 {
 293         struct pgpath *pgpath;
 294         unsigned long pg_init_delay = 0;
 295
 296         lockdep_assert_held(&m->lock);
 297
 298         if (atomic_read(&m->pg_init_in_progress) || test_bit(MPATHF_PG_INIT_DISABLED, &m->flags))
 299                 return 0;
 300
 301         atomic_inc(&m->pg_init_count);
 302         clear_bit(MPATHF_PG_INIT_REQUIRED, &m->flags);
 303
 304         /* Check here to reset pg_init_required */
 305         if (!m->current_pg)
 306                 return 0;
 307
 308         if (test_bit(MPATHF_PG_INIT_DELAY_RETRY, &m->flags))
 309                 pg_init_delay = msecs_to_jiffies(m->pg_init_delay_msecs != DM_PG_INIT_DELAY_DEFAULT ?
 310                                                  m->pg_init_delay_msecs : DM_PG_INIT_DELAY_MSECS);
 311         list_for_each_entry(pgpath, &m->current_pg->pgpaths, list) {
 312                 /* Skip failed paths */
 313                 if (!pgpath->is_active)
 314                         continue;
 315                 if (queue_delayed_work(kmpath_handlerd, &pgpath->activate_path,
 316                                        pg_init_delay))
 317                         atomic_inc(&m->pg_init_in_progress);
 318         }
 319         return atomic_read(&m->pg_init_in_progress);
 320 }
 321
 322 static int pg_init_all_paths(struct multipath *m)
 323 {
 324         int ret;
 325         unsigned long flags;
 326
 327         spin_lock_irqsave(&m->lock, flags);
 328         ret = __pg_init_all_paths(m);
 329         spin_unlock_irqrestore(&m->lock, flags);
 330
 331         return ret;
 332 }
 333
 334 static void __switch_pg(struct multipath *m, struct priority_group *pg)
 335 {
 336         m->current_pg = pg;
 337
 338         /* Must we initialise the PG first, and queue I/O till it's ready? */
 339         if (m->hw_handler_name) {
 340                 set_bit(MPATHF_PG_INIT_REQUIRED, &m->flags);
 341                 set_bit(MPATHF_QUEUE_IO, &m->flags);
 342         } else {
 343                 clear_bit(MPATHF_PG_INIT_REQUIRED, &m->flags);
 344                 clear_bit(MPATHF_QUEUE_IO, &m->flags);
 345         }
 346
 347         atomic_set(&m->pg_init_count, 0);
 348 }
 349
 350 static struct pgpath *choose_path_in_pg(struct multipath *m,
 351                                         struct priority_group *pg,
 352                                         size_t nr_bytes)
 353 {
 354         unsigned long flags;
 355         struct dm_path *path;
 356         struct pgpath *pgpath;
 357
 358         path = pg->ps.type->select_path(&pg->ps, nr_bytes);
 359         if (!path)
 360                 return ERR_PTR(-ENXIO);
 361
 362         pgpath = path_to_pgpath(path);
 363
 364         if (unlikely(READ_ONCE(m->current_pg) != pg)) {
 365                 /* Only update current_pgpath if pg changed */
 366                 spin_lock_irqsave(&m->lock, flags);
 367                 m->current_pgpath = pgpath;
 368                 __switch_pg(m, pg);
 369                 spin_unlock_irqrestore(&m->lock, flags);
 370         }
 371
 372         return pgpath;
 373 }
 374
 375 static struct pgpath *choose_pgpath(struct multipath *m, size_t nr_bytes)
 376 {
 377         unsigned long flags;
 378         struct priority_group *pg;
 379         struct pgpath *pgpath;
 380         unsigned bypassed = 1;
 381
 382         if (!atomic_read(&m->nr_valid_paths)) {
 383                 clear_bit(MPATHF_QUEUE_IO, &m->flags);
 384                 goto failed;
 385         }
 386
 387         /* Were we instructed to switch PG? */
 388         if (READ_ONCE(m->next_pg)) {
 389                 spin_lock_irqsave(&m->lock, flags);
 390                 pg = m->next_pg;
 391                 if (!pg) {
 392                         spin_unlock_irqrestore(&m->lock, flags);
 393                         goto check_current_pg;
 394                 }
 395                 m->next_pg = NULL;
 396                 spin_unlock_irqrestore(&m->lock, flags);
 397                 pgpath = choose_path_in_pg(m, pg, nr_bytes);
 398                 if (!IS_ERR_OR_NULL(pgpath))
 399                         return pgpath;
 400         }
 401
 402         /* Don't change PG until it has no remaining paths */
 403 check_current_pg:
 404         pg = READ_ONCE(m->current_pg);
 405         if (pg) {
 406                 pgpath = choose_path_in_pg(m, pg, nr_bytes);
 407                 if (!IS_ERR_OR_NULL(pgpath))
 408                         return pgpath;
 409         }
 410
 411         /*
 412          * Loop through priority groups until we find a valid path.
 413          * First time we skip PGs marked 'bypassed'.
 414          * Second time we only try the ones we skipped, but set
 415          * pg_init_delay_retry so we do not hammer controllers.
 416          */
 417         do {
 418                 list_for_each_entry(pg, &m->priority_groups, list) {
 419                         if (pg->bypassed == !!bypassed)
 420                                 continue;
 421                         pgpath = choose_path_in_pg(m, pg, nr_bytes);
 422                         if (!IS_ERR_OR_NULL(pgpath)) {
 423                                 if (!bypassed)
 424                                         set_bit(MPATHF_PG_INIT_DELAY_RETRY, &m->flags);
 425                                 return pgpath;
 426                         }
 427                 }
 428         } while (bypassed--);
 429
 430 failed:
 431         spin_lock_irqsave(&m->lock, flags);
 432         m->current_pgpath = NULL;
 433         m->current_pg = NULL;
 434         spin_unlock_irqrestore(&m->lock, flags);
 435
 436         return NULL;
 437 }
 438
 439 /*
 440  * dm_report_EIO() is a macro instead of a function to make pr_debug()
 441  * report the function name and line number of the function from which
 442  * it has been invoked.
 443  */
 444 #define dm_report_EIO(m)                                                \
 445 do {                                                                    \
 446         struct mapped_device *md = dm_table_get_md((m)->ti->table);     \
 447                                                                         \
 448         pr_debug("%s: returning EIO; QIFNP = %d; SQIFNP = %d; DNFS = %d\n", \
 449                  dm_device_name(md),                                    \
 450                  test_bit(MPATHF_QUEUE_IF_NO_PATH, &(m)->flags),        \
 451                  test_bit(MPATHF_SAVED_QUEUE_IF_NO_PATH, &(m)->flags),  \
 452                  dm_noflush_suspending((m)->ti));                       \
 453 } while (0)
 454
 455 /*
 456  * Check whether bios must be queued in the device-mapper core rather
 457  * than here in the target.
 458  *
 459  * If MPATHF_QUEUE_IF_NO_PATH and MPATHF_SAVED_QUEUE_IF_NO_PATH hold
 460  * the same value then we are not between multipath_presuspend()
 461  * and multipath_resume() calls and we have no need to check
 462  * for the DMF_NOFLUSH_SUSPENDING flag.
 463  */
 464 static bool __must_push_back(struct multipath *m, unsigned long flags)
 465 {
 466         return ((test_bit(MPATHF_QUEUE_IF_NO_PATH, &flags) !=
 467                  test_bit(MPATHF_SAVED_QUEUE_IF_NO_PATH, &flags)) &&
 468                 dm_noflush_suspending(m->ti));
 469 }
 470
 471 /*
 472  * Following functions use READ_ONCE to get atomic access to
 473  * all m->flags to avoid taking spinlock
 474  */
 475 static bool must_push_back_rq(struct multipath *m)
 476 {
 477         unsigned long flags = READ_ONCE(m->flags);
 478         return test_bit(MPATHF_QUEUE_IF_NO_PATH, &flags) || __must_push_back(m, flags);
 479 }
 480
 481 static bool must_push_back_bio(struct multipath *m)
 482 {
 483         unsigned long flags = READ_ONCE(m->flags);
 484         return __must_push_back(m, flags);
 485 }
 486
 487 /*
 488  * Map cloned requests (request-based multipath)
 489  */
 490 static int multipath_clone_and_map(struct dm_target *ti, struct request *rq,
 491                                    union map_info *map_context,
 492                                    struct request **__clone)
 493 {
 494         struct multipath *m = ti->private;
 495         size_t nr_bytes = blk_rq_bytes(rq);
 496         struct pgpath *pgpath;
 497         struct block_device *bdev;
 498         struct dm_mpath_io *mpio = get_mpio(map_context);
 499         struct request_queue *q;
 500         struct request *clone;
 501
 502         /* Do we need to select a new pgpath? */
 503         pgpath = READ_ONCE(m->current_pgpath);
 504         if (!pgpath || !test_bit(MPATHF_QUEUE_IO, &m->flags))
 505                 pgpath = choose_pgpath(m, nr_bytes);
 506
 507         if (!pgpath) {
 508                 if (must_push_back_rq(m))
 509                         return DM_MAPIO_DELAY_REQUEUE;
 510                 dm_report_EIO(m);       /* Failed */
 511                 return DM_MAPIO_KILL;
 512         } else if (test_bit(MPATHF_QUEUE_IO, &m->flags) ||
 513                    test_bit(MPATHF_PG_INIT_REQUIRED, &m->flags)) {
 514                 pg_init_all_paths(m);
 515                 return DM_MAPIO_DELAY_REQUEUE;
 516         }
 517
 518         mpio->pgpath = pgpath;
 519         mpio->nr_bytes = nr_bytes;
 520
 521         bdev = pgpath->path.dev->bdev;
 522         q = bdev_get_queue(bdev);
 523         clone = blk_get_request(q, rq->cmd_flags | REQ_NOMERGE,
 524                         BLK_MQ_REQ_NOWAIT);
 525         if (IS_ERR(clone)) {
 526                 /* EBUSY, ENODEV or EWOULDBLOCK: requeue */
 527                 if (blk_queue_dying(q)) {
 528                         atomic_inc(&m->pg_init_in_progress);
 529                         activate_or_offline_path(pgpath);
 530                         return DM_MAPIO_DELAY_REQUEUE;
 531                 }
 532
 533                 /*
 534                  * blk-mq's SCHED_RESTART can cover this requeue, so we
 535                  * needn't deal with it by DELAY_REQUEUE. More importantly,
 536                  * we have to return DM_MAPIO_REQUEUE so that blk-mq can
 537                  * get the queue busy feedback (via BLK_STS_RESOURCE),
 538                  * otherwise I/O merging can suffer.
 539                  */
 540                 if (q->mq_ops)
 541                         return DM_MAPIO_REQUEUE;
 542                 else
 543                         return DM_MAPIO_DELAY_REQUEUE;
 544         }
 545         clone->bio = clone->biotail = NULL;
 546         clone->rq_disk = bdev->bd_disk;
 547         clone->cmd_flags |= REQ_FAILFAST_TRANSPORT;
 548         *__clone = clone;
 549
 550         if (pgpath->pg->ps.type->start_io)
 551                 pgpath->pg->ps.type->start_io(&pgpath->pg->ps,
 552                                               &pgpath->path,
 553                                               nr_bytes);
 554         return DM_MAPIO_REMAPPED;
 555 }
 556
 557 static void multipath_release_clone(struct request *clone,
 558                                     union map_info *map_context)
 559 {
 560         if (unlikely(map_context)) {
 561                 /*
 562                  * non-NULL map_context means caller is still map
 563                  * method; must undo multipath_clone_and_map()
 564                  */
 565                 struct dm_mpath_io *mpio = get_mpio(map_context);
 566                 struct pgpath *pgpath = mpio->pgpath;
 567
 568                 if (pgpath && pgpath->pg->ps.type->end_io)
 569                         pgpath->pg->ps.type->end_io(&pgpath->pg->ps,
 570                                                     &pgpath->path,
 571                                                     mpio->nr_bytes);
 572         }
 573
 574         blk_put_request(clone);
 575 }
 576
 577 /*
 578  * Map cloned bios (bio-based multipath)
 579  */
 580
 581 static struct pgpath *__map_bio(struct multipath *m, struct bio *bio)
 582 {
 583         struct pgpath *pgpath;
 584         unsigned long flags;
 585         bool queue_io;
 586
 587         /* Do we need to select a new pgpath? */
 588         pgpath = READ_ONCE(m->current_pgpath);
 589         if (!pgpath || !test_bit(MPATHF_QUEUE_IO, &m->flags))
 590                 pgpath = choose_pgpath(m, bio->bi_iter.bi_size);
 591
 592         /* MPATHF_QUEUE_IO might have been cleared by choose_pgpath. */
 593         queue_io = test_bit(MPATHF_QUEUE_IO, &m->flags);
 594
 595         if ((pgpath && queue_io) ||
 596             (!pgpath && test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags))) {
 597                 /* Queue for the daemon to resubmit */
 598                 spin_lock_irqsave(&m->lock, flags);
 599                 bio_list_add(&m->queued_bios, bio);
 600                 spin_unlock_irqrestore(&m->lock, flags);
 601
 602                 /* PG_INIT_REQUIRED cannot be set without QUEUE_IO */
 603                 if (queue_io || test_bit(MPATHF_PG_INIT_REQUIRED, &m->flags))
 604                         pg_init_all_paths(m);
 605                 else if (!queue_io)
 606                         queue_work(kmultipathd, &m->process_queued_bios);
 607
 608                 return ERR_PTR(-EAGAIN);
 609         }
 610
 611         return pgpath;
 612 }
 613
 614 static int __multipath_map_bio(struct multipath *m, struct bio *bio,
 615                                struct dm_mpath_io *mpio)
 616 {
 617         struct pgpath *pgpath = __map_bio(m, bio);
 618
 619         if (IS_ERR(pgpath))
 620                 return DM_MAPIO_SUBMITTED;
 621
 622         if (!pgpath) {
 623                 if (must_push_back_bio(m))
 624                         return DM_MAPIO_REQUEUE;
 625                 dm_report_EIO(m);
 626                 return DM_MAPIO_KILL;
 627         }
 628
 629         mpio->pgpath = pgpath;
 630
 631         bio->bi_status = 0;
 632         bio_set_dev(bio, pgpath->path.dev->bdev);
 633         bio->bi_opf |= REQ_FAILFAST_TRANSPORT;
 634
 635         if (pgpath->pg->ps.type->start_io)
 636                 pgpath->pg->ps.type->start_io(&pgpath->pg->ps,
 637                                               &pgpath->path,
 638                                               mpio->nr_bytes);
 639         return DM_MAPIO_REMAPPED;
 640 }
 641
 642 static int multipath_map_bio(struct dm_target *ti, struct bio *bio)
 643 {
 644         struct multipath *m = ti->private;
 645         struct dm_mpath_io *mpio = NULL;
 646
 647         multipath_init_per_bio_data(bio, &mpio);
 648         return __multipath_map_bio(m, bio, mpio);
 649 }
 650
 651 static void process_queued_io_list(struct multipath *m)
 652 {
 653         if (m->queue_mode == DM_TYPE_MQ_REQUEST_BASED)
 654                 dm_mq_kick_requeue_list(dm_table_get_md(m->ti->table));
 655         else if (m->queue_mode == DM_TYPE_BIO_BASED)
 656                 queue_work(kmultipathd, &m->process_queued_bios);
 657 }
 658
 659 static void process_queued_bios(struct work_struct *work)
 660 {
 661         int r;
 662         unsigned long flags;
 663         struct bio *bio;
 664         struct bio_list bios;
 665         struct blk_plug plug;
 666         struct multipath *m =
 667                 container_of(work, struct multipath, process_queued_bios);
 668
 669         bio_list_init(&bios);
 670
 671         spin_lock_irqsave(&m->lock, flags);
 672
 673         if (bio_list_empty(&m->queued_bios)) {
 674                 spin_unlock_irqrestore(&m->lock, flags);
 675                 return;
 676         }
 677
 678         bio_list_merge(&bios, &m->queued_bios);
 679         bio_list_init(&m->queued_bios);
 680
 681         spin_unlock_irqrestore(&m->lock, flags);
 682
 683         blk_start_plug(&plug);
 684         while ((bio = bio_list_pop(&bios))) {
 685                 struct dm_mpath_io *mpio = get_mpio_from_bio(bio);
 686                 dm_bio_restore(get_bio_details_from_mpio(mpio), bio);
 687                 r = __multipath_map_bio(m, bio, mpio);
 688                 switch (r) {
 689                 case DM_MAPIO_KILL:
 690                         bio->bi_status = BLK_STS_IOERR;
 691                         bio_endio(bio);
 692                         break;
 693                 case DM_MAPIO_REQUEUE:
 694                         bio->bi_status = BLK_STS_DM_REQUEUE;
 695                         bio_endio(bio);
 696                         break;
 697                 case DM_MAPIO_REMAPPED:
 698                         generic_make_request(bio);
 699                         break;
 700                 case DM_MAPIO_SUBMITTED:
 701                         break;
 702                 default:
 703                         WARN_ONCE(true, "__multipath_map_bio() returned %d\n", r);
 704                 }
 705         }
 706         blk_finish_plug(&plug);
 707 }
 708
 709 /*
 710  * If we run out of usable paths, should we queue I/O or error it?
 711  */
 712 static int queue_if_no_path(struct multipath *m, bool queue_if_no_path,
 713                             bool save_old_value)
 714 {
 715         unsigned long flags;
 716
 717         spin_lock_irqsave(&m->lock, flags);
 718         assign_bit(MPATHF_SAVED_QUEUE_IF_NO_PATH, &m->flags,
 719                    (save_old_value && test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags)) ||
 720                    (!save_old_value && queue_if_no_path));
 721         assign_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags, queue_if_no_path);
 722         spin_unlock_irqrestore(&m->lock, flags);
 723
 724         if (!queue_if_no_path) {
 725                 dm_table_run_md_queue_async(m->ti->table);
 726                 process_queued_io_list(m);
 727         }
 728
 729         return 0;
 730 }
 731
 732 /*
 733  * An event is triggered whenever a path is taken out of use.
 734  * Includes path failure and PG bypass.
 735  */
 736 static void trigger_event(struct work_struct *work)
 737 {
 738         struct multipath *m =
 739                 container_of(work, struct multipath, trigger_event);
 740
 741         dm_table_event(m->ti->table);
 742 }
 743
 744 /*-----------------------------------------------------------------
 745  * Constructor/argument parsing:
 746  * <#multipath feature args> [<arg>]*
 747  * <#hw_handler args> [hw_handler [<arg>]*]
 748  * <#priority groups>
 749  * <initial priority group>
 750  *     [<selector> <#selector args> [<arg>]*
 751  *      <#paths> <#per-path selector args>
 752  *         [<path> [<arg>]* ]+ ]+
 753  *---------------------------------------------------------------*/
 754 static int parse_path_selector(struct dm_arg_set *as, struct priority_group *pg,
 755                                struct dm_target *ti)
 756 {
 757         int r;
 758         struct path_selector_type *pst;
 759         unsigned ps_argc;
 760
 761         static const struct dm_arg _args[] = {
 762                 {0, 1024, "invalid number of path selector args"},
 763         };
 764
 765         pst = dm_get_path_selector(dm_shift_arg(as));
 766         if (!pst) {
 767                 ti->error = "unknown path selector type";
 768                 return -EINVAL;
 769         }
 770
 771         r = dm_read_arg_group(_args, as, &ps_argc, &ti->error);
 772         if (r) {
 773                 dm_put_path_selector(pst);
 774                 return -EINVAL;
 775         }
 776
 777         r = pst->create(&pg->ps, ps_argc, as->argv);
 778         if (r) {
 779                 dm_put_path_selector(pst);
 780                 ti->error = "path selector constructor failed";
 781                 return r;
 782         }
 783
 784         pg->ps.type = pst;
 785         dm_consume_args(as, ps_argc);
 786
 787         return 0;
 788 }
 789
 790 static int setup_scsi_dh(struct block_device *bdev, struct multipath *m,
 791                          const char **attached_handler_name, char **error)
 792 {
 793         struct request_queue *q = bdev_get_queue(bdev);
 794         int r;
 795
 796         if (test_bit(MPATHF_RETAIN_ATTACHED_HW_HANDLER, &m->flags)) {
 797 retain:
 798                 if (*attached_handler_name) {
 799                         /*
 800                          * Clear any hw_handler_params associated with a
 801                          * handler that isn't already attached.
 802                          */
 803                         if (m->hw_handler_name && strcmp(*attached_handler_name, m->hw_handler_name)) {
 804                                 kfree(m->hw_handler_params);
 805                                 m->hw_handler_params = NULL;
 806                         }
 807
 808                         /*
 809                          * Reset hw_handler_name to match the attached handler
 810                          *
 811                          * NB. This modifies the table line to show the actual
 812                          * handler instead of the original table passed in.
 813                          */
 814                         kfree(m->hw_handler_name);
 815                         m->hw_handler_name = *attached_handler_name;
 816                         *attached_handler_name = NULL;
 817                 }
 818         }
 819
 820         if (m->hw_handler_name) {
 821                 r = scsi_dh_attach(q, m->hw_handler_name);
 822                 if (r == -EBUSY) {
 823                         char b[BDEVNAME_SIZE];
 824
 825                         printk(KERN_INFO "dm-mpath: retaining handler on device %s\n",
 826                                bdevname(bdev, b));
 827                         goto retain;
 828                 }
 829                 if (r < 0) {
 830                         *error = "error attaching hardware handler";
 831                         return r;
 832                 }
 833
 834                 if (m->hw_handler_params) {
 835                         r = scsi_dh_set_params(q, m->hw_handler_params);
 836                         if (r < 0) {
 837                                 *error = "unable to set hardware handler parameters";
 838                                 return r;
 839                         }
 840                 }
 841         }
 842
 843         return 0;
 844 }
 845
 846 static struct pgpath *parse_path(struct dm_arg_set *as, struct path_selector *ps,
 847                                  struct dm_target *ti)
 848 {
 849         int r;
 850         struct pgpath *p;
 851         struct multipath *m = ti->private;
 852         struct request_queue *q;
 853         const char *attached_handler_name = NULL;
 854
 855         /* we need at least a path arg */
 856         if (as->argc < 1) {
 857                 ti->error = "no device given";
 858                 return ERR_PTR(-EINVAL);
 859         }
 860
 861         p = alloc_pgpath();
 862         if (!p)
 863                 return ERR_PTR(-ENOMEM);
 864
 865         r = dm_get_device(ti, dm_shift_arg(as), dm_table_get_mode(ti->table),
 866                           &p->path.dev);
 867         if (r) {
 868                 ti->error = "error getting device";
 869                 goto bad;
 870         }
 871
 872         q = bdev_get_queue(p->path.dev->bdev);
 873         attached_handler_name = scsi_dh_attached_handler_name(q, GFP_KERNEL);
 874         if (attached_handler_name || m->hw_handler_name) {
 875                 INIT_DELAYED_WORK(&p->activate_path, activate_path_work);
 876                 r = setup_scsi_dh(p->path.dev->bdev, m, &attached_handler_name, &ti->error);
 877                 kfree(attached_handler_name);
 878                 if (r) {
 879                         dm_put_device(ti, p->path.dev);
 880                         goto bad;
 881                 }
 882         }
 883
 884         r = ps->type->add_path(ps, &p->path, as->argc, as->argv, &ti->error);
 885         if (r) {
 886                 dm_put_device(ti, p->path.dev);
 887                 goto bad;
 888         }
 889
 890         return p;
 891  bad:
 892         free_pgpath(p);
 893         return ERR_PTR(r);
 894 }
 895
 896 static struct priority_group *parse_priority_group(struct dm_arg_set *as,
 897                                                    struct multipath *m)
 898 {
 899         static const struct dm_arg _args[] = {
 900                 {1, 1024, "invalid number of paths"},
 901                 {0, 1024, "invalid number of selector args"}
 902         };
 903
 904         int r;
 905         unsigned i, nr_selector_args, nr_args;
 906         struct priority_group *pg;
 907         struct dm_target *ti = m->ti;
 908
 909         if (as->argc < 2) {
 910                 as->argc = 0;
 911                 ti->error = "not enough priority group arguments";
 912                 return ERR_PTR(-EINVAL);
 913         }
 914
 915         pg = alloc_priority_group();
 916         if (!pg) {
 917                 ti->error = "couldn't allocate priority group";
 918                 return ERR_PTR(-ENOMEM);
 919         }
 920         pg->m = m;
 921
 922         r = parse_path_selector(as, pg, ti);
 923         if (r)
 924                 goto bad;
 925
 926         /*
 927          * read the paths
 928          */
 929         r = dm_read_arg(_args, as, &pg->nr_pgpaths, &ti->error);
 930         if (r)
 931                 goto bad;
 932
 933         r = dm_read_arg(_args + 1, as, &nr_selector_args, &ti->error);
 934         if (r)
 935                 goto bad;
 936
 937         nr_args = 1 + nr_selector_args;
 938         for (i = 0; i < pg->nr_pgpaths; i++) {
 939                 struct pgpath *pgpath;
 940                 struct dm_arg_set path_args;
 941
 942                 if (as->argc < nr_args) {
 943                         ti->error = "not enough path parameters";
 944                         r = -EINVAL;
 945                         goto bad;
 946                 }
 947
 948                 path_args.argc = nr_args;
 949                 path_args.argv = as->argv;
 950
 951                 pgpath = parse_path(&path_args, &pg->ps, ti);
 952                 if (IS_ERR(pgpath)) {
 953                         r = PTR_ERR(pgpath);
 954                         goto bad;
 955                 }
 956
 957                 pgpath->pg = pg;
 958                 list_add_tail(&pgpath->list, &pg->pgpaths);
 959                 dm_consume_args(as, nr_args);
 960         }
 961
 962         return pg;
 963
 964  bad:
 965         free_priority_group(pg, ti);
 966         return ERR_PTR(r);
 967 }
 968
 969 static int parse_hw_handler(struct dm_arg_set *as, struct multipath *m)
 970 {
 971         unsigned hw_argc;
 972         int ret;
 973         struct dm_target *ti = m->ti;
 974
 975         static const struct dm_arg _args[] = {
 976                 {0, 1024, "invalid number of hardware handler args"},
 977         };
 978
 979         if (dm_read_arg_group(_args, as, &hw_argc, &ti->error))
 980                 return -EINVAL;
 981
 982         if (!hw_argc)
 983                 return 0;
 984
 985         if (m->queue_mode == DM_TYPE_BIO_BASED) {
 986                 dm_consume_args(as, hw_argc);
 987                 DMERR("bio-based multipath doesn't allow hardware handler args");
 988                 return 0;
 989         }
 990
 991         m->hw_handler_name = kstrdup(dm_shift_arg(as), GFP_KERNEL);
 992         if (!m->hw_handler_name)
 993                 return -EINVAL;
 994
 995         if (hw_argc > 1) {
 996                 char *p;
 997                 int i, j, len = 4;
 998
 999                 for (i = 0; i <= hw_argc - 2; i++)
1000                         len += strlen(as->argv[i]) + 1;
1001                 p = m->hw_handler_params = kzalloc(len, GFP_KERNEL);
1002                 if (!p) {
1003                         ti->error = "memory allocation failed";
1004                         ret = -ENOMEM;
1005                         goto fail;
1006                 }
1007                 j = sprintf(p, "%d", hw_argc - 1);
1008                 for (i = 0, p+=j+1; i <= hw_argc - 2; i++, p+=j+1)
1009                         j = sprintf(p, "%s", as->argv[i]);
1010         }
1011         dm_consume_args(as, hw_argc - 1);
1012
1013         return 0;
1014 fail:
1015         kfree(m->hw_handler_name);
1016         m->hw_handler_name = NULL;
1017         return ret;
1018 }
1019
1020 static int parse_features(struct dm_arg_set *as, struct multipath *m)
1021 {
1022         int r;
1023         unsigned argc;
1024         struct dm_target *ti = m->ti;
1025         const char *arg_name;
1026
1027         static const struct dm_arg _args[] = {
1028                 {0, 8, "invalid number of feature args"},
1029                 {1, 50, "pg_init_retries must be between 1 and 50"},
1030                 {0, 60000, "pg_init_delay_msecs must be between 0 and 60000"},
1031         };
1032
1033         r = dm_read_arg_group(_args, as, &argc, &ti->error);
1034         if (r)
1035                 return -EINVAL;
1036
1037         if (!argc)
1038                 return 0;
1039
1040         do {
1041                 arg_name = dm_shift_arg(as);
1042                 argc--;
1043
1044                 if (!strcasecmp(arg_name, "queue_if_no_path")) {
1045                         r = queue_if_no_path(m, true, false);
1046                         continue;
1047                 }
1048
1049                 if (!strcasecmp(arg_name, "retain_attached_hw_handler")) {
1050                         set_bit(MPATHF_RETAIN_ATTACHED_HW_HANDLER, &m->flags);
1051                         continue;
1052                 }
1053
1054                 if (!strcasecmp(arg_name, "pg_init_retries") &&
1055                     (argc >= 1)) {
1056                         r = dm_read_arg(_args + 1, as, &m->pg_init_retries, &ti->error);
1057                         argc--;
1058                         continue;
1059                 }
1060
1061                 if (!strcasecmp(arg_name, "pg_init_delay_msecs") &&
1062                     (argc >= 1)) {
1063                         r = dm_read_arg(_args + 2, as, &m->pg_init_delay_msecs, &ti->error);
1064                         argc--;
1065                         continue;
1066                 }
1067
1068                 if (!strcasecmp(arg_name, "queue_mode") &&
1069                     (argc >= 1)) {
1070                         const char *queue_mode_name = dm_shift_arg(as);
1071
1072                         if (!strcasecmp(queue_mode_name, "bio"))
1073                                 m->queue_mode = DM_TYPE_BIO_BASED;
1074                         else if (!strcasecmp(queue_mode_name, "rq"))
1075                                 m->queue_mode = DM_TYPE_REQUEST_BASED;
1076                         else if (!strcasecmp(queue_mode_name, "mq"))
1077                                 m->queue_mode = DM_TYPE_MQ_REQUEST_BASED;
1078                         else {
1079                                 ti->error = "Unknown 'queue_mode' requested";
1080                                 r = -EINVAL;
1081                         }
1082                         argc--;
1083                         continue;
1084                 }
1085
1086                 ti->error = "Unrecognised multipath feature request";
1087                 r = -EINVAL;
1088         } while (argc && !r);
1089
1090         return r;
1091 }
1092
1093 static int multipath_ctr(struct dm_target *ti, unsigned argc, char **argv)
1094 {
1095         /* target arguments */
1096         static const struct dm_arg _args[] = {
1097                 {0, 1024, "invalid number of priority groups"},
1098                 {0, 1024, "invalid initial priority group number"},
1099         };
1100
1101         int r;
1102         struct multipath *m;
1103         struct dm_arg_set as;
1104         unsigned pg_count = 0;
1105         unsigned next_pg_num;
1106
1107         as.argc = argc;
1108         as.argv = argv;
1109
1110         m = alloc_multipath(ti);
1111         if (!m) {
1112                 ti->error = "can't allocate multipath";
1113                 return -EINVAL;
1114         }
1115
1116         r = parse_features(&as, m);
1117         if (r)
1118                 goto bad;
1119
1120         r = alloc_multipath_stage2(ti, m);
1121         if (r)
1122                 goto bad;
1123
1124         r = parse_hw_handler(&as, m);
1125         if (r)
1126                 goto bad;
1127
1128         r = dm_read_arg(_args, &as, &m->nr_priority_groups, &ti->error);
1129         if (r)
1130                 goto bad;
1131
1132         r = dm_read_arg(_args + 1, &as, &next_pg_num, &ti->error);
1133         if (r)
1134                 goto bad;
1135
1136         if ((!m->nr_priority_groups && next_pg_num) ||
1137             (m->nr_priority_groups && !next_pg_num)) {
1138                 ti->error = "invalid initial priority group";
1139                 r = -EINVAL;
1140                 goto bad;
1141         }
1142
1143         /* parse the priority groups */
1144         while (as.argc) {
1145                 struct priority_group *pg;
1146                 unsigned nr_valid_paths = atomic_read(&m->nr_valid_paths);
1147
1148                 pg = parse_priority_group(&as, m);
1149                 if (IS_ERR(pg)) {
1150                         r = PTR_ERR(pg);
1151                         goto bad;
1152                 }
1153
1154                 nr_valid_paths += pg->nr_pgpaths;
1155                 atomic_set(&m->nr_valid_paths, nr_valid_paths);
1156
1157                 list_add_tail(&pg->list, &m->priority_groups);
1158                 pg_count++;
1159                 pg->pg_num = pg_count;
1160                 if (!--next_pg_num)
1161                         m->next_pg = pg;
1162         }
1163
1164         if (pg_count != m->nr_priority_groups) {
1165                 ti->error = "priority group count mismatch";
1166                 r = -EINVAL;
1167                 goto bad;
1168         }
1169
1170         ti->num_flush_bios = 1;
1171         ti->num_discard_bios = 1;
1172         ti->num_write_same_bios = 1;
1173         ti->num_write_zeroes_bios = 1;
1174         if (m->queue_mode == DM_TYPE_BIO_BASED)
1175                 ti->per_io_data_size = multipath_per_bio_data_size();
1176         else
1177                 ti->per_io_data_size = sizeof(struct dm_mpath_io);
1178
1179         return 0;
1180
1181  bad:
1182         free_multipath(m);
1183         return r;
1184 }
1185
1186 static void multipath_wait_for_pg_init_completion(struct multipath *m)
1187 {
1188         DEFINE_WAIT(wait);
1189
1190         while (1) {
1191                 prepare_to_wait(&m->pg_init_wait, &wait, TASK_UNINTERRUPTIBLE);
1192
1193                 if (!atomic_read(&m->pg_init_in_progress))
1194                         break;
1195
1196                 io_schedule();
1197         }
1198         finish_wait(&m->pg_init_wait, &wait);
1199 }
1200
1201 static void flush_multipath_work(struct multipath *m)
1202 {
1203         if (m->hw_handler_name) {
1204                 set_bit(MPATHF_PG_INIT_DISABLED, &m->flags);
1205                 smp_mb__after_atomic();
1206
1207                 flush_workqueue(kmpath_handlerd);
1208                 multipath_wait_for_pg_init_completion(m);
1209
1210                 clear_bit(MPATHF_PG_INIT_DISABLED, &m->flags);
1211                 smp_mb__after_atomic();
1212         }
1213
1214         flush_workqueue(kmultipathd);
1215         flush_work(&m->trigger_event);
1216 }
1217
1218 static void multipath_dtr(struct dm_target *ti)
1219 {
1220         struct multipath *m = ti->private;
1221
1222         flush_multipath_work(m);
1223         free_multipath(m);
1224 }
1225
1226 /*
1227  * Take a path out of use.
1228  */
1229 static int fail_path(struct pgpath *pgpath)
1230 {
1231         unsigned long flags;
1232         struct multipath *m = pgpath->pg->m;
1233
1234         spin_lock_irqsave(&m->lock, flags);
1235
1236         if (!pgpath->is_active)
1237                 goto out;
1238
1239         DMWARN("Failing path %s.", pgpath->path.dev->name);
1240
1241         pgpath->pg->ps.type->fail_path(&pgpath->pg->ps, &pgpath->path);
1242         pgpath->is_active = false;
1243         pgpath->fail_count++;
1244
1245         atomic_dec(&m->nr_valid_paths);
1246
1247         if (pgpath == m->current_pgpath)
1248                 m->current_pgpath = NULL;
1249
1250         dm_path_uevent(DM_UEVENT_PATH_FAILED, m->ti,
1251                        pgpath->path.dev->name, atomic_read(&m->nr_valid_paths));
1252
1253         schedule_work(&m->trigger_event);
1254
1255 out:
1256         spin_unlock_irqrestore(&m->lock, flags);
1257
1258         return 0;
1259 }
1260
1261 /*
1262  * Reinstate a previously-failed path
1263  */
1264 static int reinstate_path(struct pgpath *pgpath)
1265 {
1266         int r = 0, run_queue = 0;
1267         unsigned long flags;
1268         struct multipath *m = pgpath->pg->m;
1269         unsigned nr_valid_paths;
1270
1271         spin_lock_irqsave(&m->lock, flags);
1272
1273         if (pgpath->is_active)
1274                 goto out;
1275
1276         DMWARN("Reinstating path %s.", pgpath->path.dev->name);
1277
1278         r = pgpath->pg->ps.type->reinstate_path(&pgpath->pg->ps, &pgpath->path);
1279         if (r)
1280                 goto out;
1281
1282         pgpath->is_active = true;
1283
1284         nr_valid_paths = atomic_inc_return(&m->nr_valid_paths);
1285         if (nr_valid_paths == 1) {
1286                 m->current_pgpath = NULL;
1287                 run_queue = 1;
1288         } else if (m->hw_handler_name && (m->current_pg == pgpath->pg)) {
1289                 if (queue_work(kmpath_handlerd, &pgpath->activate_path.work))
1290                         atomic_inc(&m->pg_init_in_progress);
1291         }
1292
1293         dm_path_uevent(DM_UEVENT_PATH_REINSTATED, m->ti,
1294                        pgpath->path.dev->name, nr_valid_paths);
1295
1296         schedule_work(&m->trigger_event);
1297
1298 out:
1299         spin_unlock_irqrestore(&m->lock, flags);
1300         if (run_queue) {
1301                 dm_table_run_md_queue_async(m->ti->table);
1302                 process_queued_io_list(m);
1303         }
1304
1305         return r;
1306 }
1307
1308 /*
1309  * Fail or reinstate all paths that match the provided struct dm_dev.
1310  */
1311 static int action_dev(struct multipath *m, struct dm_dev *dev,
1312                       action_fn action)
1313 {
1314         int r = -EINVAL;
1315         struct pgpath *pgpath;
1316         struct priority_group *pg;
1317
1318         list_for_each_entry(pg, &m->priority_groups, list) {
1319                 list_for_each_entry(pgpath, &pg->pgpaths, list) {
1320                         if (pgpath->path.dev == dev)
1321                                 r = action(pgpath);
1322                 }
1323         }
1324
1325         return r;
1326 }
1327
1328 /*
1329  * Temporarily try to avoid having to use the specified PG
1330  */
1331 static void bypass_pg(struct multipath *m, struct priority_group *pg,
1332                       bool bypassed)
1333 {
1334         unsigned long flags;
1335
1336         spin_lock_irqsave(&m->lock, flags);
1337
1338         pg->bypassed = bypassed;
1339         m->current_pgpath = NULL;
1340         m->current_pg = NULL;
1341
1342         spin_unlock_irqrestore(&m->lock, flags);
1343
1344         schedule_work(&m->trigger_event);
1345 }
1346
1347 /*
1348  * Switch to using the specified PG from the next I/O that gets mapped
1349  */
1350 static int switch_pg_num(struct multipath *m, const char *pgstr)
1351 {
1352         struct priority_group *pg;
1353         unsigned pgnum;
1354         unsigned long flags;
1355         char dummy;
1356
1357         if (!pgstr || (sscanf(pgstr, "%u%c", &pgnum, &dummy) != 1) || !pgnum ||
1358             !m->nr_priority_groups || (pgnum > m->nr_priority_groups)) {
1359                 DMWARN("invalid PG number supplied to switch_pg_num");
1360                 return -EINVAL;
1361         }
1362
1363         spin_lock_irqsave(&m->lock, flags);
1364         list_for_each_entry(pg, &m->priority_groups, list) {
1365                 pg->bypassed = false;
1366                 if (--pgnum)
1367                         continue;
1368
1369                 m->current_pgpath = NULL;
1370                 m->current_pg = NULL;
1371                 m->next_pg = pg;
1372         }
1373         spin_unlock_irqrestore(&m->lock, flags);
1374
1375         schedule_work(&m->trigger_event);
1376         return 0;
1377 }
1378
1379 /*
1380  * Set/clear bypassed status of a PG.
1381  * PGs are numbered upwards from 1 in the order they were declared.
1382  */
1383 static int bypass_pg_num(struct multipath *m, const char *pgstr, bool bypassed)
1384 {
1385         struct priority_group *pg;
1386         unsigned pgnum;
1387         char dummy;
1388
1389         if (!pgstr || (sscanf(pgstr, "%u%c", &pgnum, &dummy) != 1) || !pgnum ||
1390             !m->nr_priority_groups || (pgnum > m->nr_priority_groups)) {
1391                 DMWARN("invalid PG number supplied to bypass_pg");
1392                 return -EINVAL;
1393         }
1394
1395         list_for_each_entry(pg, &m->priority_groups, list) {
1396                 if (!--pgnum)
1397                         break;
1398         }
1399
1400         bypass_pg(m, pg, bypassed);
1401         return 0;
1402 }
1403
1404 /*
1405  * Should we retry pg_init immediately?
1406  */
1407 static bool pg_init_limit_reached(struct multipath *m, struct pgpath *pgpath)
1408 {
1409         unsigned long flags;
1410         bool limit_reached = false;
1411
1412         spin_lock_irqsave(&m->lock, flags);
1413
1414         if (atomic_read(&m->pg_init_count) <= m->pg_init_retries &&
1415             !test_bit(MPATHF_PG_INIT_DISABLED, &m->flags))
1416                 set_bit(MPATHF_PG_INIT_REQUIRED, &m->flags);
1417         else
1418                 limit_reached = true;
1419
1420         spin_unlock_irqrestore(&m->lock, flags);
1421
1422         return limit_reached;
1423 }
1424
1425 static void pg_init_done(void *data, int errors)
1426 {
1427         struct pgpath *pgpath = data;
1428         struct priority_group *pg = pgpath->pg;
1429         struct multipath *m = pg->m;
1430         unsigned long flags;
1431         bool delay_retry = false;
1432
1433         /* device or driver problems */
1434         switch (errors) {
1435         case SCSI_DH_OK:
1436                 break;
1437         case SCSI_DH_NOSYS:
1438                 if (!m->hw_handler_name) {
1439                         errors = 0;
1440                         break;
1441                 }
1442                 DMERR("Could not failover the device: Handler scsi_dh_%s "
1443                       "Error %d.", m->hw_handler_name, errors);
1444                 /*
1445                  * Fail path for now, so we do not ping pong
1446                  */
1447                 fail_path(pgpath);
1448                 break;
1449         case SCSI_DH_DEV_TEMP_BUSY:
1450                 /*
1451                  * Probably doing something like FW upgrade on the
1452                  * controller so try the other pg.
1453                  */
1454                 bypass_pg(m, pg, true);
1455                 break;
1456         case SCSI_DH_RETRY:
1457                 /* Wait before retrying. */
1458                 delay_retry = 1;
1459                 /* fall through */
1460         case SCSI_DH_IMM_RETRY:
1461         case SCSI_DH_RES_TEMP_UNAVAIL:
1462                 if (pg_init_limit_reached(m, pgpath))
1463                         fail_path(pgpath);
1464                 errors = 0;
1465                 break;
1466         case SCSI_DH_DEV_OFFLINED:
1467         default:
1468                 /*
1469                  * We probably do not want to fail the path for a device
1470                  * error, but this is what the old dm did. In future
1471                  * patches we can do more advanced handling.
1472                  */
1473                 fail_path(pgpath);
1474         }
1475
1476         spin_lock_irqsave(&m->lock, flags);
1477         if (errors) {
1478                 if (pgpath == m->current_pgpath) {
1479                         DMERR("Could not failover device. Error %d.", errors);
1480                         m->current_pgpath = NULL;
1481                         m->current_pg = NULL;
1482                 }
1483         } else if (!test_bit(MPATHF_PG_INIT_REQUIRED, &m->flags))
1484                 pg->bypassed = false;
1485
1486         if (atomic_dec_return(&m->pg_init_in_progress) > 0)
1487                 /* Activations of other paths are still on going */
1488                 goto out;
1489
1490         if (test_bit(MPATHF_PG_INIT_REQUIRED, &m->flags)) {
1491                 if (delay_retry)
1492                         set_bit(MPATHF_PG_INIT_DELAY_RETRY, &m->flags);
1493                 else
1494                         clear_bit(MPATHF_PG_INIT_DELAY_RETRY, &m->flags);
1495
1496                 if (__pg_init_all_paths(m))
1497                         goto out;
1498         }
1499         clear_bit(MPATHF_QUEUE_IO, &m->flags);
1500
1501         process_queued_io_list(m);
1502
1503         /*
1504          * Wake up any thread waiting to suspend.
1505          */
1506         wake_up(&m->pg_init_wait);
1507
1508 out:
1509         spin_unlock_irqrestore(&m->lock, flags);
1510 }
1511
1512 static void activate_or_offline_path(struct pgpath *pgpath)
1513 {
1514         struct request_queue *q = bdev_get_queue(pgpath->path.dev->bdev);
1515
1516         if (pgpath->is_active && !blk_queue_dying(q))
1517                 scsi_dh_activate(q, pg_init_done, pgpath);
1518         else
1519                 pg_init_done(pgpath, SCSI_DH_DEV_OFFLINED);
1520 }
1521
1522 static void activate_path_work(struct work_struct *work)
1523 {
1524         struct pgpath *pgpath =
1525                 container_of(work, struct pgpath, activate_path.work);
1526
1527         activate_or_offline_path(pgpath);
1528 }
1529
1530 static int multipath_end_io(struct dm_target *ti, struct request *clone,
1531                             blk_status_t error, union map_info *map_context)
1532 {
1533         struct dm_mpath_io *mpio = get_mpio(map_context);
1534         struct pgpath *pgpath = mpio->pgpath;
1535         int r = DM_ENDIO_DONE;
1536
1537         /*
1538          * We don't queue any clone request inside the multipath target
1539          * during end I/O handling, since those clone requests don't have
1540          * bio clones.  If we queue them inside the multipath target,
1541          * we need to make bio clones, that requires memory allocation.
1542          * (See drivers/md/dm-rq.c:end_clone_bio() about why the clone requests
1543          *  don't have bio clones.)
1544          * Instead of queueing the clone request here, we queue the original
1545          * request into dm core, which will remake a clone request and
1546          * clone bios for it and resubmit it later.
1547          */
1548         if (error && blk_path_error(error)) {
1549                 struct multipath *m = ti->private;
1550
1551                 if (error == BLK_STS_RESOURCE)
1552                         r = DM_ENDIO_DELAY_REQUEUE;
1553                 else
1554                         r = DM_ENDIO_REQUEUE;
1555
1556                 if (pgpath)
1557                         fail_path(pgpath);
1558
1559                 if (atomic_read(&m->nr_valid_paths) == 0 &&
1560                     !must_push_back_rq(m)) {
1561                         if (error == BLK_STS_IOERR)
1562                                 dm_report_EIO(m);
1563                         /* complete with the original error */
1564                         r = DM_ENDIO_DONE;
1565                 }
1566         }
1567
1568         if (pgpath) {
1569                 struct path_selector *ps = &pgpath->pg->ps;
1570
1571                 if (ps->type->end_io)
1572                         ps->type->end_io(ps, &pgpath->path, mpio->nr_bytes);
1573         }
1574
1575         return r;
1576 }
1577
1578 static int multipath_end_io_bio(struct dm_target *ti, struct bio *clone,
1579                                 blk_status_t *error)
1580 {
1581         struct multipath *m = ti->private;
1582         struct dm_mpath_io *mpio = get_mpio_from_bio(clone);
1583         struct pgpath *pgpath = mpio->pgpath;
1584         unsigned long flags;
1585         int r = DM_ENDIO_DONE;
1586
1587         if (!*error || !blk_path_error(*error))
1588                 goto done;
1589
1590         if (pgpath)
1591                 fail_path(pgpath);
1592
1593         if (atomic_read(&m->nr_valid_paths) == 0 &&
1594             !test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags)) {
1595                 if (must_push_back_bio(m)) {
1596                         r = DM_ENDIO_REQUEUE;
1597                 } else {
1598                         dm_report_EIO(m);
1599                         *error = BLK_STS_IOERR;
1600                 }
1601                 goto done;
1602         }
1603
1604         spin_lock_irqsave(&m->lock, flags);
1605         bio_list_add(&m->queued_bios, clone);
1606         spin_unlock_irqrestore(&m->lock, flags);
1607         if (!test_bit(MPATHF_QUEUE_IO, &m->flags))
1608                 queue_work(kmultipathd, &m->process_queued_bios);
1609
1610         r = DM_ENDIO_INCOMPLETE;
1611 done:
1612         if (pgpath) {
1613                 struct path_selector *ps = &pgpath->pg->ps;
1614
1615                 if (ps->type->end_io)
1616                         ps->type->end_io(ps, &pgpath->path, mpio->nr_bytes);
1617         }
1618
1619         return r;
1620 }
1621
1622 /*
1623  * Suspend can't complete until all the I/O is processed so if
1624  * the last path fails we must error any remaining I/O.
1625  * Note that if the freeze_bdev fails while suspending, the
1626  * queue_if_no_path state is lost - userspace should reset it.
1627  */
1628 static void multipath_presuspend(struct dm_target *ti)
1629 {
1630         struct multipath *m = ti->private;
1631
1632         queue_if_no_path(m, false, true);
1633 }
1634
1635 static void multipath_postsuspend(struct dm_target *ti)
1636 {
1637         struct multipath *m = ti->private;
1638
1639         mutex_lock(&m->work_mutex);
1640         flush_multipath_work(m);
1641         mutex_unlock(&m->work_mutex);
1642 }
1643
1644 /*
1645  * Restore the queue_if_no_path setting.
1646  */
1647 static void multipath_resume(struct dm_target *ti)
1648 {
1649         struct multipath *m = ti->private;
1650         unsigned long flags;
1651
1652         spin_lock_irqsave(&m->lock, flags);
1653         assign_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags,
1654                    test_bit(MPATHF_SAVED_QUEUE_IF_NO_PATH, &m->flags));
1655         spin_unlock_irqrestore(&m->lock, flags);
1656 }
1657
1658 /*
1659  * Info output has the following format:
1660  * num_multipath_feature_args [multipath_feature_args]*
1661  * num_handler_status_args [handler_status_args]*
1662  * num_groups init_group_number
1663  *            [A|D|E num_ps_status_args [ps_status_args]*
1664  *             num_paths num_selector_args
1665  *             [path_dev A|F fail_count [selector_args]* ]+ ]+
1666  *
1667  * Table output has the following format (identical to the constructor string):
1668  * num_feature_args [features_args]*
1669  * num_handler_args hw_handler [hw_handler_args]*
1670  * num_groups init_group_number
1671  *     [priority selector-name num_ps_args [ps_args]*
1672  *      num_paths num_selector_args [path_dev [selector_args]* ]+ ]+
1673  */
1674 static void multipath_status(struct dm_target *ti, status_type_t type,
1675                              unsigned status_flags, char *result, unsigned maxlen)
1676 {
1677         int sz = 0;
1678         unsigned long flags;
1679         struct multipath *m = ti->private;
1680         struct priority_group *pg;
1681         struct pgpath *p;
1682         unsigned pg_num;
1683         char state;
1684
1685         spin_lock_irqsave(&m->lock, flags);
1686
1687         /* Features */
1688         if (type == STATUSTYPE_INFO)
1689                 DMEMIT("2 %u %u ", test_bit(MPATHF_QUEUE_IO, &m->flags),
1690                        atomic_read(&m->pg_init_count));
1691         else {
1692                 DMEMIT("%u ", test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags) +
1693                               (m->pg_init_retries > 0) * 2 +
1694                               (m->pg_init_delay_msecs != DM_PG_INIT_DELAY_DEFAULT) * 2 +
1695                               test_bit(MPATHF_RETAIN_ATTACHED_HW_HANDLER, &m->flags) +
1696                               (m->queue_mode != DM_TYPE_REQUEST_BASED) * 2);
1697
1698                 if (test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags))
1699                         DMEMIT("queue_if_no_path ");
1700                 if (m->pg_init_retries)
1701                         DMEMIT("pg_init_retries %u ", m->pg_init_retries);
1702                 if (m->pg_init_delay_msecs != DM_PG_INIT_DELAY_DEFAULT)
1703                         DMEMIT("pg_init_delay_msecs %u ", m->pg_init_delay_msecs);
1704                 if (test_bit(MPATHF_RETAIN_ATTACHED_HW_HANDLER, &m->flags))
1705                         DMEMIT("retain_attached_hw_handler ");
1706                 if (m->queue_mode != DM_TYPE_REQUEST_BASED) {
1707                         switch(m->queue_mode) {
1708                         case DM_TYPE_BIO_BASED:
1709                                 DMEMIT("queue_mode bio ");
1710                                 break;
1711                         case DM_TYPE_MQ_REQUEST_BASED:
1712                                 DMEMIT("queue_mode mq ");
1713                                 break;
1714                         default:
1715                                 WARN_ON_ONCE(true);
1716                                 break;
1717                         }
1718                 }
1719         }
1720
1721         if (!m->hw_handler_name || type == STATUSTYPE_INFO)
1722                 DMEMIT("0 ");
1723         else
1724                 DMEMIT("1 %s ", m->hw_handler_name);
1725
1726         DMEMIT("%u ", m->nr_priority_groups);
1727
1728         if (m->next_pg)
1729                 pg_num = m->next_pg->pg_num;
1730         else if (m->current_pg)
1731                 pg_num = m->current_pg->pg_num;
1732         else
1733                 pg_num = (m->nr_priority_groups ? 1 : 0);
1734
1735         DMEMIT("%u ", pg_num);
1736
1737         switch (type) {
1738         case STATUSTYPE_INFO:
1739                 list_for_each_entry(pg, &m->priority_groups, list) {
1740                         if (pg->bypassed)
1741                                 state = 'D';    /* Disabled */
1742                         else if (pg == m->current_pg)
1743                                 state = 'A';    /* Currently Active */
1744                         else
1745                                 state = 'E';    /* Enabled */
1746
1747                         DMEMIT("%c ", state);
1748
1749                         if (pg->ps.type->status)
1750                                 sz += pg->ps.type->status(&pg->ps, NULL, type,
1751                                                           result + sz,
1752                                                           maxlen - sz);
1753                         else
1754                                 DMEMIT("0 ");
1755
1756                         DMEMIT("%u %u ", pg->nr_pgpaths,
1757                                pg->ps.type->info_args);
1758
1759                         list_for_each_entry(p, &pg->pgpaths, list) {
1760                                 DMEMIT("%s %s %u ", p->path.dev->name,
1761                                        p->is_active ? "A" : "F",
1762                                        p->fail_count);
1763                                 if (pg->ps.type->status)
1764                                         sz += pg->ps.type->status(&pg->ps,
1765                                               &p->path, type, result + sz,
1766                                               maxlen - sz);
1767                         }
1768                 }
1769                 break;
1770
1771         case STATUSTYPE_TABLE:
1772                 list_for_each_entry(pg, &m->priority_groups, list) {
1773                         DMEMIT("%s ", pg->ps.type->name);
1774
1775                         if (pg->ps.type->status)
1776                                 sz += pg->ps.type->status(&pg->ps, NULL, type,
1777                                                           result + sz,
1778                                                           maxlen - sz);
1779                         else
1780                                 DMEMIT("0 ");
1781
1782                         DMEMIT("%u %u ", pg->nr_pgpaths,
1783                                pg->ps.type->table_args);
1784
1785                         list_for_each_entry(p, &pg->pgpaths, list) {
1786                                 DMEMIT("%s ", p->path.dev->name);
1787                                 if (pg->ps.type->status)
1788                                         sz += pg->ps.type->status(&pg->ps,
1789                                               &p->path, type, result + sz,
1790                                               maxlen - sz);
1791                         }
1792                 }
1793                 break;
1794         }
1795
1796         spin_unlock_irqrestore(&m->lock, flags);
1797 }
1798
1799 static int multipath_message(struct dm_target *ti, unsigned argc, char **argv,
1800                              char *result, unsigned maxlen)
1801 {
1802         int r = -EINVAL;
1803         struct dm_dev *dev;
1804         struct multipath *m = ti->private;
1805         action_fn action;
1806
1807         mutex_lock(&m->work_mutex);
1808
1809         if (dm_suspended(ti)) {
1810                 r = -EBUSY;
1811                 goto out;
1812         }
1813
1814         if (argc == 1) {
1815                 if (!strcasecmp(argv[0], "queue_if_no_path")) {
1816                         r = queue_if_no_path(m, true, false);
1817                         goto out;
1818                 } else if (!strcasecmp(argv[0], "fail_if_no_path")) {
1819                         r = queue_if_no_path(m, false, false);
1820                         goto out;
1821                 }
1822         }
1823
1824         if (argc != 2) {
1825                 DMWARN("Invalid multipath message arguments. Expected 2 arguments, got %d.", argc);
1826                 goto out;
1827         }
1828
1829         if (!strcasecmp(argv[0], "disable_group")) {
1830                 r = bypass_pg_num(m, argv[1], true);
1831                 goto out;
1832         } else if (!strcasecmp(argv[0], "enable_group")) {
1833                 r = bypass_pg_num(m, argv[1], false);
1834                 goto out;
1835         } else if (!strcasecmp(argv[0], "switch_group")) {
1836                 r = switch_pg_num(m, argv[1]);
1837                 goto out;
1838         } else if (!strcasecmp(argv[0], "reinstate_path"))
1839                 action = reinstate_path;
1840         else if (!strcasecmp(argv[0], "fail_path"))
1841                 action = fail_path;
1842         else {
1843                 DMWARN("Unrecognised multipath message received: %s", argv[0]);
1844                 goto out;
1845         }
1846
1847         r = dm_get_device(ti, argv[1], dm_table_get_mode(ti->table), &dev);
1848         if (r) {
1849                 DMWARN("message: error getting device %s",
1850                        argv[1]);
1851                 goto out;
1852         }
1853
1854         r = action_dev(m, dev, action);
1855
1856         dm_put_device(ti, dev);
1857
1858 out:
1859         mutex_unlock(&m->work_mutex);
1860         return r;
1861 }
1862
1863 static int multipath_prepare_ioctl(struct dm_target *ti,
1864                                    struct block_device **bdev)
1865 {
1866         struct multipath *m = ti->private;
1867         struct pgpath *current_pgpath;
1868         int r;
1869
1870         current_pgpath = READ_ONCE(m->current_pgpath);
1871         if (!current_pgpath || !test_bit(MPATHF_QUEUE_IO, &m->flags))
1872                 current_pgpath = choose_pgpath(m, 0);
1873
1874         if (current_pgpath) {
1875                 if (!test_bit(MPATHF_QUEUE_IO, &m->flags)) {
1876                         *bdev = current_pgpath->path.dev->bdev;
1877                         r = 0;
1878                 } else {
1879                         /* pg_init has not started or completed */
1880                         r = -ENOTCONN;
1881                 }
1882         } else {
1883                 /* No path is available */
1884                 if (test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags))
1885                         r = -ENOTCONN;
1886                 else
1887                         r = -EIO;
1888         }
1889
1890         if (r == -ENOTCONN) {
1891                 if (!READ_ONCE(m->current_pg)) {
1892                         /* Path status changed, redo selection */
1893                         (void) choose_pgpath(m, 0);
1894                 }
1895                 if (test_bit(MPATHF_PG_INIT_REQUIRED, &m->flags))
1896                         pg_init_all_paths(m);
1897                 dm_table_run_md_queue_async(m->ti->table);
1898                 process_queued_io_list(m);
1899         }
1900
1901         /*
1902          * Only pass ioctls through if the device sizes match exactly.
1903          */
1904         if (!r && ti->len != i_size_read((*bdev)->bd_inode) >> SECTOR_SHIFT)
1905                 return 1;
1906         return r;
1907 }
1908
1909 static int multipath_iterate_devices(struct dm_target *ti,
1910                                      iterate_devices_callout_fn fn, void *data)
1911 {
1912         struct multipath *m = ti->private;
1913         struct priority_group *pg;
1914         struct pgpath *p;
1915         int ret = 0;
1916
1917         list_for_each_entry(pg, &m->priority_groups, list) {
1918                 list_for_each_entry(p, &pg->pgpaths, list) {
1919                         ret = fn(ti, p->path.dev, ti->begin, ti->len, data);
1920                         if (ret)
1921                                 goto out;
1922                 }
1923         }
1924
1925 out:
1926         return ret;
1927 }
1928
1929 static int pgpath_busy(struct pgpath *pgpath)
1930 {
1931         struct request_queue *q = bdev_get_queue(pgpath->path.dev->bdev);
1932
1933         return blk_lld_busy(q);
1934 }
1935
1936 /*
1937  * We return "busy", only when we can map I/Os but underlying devices
1938  * are busy (so even if we map I/Os now, the I/Os will wait on
1939  * the underlying queue).
1940  * In other words, if we want to kill I/Os or queue them inside us
1941  * due to map unavailability, we don't return "busy".  Otherwise,
1942  * dm core won't give us the I/Os and we can't do what we want.
1943  */
1944 static int multipath_busy(struct dm_target *ti)
1945 {
1946         bool busy = false, has_active = false;
1947         struct multipath *m = ti->private;
1948         struct priority_group *pg, *next_pg;
1949         struct pgpath *pgpath;
1950
1951         /* pg_init in progress */
1952         if (atomic_read(&m->pg_init_in_progress))
1953                 return true;
1954
1955         /* no paths available, for blk-mq: rely on IO mapping to delay requeue */
1956         if (!atomic_read(&m->nr_valid_paths) && test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags))
1957                 return (m->queue_mode != DM_TYPE_MQ_REQUEST_BASED);
1958
1959         /* Guess which priority_group will be used at next mapping time */
1960         pg = READ_ONCE(m->current_pg);
1961         next_pg = READ_ONCE(m->next_pg);
1962         if (unlikely(!READ_ONCE(m->current_pgpath) && next_pg))
1963                 pg = next_pg;
1964
1965         if (!pg) {
1966                 /*
1967                  * We don't know which pg will be used at next mapping time.
1968                  * We don't call choose_pgpath() here to avoid to trigger
1969                  * pg_init just by busy checking.
1970                  * So we don't know whether underlying devices we will be using
1971                  * at next mapping time are busy or not. Just try mapping.
1972                  */
1973                 return busy;
1974         }
1975
1976         /*
1977          * If there is one non-busy active path at least, the path selector
1978          * will be able to select it. So we consider such a pg as not busy.
1979          */
1980         busy = true;
1981         list_for_each_entry(pgpath, &pg->pgpaths, list) {
1982                 if (pgpath->is_active) {
1983                         has_active = true;
1984                         if (!pgpath_busy(pgpath)) {
1985                                 busy = false;
1986                                 break;
1987                         }
1988                 }
1989         }
1990
1991         if (!has_active) {
1992                 /*
1993                  * No active path in this pg, so this pg won't be used and
1994                  * the current_pg will be changed at next mapping time.
1995                  * We need to try mapping to determine it.
1996                  */
1997                 busy = false;
1998         }
1999
2000         return busy;
2001 }
2002
2003 /*-----------------------------------------------------------------
2004  * Module setup
2005  *---------------------------------------------------------------*/
2006 static struct target_type multipath_target = {
2007         .name = "multipath",
2008         .version = {1, 13, 0},
2009         .features = DM_TARGET_SINGLETON | DM_TARGET_IMMUTABLE |
2010                     DM_TARGET_PASSES_INTEGRITY,
2011         .module = THIS_MODULE,
2012         .ctr = multipath_ctr,
2013         .dtr = multipath_dtr,
2014         .clone_and_map_rq = multipath_clone_and_map,
2015         .release_clone_rq = multipath_release_clone,
2016         .rq_end_io = multipath_end_io,
2017         .map = multipath_map_bio,
2018         .end_io = multipath_end_io_bio,
2019         .presuspend = multipath_presuspend,
2020         .postsuspend = multipath_postsuspend,
2021         .resume = multipath_resume,
2022         .status = multipath_status,
2023         .message = multipath_message,
2024         .prepare_ioctl = multipath_prepare_ioctl,
2025         .iterate_devices = multipath_iterate_devices,
2026         .busy = multipath_busy,
2027 };
2028
2029 static int __init dm_multipath_init(void)
2030 {
2031         int r;
2032
2033         kmultipathd = alloc_workqueue("kmpathd", WQ_MEM_RECLAIM, 0);
2034         if (!kmultipathd) {
2035                 DMERR("failed to create workqueue kmpathd");
2036                 r = -ENOMEM;
2037                 goto bad_alloc_kmultipathd;
2038         }
2039
2040         /*
2041          * A separate workqueue is used to handle the device handlers
2042          * to avoid overloading existing workqueue. Overloading the
2043          * old workqueue would also create a bottleneck in the
2044          * path of the storage hardware device activation.
2045          */
2046         kmpath_handlerd = alloc_ordered_workqueue("kmpath_handlerd",
2047                                                   WQ_MEM_RECLAIM);
2048         if (!kmpath_handlerd) {
2049                 DMERR("failed to create workqueue kmpath_handlerd");
2050                 r = -ENOMEM;
2051                 goto bad_alloc_kmpath_handlerd;
2052         }
2053
2054         r = dm_register_target(&multipath_target);
2055         if (r < 0) {
2056                 DMERR("request-based register failed %d", r);
2057                 r = -EINVAL;
2058                 goto bad_register_target;
2059         }
2060
2061         return 0;
2062
2063 bad_register_target:
2064         destroy_workqueue(kmpath_handlerd);
2065 bad_alloc_kmpath_handlerd:
2066         destroy_workqueue(kmultipathd);
2067 bad_alloc_kmultipathd:
2068         return r;
2069 }
2070
2071 static void __exit dm_multipath_exit(void)
2072 {
2073         destroy_workqueue(kmpath_handlerd);
2074         destroy_workqueue(kmultipathd);
2075
2076         dm_unregister_target(&multipath_target);
2077 }
2078
2079 module_init(dm_multipath_init);
2080 module_exit(dm_multipath_exit);
2081
2082 MODULE_DESCRIPTION(DM_NAME " multipath target");
2083 MODULE_AUTHOR("Sistina Software <dm-devel@redhat.com>");
2084 MODULE_LICENSE("GPL");