drivers/nvme/target/fc.c

   1 /*
   2  * Copyright (c) 2016 Avago Technologies.  All rights reserved.
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of version 2 of the GNU General Public License as
   6  * published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope that it will be useful.
   9  * ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND WARRANTIES,
  10  * INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY, FITNESS FOR A
  11  * PARTICULAR PURPOSE, OR NON-INFRINGEMENT, ARE DISCLAIMED, EXCEPT TO
  12  * THE EXTENT THAT SUCH DISCLAIMERS ARE HELD TO BE LEGALLY INVALID.
  13  * See the GNU General Public License for more details, a copy of which
  14  * can be found in the file COPYING included with this package
  15  *
  16  */
  17 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  18 #include <linux/module.h>
  19 #include <linux/slab.h>
  20 #include <linux/blk-mq.h>
  21 #include <linux/parser.h>
  22 #include <linux/random.h>
  23 #include <uapi/scsi/fc/fc_fs.h>
  24 #include <uapi/scsi/fc/fc_els.h>
  25
  26 #include "nvmet.h"
  27 #include <linux/nvme-fc-driver.h>
  28 #include <linux/nvme-fc.h>
  29
  30
  31 /* *************************** Data Structures/Defines ****************** */
  32
  33
  34 #define NVMET_LS_CTX_COUNT              256
  35
  36 /* for this implementation, assume small single frame rqst/rsp */
  37 #define NVME_FC_MAX_LS_BUFFER_SIZE              2048
  38
  39 struct nvmet_fc_tgtport;
  40 struct nvmet_fc_tgt_assoc;
  41
  42 struct nvmet_fc_ls_iod {
  43         struct nvmefc_tgt_ls_req        *lsreq;
  44         struct nvmefc_tgt_fcp_req       *fcpreq;        /* only if RS */
  45
  46         struct list_head                ls_list;        /* tgtport->ls_list */
  47
  48         struct nvmet_fc_tgtport         *tgtport;
  49         struct nvmet_fc_tgt_assoc       *assoc;
  50
  51         u8                              *rqstbuf;
  52         u8                              *rspbuf;
  53         u16                             rqstdatalen;
  54         dma_addr_t                      rspdma;
  55
  56         struct scatterlist              sg[2];
  57
  58         struct work_struct              work;
  59 } __aligned(sizeof(unsigned long long));
  60
  61 /* desired maximum for a single sequence - if sg list allows it */
  62 #define NVMET_FC_MAX_SEQ_LENGTH         (256 * 1024)
  63
  64 enum nvmet_fcp_datadir {
  65         NVMET_FCP_NODATA,
  66         NVMET_FCP_WRITE,
  67         NVMET_FCP_READ,
  68         NVMET_FCP_ABORTED,
  69 };
  70
  71 struct nvmet_fc_fcp_iod {
  72         struct nvmefc_tgt_fcp_req       *fcpreq;
  73
  74         struct nvme_fc_cmd_iu           cmdiubuf;
  75         struct nvme_fc_ersp_iu          rspiubuf;
  76         dma_addr_t                      rspdma;
  77         struct scatterlist              *next_sg;
  78         struct scatterlist              *data_sg;
  79         int                             data_sg_cnt;
  80         u32                             offset;
  81         enum nvmet_fcp_datadir          io_dir;
  82         bool                            active;
  83         bool                            abort;
  84         bool                            aborted;
  85         bool                            writedataactive;
  86         spinlock_t                      flock;
  87
  88         struct nvmet_req                req;
  89         struct work_struct              work;
  90         struct work_struct              done_work;
  91         struct work_struct              defer_work;
  92
  93         struct nvmet_fc_tgtport         *tgtport;
  94         struct nvmet_fc_tgt_queue       *queue;
  95
  96         struct list_head                fcp_list;       /* tgtport->fcp_list */
  97 };
  98
  99 struct nvmet_fc_tgtport {
 100
 101         struct nvmet_fc_target_port     fc_target_port;
 102
 103         struct list_head                tgt_list; /* nvmet_fc_target_list */
 104         struct device                   *dev;   /* dev for dma mapping */
 105         struct nvmet_fc_target_template *ops;
 106
 107         struct nvmet_fc_ls_iod          *iod;
 108         spinlock_t                      lock;
 109         struct list_head                ls_list;
 110         struct list_head                ls_busylist;
 111         struct list_head                assoc_list;
 112         struct ida                      assoc_cnt;
 113         struct nvmet_port               *port;
 114         struct kref                     ref;
 115         u32                             max_sg_cnt;
 116 };
 117
 118 struct nvmet_fc_defer_fcp_req {
 119         struct list_head                req_list;
 120         struct nvmefc_tgt_fcp_req       *fcp_req;
 121 };
 122
 123 struct nvmet_fc_tgt_queue {
 124         bool                            ninetypercent;
 125         u16                             qid;
 126         u16                             sqsize;
 127         u16                             ersp_ratio;
 128         __le16                          sqhd;
 129         int                             cpu;
 130         atomic_t                        connected;
 131         atomic_t                        sqtail;
 132         atomic_t                        zrspcnt;
 133         atomic_t                        rsn;
 134         spinlock_t                      qlock;
 135         struct nvmet_port               *port;
 136         struct nvmet_cq                 nvme_cq;
 137         struct nvmet_sq                 nvme_sq;
 138         struct nvmet_fc_tgt_assoc       *assoc;
 139         struct nvmet_fc_fcp_iod         *fod;           /* array of fcp_iods */
 140         struct list_head                fod_list;
 141         struct list_head                pending_cmd_list;
 142         struct list_head                avail_defer_list;
 143         struct workqueue_struct         *work_q;
 144         struct kref                     ref;
 145 } __aligned(sizeof(unsigned long long));
 146
 147 struct nvmet_fc_tgt_assoc {
 148         u64                             association_id;
 149         u32                             a_id;
 150         struct nvmet_fc_tgtport         *tgtport;
 151         struct list_head                a_list;
 152         struct nvmet_fc_tgt_queue       *queues[NVMET_NR_QUEUES + 1];
 153         struct kref                     ref;
 154         struct work_struct              del_work;
 155 };
 156
 157
 158 static inline int
 159 nvmet_fc_iodnum(struct nvmet_fc_ls_iod *iodptr)
 160 {
 161         return (iodptr - iodptr->tgtport->iod);
 162 }
 163
 164 static inline int
 165 nvmet_fc_fodnum(struct nvmet_fc_fcp_iod *fodptr)
 166 {
 167         return (fodptr - fodptr->queue->fod);
 168 }
 169
 170
 171 /*
 172  * Association and Connection IDs:
 173  *
 174  * Association ID will have random number in upper 6 bytes and zero
 175  *   in lower 2 bytes
 176  *
 177  * Connection IDs will be Association ID with QID or'd in lower 2 bytes
 178  *
 179  * note: Association ID = Connection ID for queue 0
 180  */
 181 #define BYTES_FOR_QID                   sizeof(u16)
 182 #define BYTES_FOR_QID_SHIFT             (BYTES_FOR_QID * 8)
 183 #define NVMET_FC_QUEUEID_MASK           ((u64)((1 << BYTES_FOR_QID_SHIFT) - 1))
 184
 185 static inline u64
 186 nvmet_fc_makeconnid(struct nvmet_fc_tgt_assoc *assoc, u16 qid)
 187 {
 188         return (assoc->association_id | qid);
 189 }
 190
 191 static inline u64
 192 nvmet_fc_getassociationid(u64 connectionid)
 193 {
 194         return connectionid & ~NVMET_FC_QUEUEID_MASK;
 195 }
 196
 197 static inline u16
 198 nvmet_fc_getqueueid(u64 connectionid)
 199 {
 200         return (u16)(connectionid & NVMET_FC_QUEUEID_MASK);
 201 }
 202
 203 static inline struct nvmet_fc_tgtport *
 204 targetport_to_tgtport(struct nvmet_fc_target_port *targetport)
 205 {
 206         return container_of(targetport, struct nvmet_fc_tgtport,
 207                                  fc_target_port);
 208 }
 209
 210 static inline struct nvmet_fc_fcp_iod *
 211 nvmet_req_to_fod(struct nvmet_req *nvme_req)
 212 {
 213         return container_of(nvme_req, struct nvmet_fc_fcp_iod, req);
 214 }
 215
 216
 217 /* *************************** Globals **************************** */
 218
 219
 220 static DEFINE_SPINLOCK(nvmet_fc_tgtlock);
 221
 222 static LIST_HEAD(nvmet_fc_target_list);
 223 static DEFINE_IDA(nvmet_fc_tgtport_cnt);
 224
 225
 226 static void nvmet_fc_handle_ls_rqst_work(struct work_struct *work);
 227 static void nvmet_fc_handle_fcp_rqst_work(struct work_struct *work);
 228 static void nvmet_fc_fcp_rqst_op_done_work(struct work_struct *work);
 229 static void nvmet_fc_fcp_rqst_op_defer_work(struct work_struct *work);
 230 static void nvmet_fc_tgt_a_put(struct nvmet_fc_tgt_assoc *assoc);
 231 static int nvmet_fc_tgt_a_get(struct nvmet_fc_tgt_assoc *assoc);
 232 static void nvmet_fc_tgt_q_put(struct nvmet_fc_tgt_queue *queue);
 233 static int nvmet_fc_tgt_q_get(struct nvmet_fc_tgt_queue *queue);
 234 static void nvmet_fc_tgtport_put(struct nvmet_fc_tgtport *tgtport);
 235 static int nvmet_fc_tgtport_get(struct nvmet_fc_tgtport *tgtport);
 236 static void nvmet_fc_handle_fcp_rqst(struct nvmet_fc_tgtport *tgtport,
 237                                         struct nvmet_fc_fcp_iod *fod);
 238 static void nvmet_fc_delete_target_assoc(struct nvmet_fc_tgt_assoc *assoc);
 239
 240
 241 /* *********************** FC-NVME DMA Handling **************************** */
 242
 243 /*
 244  * The fcloop device passes in a NULL device pointer. Real LLD's will
 245  * pass in a valid device pointer. If NULL is passed to the dma mapping
 246  * routines, depending on the platform, it may or may not succeed, and
 247  * may crash.
 248  *
 249  * As such:
 250  * Wrapper all the dma routines and check the dev pointer.
 251  *
 252  * If simple mappings (return just a dma address, we'll noop them,
 253  * returning a dma address of 0.
 254  *
 255  * On more complex mappings (dma_map_sg), a pseudo routine fills
 256  * in the scatter list, setting all dma addresses to 0.
 257  */
 258
 259 static inline dma_addr_t
 260 fc_dma_map_single(struct device *dev, void *ptr, size_t size,
 261                 enum dma_data_direction dir)
 262 {
 263         return dev ? dma_map_single(dev, ptr, size, dir) : (dma_addr_t)0L;
 264 }
 265
 266 static inline int
 267 fc_dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
 268 {
 269         return dev ? dma_mapping_error(dev, dma_addr) : 0;
 270 }
 271
 272 static inline void
 273 fc_dma_unmap_single(struct device *dev, dma_addr_t addr, size_t size,
 274         enum dma_data_direction dir)
 275 {
 276         if (dev)
 277                 dma_unmap_single(dev, addr, size, dir);
 278 }
 279
 280 static inline void
 281 fc_dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr, size_t size,
 282                 enum dma_data_direction dir)
 283 {
 284         if (dev)
 285                 dma_sync_single_for_cpu(dev, addr, size, dir);
 286 }
 287
 288 static inline void
 289 fc_dma_sync_single_for_device(struct device *dev, dma_addr_t addr, size_t size,
 290                 enum dma_data_direction dir)
 291 {
 292         if (dev)
 293                 dma_sync_single_for_device(dev, addr, size, dir);
 294 }
 295
 296 /* pseudo dma_map_sg call */
 297 static int
 298 fc_map_sg(struct scatterlist *sg, int nents)
 299 {
 300         struct scatterlist *s;
 301         int i;
 302
 303         WARN_ON(nents == 0 || sg[0].length == 0);
 304
 305         for_each_sg(sg, s, nents, i) {
 306                 s->dma_address = 0L;
 307 #ifdef CONFIG_NEED_SG_DMA_LENGTH
 308                 s->dma_length = s->length;
 309 #endif
 310         }
 311         return nents;
 312 }
 313
 314 static inline int
 315 fc_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents,
 316                 enum dma_data_direction dir)
 317 {
 318         return dev ? dma_map_sg(dev, sg, nents, dir) : fc_map_sg(sg, nents);
 319 }
 320
 321 static inline void
 322 fc_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,
 323                 enum dma_data_direction dir)
 324 {
 325         if (dev)
 326                 dma_unmap_sg(dev, sg, nents, dir);
 327 }
 328
 329
 330 /* *********************** FC-NVME Port Management ************************ */
 331
 332
 333 static int
 334 nvmet_fc_alloc_ls_iodlist(struct nvmet_fc_tgtport *tgtport)
 335 {
 336         struct nvmet_fc_ls_iod *iod;
 337         int i;
 338
 339         iod = kcalloc(NVMET_LS_CTX_COUNT, sizeof(struct nvmet_fc_ls_iod),
 340                         GFP_KERNEL);
 341         if (!iod)
 342                 return -ENOMEM;
 343
 344         tgtport->iod = iod;
 345
 346         for (i = 0; i < NVMET_LS_CTX_COUNT; iod++, i++) {
 347                 INIT_WORK(&iod->work, nvmet_fc_handle_ls_rqst_work);
 348                 iod->tgtport = tgtport;
 349                 list_add_tail(&iod->ls_list, &tgtport->ls_list);
 350
 351                 iod->rqstbuf = kcalloc(2, NVME_FC_MAX_LS_BUFFER_SIZE,
 352                         GFP_KERNEL);
 353                 if (!iod->rqstbuf)
 354                         goto out_fail;
 355
 356                 iod->rspbuf = iod->rqstbuf + NVME_FC_MAX_LS_BUFFER_SIZE;
 357
 358                 iod->rspdma = fc_dma_map_single(tgtport->dev, iod->rspbuf,
 359                                                 NVME_FC_MAX_LS_BUFFER_SIZE,
 360                                                 DMA_TO_DEVICE);
 361                 if (fc_dma_mapping_error(tgtport->dev, iod->rspdma))
 362                         goto out_fail;
 363         }
 364
 365         return 0;
 366
 367 out_fail:
 368         kfree(iod->rqstbuf);
 369         list_del(&iod->ls_list);
 370         for (iod--, i--; i >= 0; iod--, i--) {
 371                 fc_dma_unmap_single(tgtport->dev, iod->rspdma,
 372                                 NVME_FC_MAX_LS_BUFFER_SIZE, DMA_TO_DEVICE);
 373                 kfree(iod->rqstbuf);
 374                 list_del(&iod->ls_list);
 375         }
 376
 377         kfree(iod);
 378
 379         return -EFAULT;
 380 }
 381
 382 static void
 383 nvmet_fc_free_ls_iodlist(struct nvmet_fc_tgtport *tgtport)
 384 {
 385         struct nvmet_fc_ls_iod *iod = tgtport->iod;
 386         int i;
 387
 388         for (i = 0; i < NVMET_LS_CTX_COUNT; iod++, i++) {
 389                 fc_dma_unmap_single(tgtport->dev,
 390                                 iod->rspdma, NVME_FC_MAX_LS_BUFFER_SIZE,
 391                                 DMA_TO_DEVICE);
 392                 kfree(iod->rqstbuf);
 393                 list_del(&iod->ls_list);
 394         }
 395         kfree(tgtport->iod);
 396 }
 397
 398 static struct nvmet_fc_ls_iod *
 399 nvmet_fc_alloc_ls_iod(struct nvmet_fc_tgtport *tgtport)
 400 {
 401         struct nvmet_fc_ls_iod *iod;
 402         unsigned long flags;
 403
 404         spin_lock_irqsave(&tgtport->lock, flags);
 405         iod = list_first_entry_or_null(&tgtport->ls_list,
 406                                         struct nvmet_fc_ls_iod, ls_list);
 407         if (iod)
 408                 list_move_tail(&iod->ls_list, &tgtport->ls_busylist);
 409         spin_unlock_irqrestore(&tgtport->lock, flags);
 410         return iod;
 411 }
 412
 413
 414 static void
 415 nvmet_fc_free_ls_iod(struct nvmet_fc_tgtport *tgtport,
 416                         struct nvmet_fc_ls_iod *iod)
 417 {
 418         unsigned long flags;
 419
 420         spin_lock_irqsave(&tgtport->lock, flags);
 421         list_move(&iod->ls_list, &tgtport->ls_list);
 422         spin_unlock_irqrestore(&tgtport->lock, flags);
 423 }
 424
 425 static void
 426 nvmet_fc_prep_fcp_iodlist(struct nvmet_fc_tgtport *tgtport,
 427                                 struct nvmet_fc_tgt_queue *queue)
 428 {
 429         struct nvmet_fc_fcp_iod *fod = queue->fod;
 430         int i;
 431
 432         for (i = 0; i < queue->sqsize; fod++, i++) {
 433                 INIT_WORK(&fod->work, nvmet_fc_handle_fcp_rqst_work);
 434                 INIT_WORK(&fod->done_work, nvmet_fc_fcp_rqst_op_done_work);
 435                 INIT_WORK(&fod->defer_work, nvmet_fc_fcp_rqst_op_defer_work);
 436                 fod->tgtport = tgtport;
 437                 fod->queue = queue;
 438                 fod->active = false;
 439                 fod->abort = false;
 440                 fod->aborted = false;
 441                 fod->fcpreq = NULL;
 442                 list_add_tail(&fod->fcp_list, &queue->fod_list);
 443                 spin_lock_init(&fod->flock);
 444
 445                 fod->rspdma = fc_dma_map_single(tgtport->dev, &fod->rspiubuf,
 446                                         sizeof(fod->rspiubuf), DMA_TO_DEVICE);
 447                 if (fc_dma_mapping_error(tgtport->dev, fod->rspdma)) {
 448                         list_del(&fod->fcp_list);
 449                         for (fod--, i--; i >= 0; fod--, i--) {
 450                                 fc_dma_unmap_single(tgtport->dev, fod->rspdma,
 451                                                 sizeof(fod->rspiubuf),
 452                                                 DMA_TO_DEVICE);
 453                                 fod->rspdma = 0L;
 454                                 list_del(&fod->fcp_list);
 455                         }
 456
 457                         return;
 458                 }
 459         }
 460 }
 461
 462 static void
 463 nvmet_fc_destroy_fcp_iodlist(struct nvmet_fc_tgtport *tgtport,
 464                                 struct nvmet_fc_tgt_queue *queue)
 465 {
 466         struct nvmet_fc_fcp_iod *fod = queue->fod;
 467         int i;
 468
 469         for (i = 0; i < queue->sqsize; fod++, i++) {
 470                 if (fod->rspdma)
 471                         fc_dma_unmap_single(tgtport->dev, fod->rspdma,
 472                                 sizeof(fod->rspiubuf), DMA_TO_DEVICE);
 473         }
 474 }
 475
 476 static struct nvmet_fc_fcp_iod *
 477 nvmet_fc_alloc_fcp_iod(struct nvmet_fc_tgt_queue *queue)
 478 {
 479         struct nvmet_fc_fcp_iod *fod;
 480
 481         lockdep_assert_held(&queue->qlock);
 482
 483         fod = list_first_entry_or_null(&queue->fod_list,
 484                                         struct nvmet_fc_fcp_iod, fcp_list);
 485         if (fod) {
 486                 list_del(&fod->fcp_list);
 487                 fod->active = true;
 488                 /*
 489                  * no queue reference is taken, as it was taken by the
 490                  * queue lookup just prior to the allocation. The iod
 491                  * will "inherit" that reference.
 492                  */
 493         }
 494         return fod;
 495 }
 496
 497
 498 static void
 499 nvmet_fc_queue_fcp_req(struct nvmet_fc_tgtport *tgtport,
 500                        struct nvmet_fc_tgt_queue *queue,
 501                        struct nvmefc_tgt_fcp_req *fcpreq)
 502 {
 503         struct nvmet_fc_fcp_iod *fod = fcpreq->nvmet_fc_private;
 504
 505         /*
 506          * put all admin cmds on hw queue id 0. All io commands go to
 507          * the respective hw queue based on a modulo basis
 508          */
 509         fcpreq->hwqid = queue->qid ?
 510                         ((queue->qid - 1) % tgtport->ops->max_hw_queues) : 0;
 511
 512         if (tgtport->ops->target_features & NVMET_FCTGTFEAT_CMD_IN_ISR)
 513                 queue_work_on(queue->cpu, queue->work_q, &fod->work);
 514         else
 515                 nvmet_fc_handle_fcp_rqst(tgtport, fod);
 516 }
 517
 518 static void
 519 nvmet_fc_fcp_rqst_op_defer_work(struct work_struct *work)
 520 {
 521         struct nvmet_fc_fcp_iod *fod =
 522                 container_of(work, struct nvmet_fc_fcp_iod, defer_work);
 523
 524         /* Submit deferred IO for processing */
 525         nvmet_fc_queue_fcp_req(fod->tgtport, fod->queue, fod->fcpreq);
 526
 527 }
 528
 529 static void
 530 nvmet_fc_free_fcp_iod(struct nvmet_fc_tgt_queue *queue,
 531                         struct nvmet_fc_fcp_iod *fod)
 532 {
 533         struct nvmefc_tgt_fcp_req *fcpreq = fod->fcpreq;
 534         struct nvmet_fc_tgtport *tgtport = fod->tgtport;
 535         struct nvmet_fc_defer_fcp_req *deferfcp;
 536         unsigned long flags;
 537
 538         fc_dma_sync_single_for_cpu(tgtport->dev, fod->rspdma,
 539                                 sizeof(fod->rspiubuf), DMA_TO_DEVICE);
 540
 541         fcpreq->nvmet_fc_private = NULL;
 542
 543         fod->active = false;
 544         fod->abort = false;
 545         fod->aborted = false;
 546         fod->writedataactive = false;
 547         fod->fcpreq = NULL;
 548
 549         tgtport->ops->fcp_req_release(&tgtport->fc_target_port, fcpreq);
 550
 551         /* release the queue lookup reference on the completed IO */
 552         nvmet_fc_tgt_q_put(queue);
 553
 554         spin_lock_irqsave(&queue->qlock, flags);
 555         deferfcp = list_first_entry_or_null(&queue->pending_cmd_list,
 556                                 struct nvmet_fc_defer_fcp_req, req_list);
 557         if (!deferfcp) {
 558                 list_add_tail(&fod->fcp_list, &fod->queue->fod_list);
 559                 spin_unlock_irqrestore(&queue->qlock, flags);
 560                 return;
 561         }
 562
 563         /* Re-use the fod for the next pending cmd that was deferred */
 564         list_del(&deferfcp->req_list);
 565
 566         fcpreq = deferfcp->fcp_req;
 567
 568         /* deferfcp can be reused for another IO at a later date */
 569         list_add_tail(&deferfcp->req_list, &queue->avail_defer_list);
 570
 571         spin_unlock_irqrestore(&queue->qlock, flags);
 572
 573         /* Save NVME CMD IO in fod */
 574         memcpy(&fod->cmdiubuf, fcpreq->rspaddr, fcpreq->rsplen);
 575
 576         /* Setup new fcpreq to be processed */
 577         fcpreq->rspaddr = NULL;
 578         fcpreq->rsplen  = 0;
 579         fcpreq->nvmet_fc_private = fod;
 580         fod->fcpreq = fcpreq;
 581         fod->active = true;
 582
 583         /* inform LLDD IO is now being processed */
 584         tgtport->ops->defer_rcv(&tgtport->fc_target_port, fcpreq);
 585
 586         /*
 587          * Leave the queue lookup get reference taken when
 588          * fod was originally allocated.
 589          */
 590
 591         queue_work(queue->work_q, &fod->defer_work);
 592 }
 593
 594 static int
 595 nvmet_fc_queue_to_cpu(struct nvmet_fc_tgtport *tgtport, int qid)
 596 {
 597         int cpu, idx, cnt;
 598
 599         if (tgtport->ops->max_hw_queues == 1)
 600                 return WORK_CPU_UNBOUND;
 601
 602         /* Simple cpu selection based on qid modulo active cpu count */
 603         idx = !qid ? 0 : (qid - 1) % num_active_cpus();
 604
 605         /* find the n'th active cpu */
 606         for (cpu = 0, cnt = 0; ; ) {
 607                 if (cpu_active(cpu)) {
 608                         if (cnt == idx)
 609                                 break;
 610                         cnt++;
 611                 }
 612                 cpu = (cpu + 1) % num_possible_cpus();
 613         }
 614
 615         return cpu;
 616 }
 617
 618 static struct nvmet_fc_tgt_queue *
 619 nvmet_fc_alloc_target_queue(struct nvmet_fc_tgt_assoc *assoc,
 620                         u16 qid, u16 sqsize)
 621 {
 622         struct nvmet_fc_tgt_queue *queue;
 623         unsigned long flags;
 624         int ret;
 625
 626         if (qid > NVMET_NR_QUEUES)
 627                 return NULL;
 628
 629         queue = kzalloc((sizeof(*queue) +
 630                                 (sizeof(struct nvmet_fc_fcp_iod) * sqsize)),
 631                                 GFP_KERNEL);
 632         if (!queue)
 633                 return NULL;
 634
 635         if (!nvmet_fc_tgt_a_get(assoc))
 636                 goto out_free_queue;
 637
 638         queue->work_q = alloc_workqueue("ntfc%d.%d.%d", 0, 0,
 639                                 assoc->tgtport->fc_target_port.port_num,
 640                                 assoc->a_id, qid);
 641         if (!queue->work_q)
 642                 goto out_a_put;
 643
 644         queue->fod = (struct nvmet_fc_fcp_iod *)&queue[1];
 645         queue->qid = qid;
 646         queue->sqsize = sqsize;
 647         queue->assoc = assoc;
 648         queue->port = assoc->tgtport->port;
 649         queue->cpu = nvmet_fc_queue_to_cpu(assoc->tgtport, qid);
 650         INIT_LIST_HEAD(&queue->fod_list);
 651         INIT_LIST_HEAD(&queue->avail_defer_list);
 652         INIT_LIST_HEAD(&queue->pending_cmd_list);
 653         atomic_set(&queue->connected, 0);
 654         atomic_set(&queue->sqtail, 0);
 655         atomic_set(&queue->rsn, 1);
 656         atomic_set(&queue->zrspcnt, 0);
 657         spin_lock_init(&queue->qlock);
 658         kref_init(&queue->ref);
 659
 660         nvmet_fc_prep_fcp_iodlist(assoc->tgtport, queue);
 661
 662         ret = nvmet_sq_init(&queue->nvme_sq);
 663         if (ret)
 664                 goto out_fail_iodlist;
 665
 666         WARN_ON(assoc->queues[qid]);
 667         spin_lock_irqsave(&assoc->tgtport->lock, flags);
 668         assoc->queues[qid] = queue;
 669         spin_unlock_irqrestore(&assoc->tgtport->lock, flags);
 670
 671         return queue;
 672
 673 out_fail_iodlist:
 674         nvmet_fc_destroy_fcp_iodlist(assoc->tgtport, queue);
 675         destroy_workqueue(queue->work_q);
 676 out_a_put:
 677         nvmet_fc_tgt_a_put(assoc);
 678 out_free_queue:
 679         kfree(queue);
 680         return NULL;
 681 }
 682
 683
 684 static void
 685 nvmet_fc_tgt_queue_free(struct kref *ref)
 686 {
 687         struct nvmet_fc_tgt_queue *queue =
 688                 container_of(ref, struct nvmet_fc_tgt_queue, ref);
 689         unsigned long flags;
 690
 691         spin_lock_irqsave(&queue->assoc->tgtport->lock, flags);
 692         queue->assoc->queues[queue->qid] = NULL;
 693         spin_unlock_irqrestore(&queue->assoc->tgtport->lock, flags);
 694
 695         nvmet_fc_destroy_fcp_iodlist(queue->assoc->tgtport, queue);
 696
 697         nvmet_fc_tgt_a_put(queue->assoc);
 698
 699         destroy_workqueue(queue->work_q);
 700
 701         kfree(queue);
 702 }
 703
 704 static void
 705 nvmet_fc_tgt_q_put(struct nvmet_fc_tgt_queue *queue)
 706 {
 707         kref_put(&queue->ref, nvmet_fc_tgt_queue_free);
 708 }
 709
 710 static int
 711 nvmet_fc_tgt_q_get(struct nvmet_fc_tgt_queue *queue)
 712 {
 713         return kref_get_unless_zero(&queue->ref);
 714 }
 715
 716
 717 static void
 718 nvmet_fc_delete_target_queue(struct nvmet_fc_tgt_queue *queue)
 719 {
 720         struct nvmet_fc_tgtport *tgtport = queue->assoc->tgtport;
 721         struct nvmet_fc_fcp_iod *fod = queue->fod;
 722         struct nvmet_fc_defer_fcp_req *deferfcp, *tempptr;
 723         unsigned long flags;
 724         int i, writedataactive;
 725         bool disconnect;
 726
 727         disconnect = atomic_xchg(&queue->connected, 0);
 728
 729         spin_lock_irqsave(&queue->qlock, flags);
 730         /* about outstanding io's */
 731         for (i = 0; i < queue->sqsize; fod++, i++) {
 732                 if (fod->active) {
 733                         spin_lock(&fod->flock);
 734                         fod->abort = true;
 735                         writedataactive = fod->writedataactive;
 736                         spin_unlock(&fod->flock);
 737                         /*
 738                          * only call lldd abort routine if waiting for
 739                          * writedata. other outstanding ops should finish
 740                          * on their own.
 741                          */
 742                         if (writedataactive) {
 743                                 spin_lock(&fod->flock);
 744                                 fod->aborted = true;
 745                                 spin_unlock(&fod->flock);
 746                                 tgtport->ops->fcp_abort(
 747                                         &tgtport->fc_target_port, fod->fcpreq);
 748                         }
 749                 }
 750         }
 751
 752         /* Cleanup defer'ed IOs in queue */
 753         list_for_each_entry_safe(deferfcp, tempptr, &queue->avail_defer_list,
 754                                 req_list) {
 755                 list_del(&deferfcp->req_list);
 756                 kfree(deferfcp);
 757         }
 758
 759         for (;;) {
 760                 deferfcp = list_first_entry_or_null(&queue->pending_cmd_list,
 761                                 struct nvmet_fc_defer_fcp_req, req_list);
 762                 if (!deferfcp)
 763                         break;
 764
 765                 list_del(&deferfcp->req_list);
 766                 spin_unlock_irqrestore(&queue->qlock, flags);
 767
 768                 tgtport->ops->defer_rcv(&tgtport->fc_target_port,
 769                                 deferfcp->fcp_req);
 770
 771                 tgtport->ops->fcp_abort(&tgtport->fc_target_port,
 772                                 deferfcp->fcp_req);
 773
 774                 tgtport->ops->fcp_req_release(&tgtport->fc_target_port,
 775                                 deferfcp->fcp_req);
 776
 777                 /* release the queue lookup reference */
 778                 nvmet_fc_tgt_q_put(queue);
 779
 780                 kfree(deferfcp);
 781
 782                 spin_lock_irqsave(&queue->qlock, flags);
 783         }
 784         spin_unlock_irqrestore(&queue->qlock, flags);
 785
 786         flush_workqueue(queue->work_q);
 787
 788         if (disconnect)
 789                 nvmet_sq_destroy(&queue->nvme_sq);
 790
 791         nvmet_fc_tgt_q_put(queue);
 792 }
 793
 794 static struct nvmet_fc_tgt_queue *
 795 nvmet_fc_find_target_queue(struct nvmet_fc_tgtport *tgtport,
 796                                 u64 connection_id)
 797 {
 798         struct nvmet_fc_tgt_assoc *assoc;
 799         struct nvmet_fc_tgt_queue *queue;
 800         u64 association_id = nvmet_fc_getassociationid(connection_id);
 801         u16 qid = nvmet_fc_getqueueid(connection_id);
 802         unsigned long flags;
 803
 804         if (qid > NVMET_NR_QUEUES)
 805                 return NULL;
 806
 807         spin_lock_irqsave(&tgtport->lock, flags);
 808         list_for_each_entry(assoc, &tgtport->assoc_list, a_list) {
 809                 if (association_id == assoc->association_id) {
 810                         queue = assoc->queues[qid];
 811                         if (queue &&
 812                             (!atomic_read(&queue->connected) ||
 813                              !nvmet_fc_tgt_q_get(queue)))
 814                                 queue = NULL;
 815                         spin_unlock_irqrestore(&tgtport->lock, flags);
 816                         return queue;
 817                 }
 818         }
 819         spin_unlock_irqrestore(&tgtport->lock, flags);
 820         return NULL;
 821 }
 822
 823 static void
 824 nvmet_fc_delete_assoc(struct work_struct *work)
 825 {
 826         struct nvmet_fc_tgt_assoc *assoc =
 827                 container_of(work, struct nvmet_fc_tgt_assoc, del_work);
 828
 829         nvmet_fc_delete_target_assoc(assoc);
 830         nvmet_fc_tgt_a_put(assoc);
 831 }
 832
 833 static struct nvmet_fc_tgt_assoc *
 834 nvmet_fc_alloc_target_assoc(struct nvmet_fc_tgtport *tgtport)
 835 {
 836         struct nvmet_fc_tgt_assoc *assoc, *tmpassoc;
 837         unsigned long flags;
 838         u64 ran;
 839         int idx;
 840         bool needrandom = true;
 841
 842         assoc = kzalloc(sizeof(*assoc), GFP_KERNEL);
 843         if (!assoc)
 844                 return NULL;
 845
 846         idx = ida_simple_get(&tgtport->assoc_cnt, 0, 0, GFP_KERNEL);
 847         if (idx < 0)
 848                 goto out_free_assoc;
 849
 850         if (!nvmet_fc_tgtport_get(tgtport))
 851                 goto out_ida_put;
 852
 853         assoc->tgtport = tgtport;
 854         assoc->a_id = idx;
 855         INIT_LIST_HEAD(&assoc->a_list);
 856         kref_init(&assoc->ref);
 857         INIT_WORK(&assoc->del_work, nvmet_fc_delete_assoc);
 858
 859         while (needrandom) {
 860                 get_random_bytes(&ran, sizeof(ran) - BYTES_FOR_QID);
 861                 ran = ran << BYTES_FOR_QID_SHIFT;
 862
 863                 spin_lock_irqsave(&tgtport->lock, flags);
 864                 needrandom = false;
 865                 list_for_each_entry(tmpassoc, &tgtport->assoc_list, a_list)
 866                         if (ran == tmpassoc->association_id) {
 867                                 needrandom = true;
 868                                 break;
 869                         }
 870                 if (!needrandom) {
 871                         assoc->association_id = ran;
 872                         list_add_tail(&assoc->a_list, &tgtport->assoc_list);
 873                 }
 874                 spin_unlock_irqrestore(&tgtport->lock, flags);
 875         }
 876
 877         return assoc;
 878
 879 out_ida_put:
 880         ida_simple_remove(&tgtport->assoc_cnt, idx);
 881 out_free_assoc:
 882         kfree(assoc);
 883         return NULL;
 884 }
 885
 886 static void
 887 nvmet_fc_target_assoc_free(struct kref *ref)
 888 {
 889         struct nvmet_fc_tgt_assoc *assoc =
 890                 container_of(ref, struct nvmet_fc_tgt_assoc, ref);
 891         struct nvmet_fc_tgtport *tgtport = assoc->tgtport;
 892         unsigned long flags;
 893
 894         spin_lock_irqsave(&tgtport->lock, flags);
 895         list_del(&assoc->a_list);
 896         spin_unlock_irqrestore(&tgtport->lock, flags);
 897         ida_simple_remove(&tgtport->assoc_cnt, assoc->a_id);
 898         kfree(assoc);
 899         nvmet_fc_tgtport_put(tgtport);
 900 }
 901
 902 static void
 903 nvmet_fc_tgt_a_put(struct nvmet_fc_tgt_assoc *assoc)
 904 {
 905         kref_put(&assoc->ref, nvmet_fc_target_assoc_free);
 906 }
 907
 908 static int
 909 nvmet_fc_tgt_a_get(struct nvmet_fc_tgt_assoc *assoc)
 910 {
 911         return kref_get_unless_zero(&assoc->ref);
 912 }
 913
 914 static void
 915 nvmet_fc_delete_target_assoc(struct nvmet_fc_tgt_assoc *assoc)
 916 {
 917         struct nvmet_fc_tgtport *tgtport = assoc->tgtport;
 918         struct nvmet_fc_tgt_queue *queue;
 919         unsigned long flags;
 920         int i;
 921
 922         spin_lock_irqsave(&tgtport->lock, flags);
 923         for (i = NVMET_NR_QUEUES; i >= 0; i--) {
 924                 queue = assoc->queues[i];
 925                 if (queue) {
 926                         if (!nvmet_fc_tgt_q_get(queue))
 927                                 continue;
 928                         spin_unlock_irqrestore(&tgtport->lock, flags);
 929                         nvmet_fc_delete_target_queue(queue);
 930                         nvmet_fc_tgt_q_put(queue);
 931                         spin_lock_irqsave(&tgtport->lock, flags);
 932                 }
 933         }
 934         spin_unlock_irqrestore(&tgtport->lock, flags);
 935
 936         nvmet_fc_tgt_a_put(assoc);
 937 }
 938
 939 static struct nvmet_fc_tgt_assoc *
 940 nvmet_fc_find_target_assoc(struct nvmet_fc_tgtport *tgtport,
 941                                 u64 association_id)
 942 {
 943         struct nvmet_fc_tgt_assoc *assoc;
 944         struct nvmet_fc_tgt_assoc *ret = NULL;
 945         unsigned long flags;
 946
 947         spin_lock_irqsave(&tgtport->lock, flags);
 948         list_for_each_entry(assoc, &tgtport->assoc_list, a_list) {
 949                 if (association_id == assoc->association_id) {
 950                         ret = assoc;
 951                         nvmet_fc_tgt_a_get(assoc);
 952                         break;
 953                 }
 954         }
 955         spin_unlock_irqrestore(&tgtport->lock, flags);
 956
 957         return ret;
 958 }
 959
 960
 961 /**
 962  * nvme_fc_register_targetport - transport entry point called by an
 963  *                              LLDD to register the existence of a local
 964  *                              NVME subystem FC port.
 965  * @pinfo:     pointer to information about the port to be registered
 966  * @template:  LLDD entrypoints and operational parameters for the port
 967  * @dev:       physical hardware device node port corresponds to. Will be
 968  *             used for DMA mappings
 969  * @portptr:   pointer to a local port pointer. Upon success, the routine
 970  *             will allocate a nvme_fc_local_port structure and place its
 971  *             address in the local port pointer. Upon failure, local port
 972  *             pointer will be set to NULL.
 973  *
 974  * Returns:
 975  * a completion status. Must be 0 upon success; a negative errno
 976  * (ex: -ENXIO) upon failure.
 977  */
 978 int
 979 nvmet_fc_register_targetport(struct nvmet_fc_port_info *pinfo,
 980                         struct nvmet_fc_target_template *template,
 981                         struct device *dev,
 982                         struct nvmet_fc_target_port **portptr)
 983 {
 984         struct nvmet_fc_tgtport *newrec;
 985         unsigned long flags;
 986         int ret, idx;
 987
 988         if (!template->xmt_ls_rsp || !template->fcp_op ||
 989             !template->fcp_abort ||
 990             !template->fcp_req_release || !template->targetport_delete ||
 991             !template->max_hw_queues || !template->max_sgl_segments ||
 992             !template->max_dif_sgl_segments || !template->dma_boundary) {
 993                 ret = -EINVAL;
 994                 goto out_regtgt_failed;
 995         }
 996
 997         newrec = kzalloc((sizeof(*newrec) + template->target_priv_sz),
 998                          GFP_KERNEL);
 999         if (!newrec) {
1000                 ret = -ENOMEM;
1001                 goto out_regtgt_failed;
1002         }
1003
1004         idx = ida_simple_get(&nvmet_fc_tgtport_cnt, 0, 0, GFP_KERNEL);
1005         if (idx < 0) {
1006                 ret = -ENOSPC;
1007                 goto out_fail_kfree;
1008         }
1009
1010         if (!get_device(dev) && dev) {
1011                 ret = -ENODEV;
1012                 goto out_ida_put;
1013         }
1014
1015         newrec->fc_target_port.node_name = pinfo->node_name;
1016         newrec->fc_target_port.port_name = pinfo->port_name;
1017         newrec->fc_target_port.private = &newrec[1];
1018         newrec->fc_target_port.port_id = pinfo->port_id;
1019         newrec->fc_target_port.port_num = idx;
1020         INIT_LIST_HEAD(&newrec->tgt_list);
1021         newrec->dev = dev;
1022         newrec->ops = template;
1023         spin_lock_init(&newrec->lock);
1024         INIT_LIST_HEAD(&newrec->ls_list);
1025         INIT_LIST_HEAD(&newrec->ls_busylist);
1026         INIT_LIST_HEAD(&newrec->assoc_list);
1027         kref_init(&newrec->ref);
1028         ida_init(&newrec->assoc_cnt);
1029         newrec->max_sg_cnt = template->max_sgl_segments;
1030
1031         ret = nvmet_fc_alloc_ls_iodlist(newrec);
1032         if (ret) {
1033                 ret = -ENOMEM;
1034                 goto out_free_newrec;
1035         }
1036
1037         spin_lock_irqsave(&nvmet_fc_tgtlock, flags);
1038         list_add_tail(&newrec->tgt_list, &nvmet_fc_target_list);
1039         spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags);
1040
1041         *portptr = &newrec->fc_target_port;
1042         return 0;
1043
1044 out_free_newrec:
1045         put_device(dev);
1046 out_ida_put:
1047         ida_simple_remove(&nvmet_fc_tgtport_cnt, idx);
1048 out_fail_kfree:
1049         kfree(newrec);
1050 out_regtgt_failed:
1051         *portptr = NULL;
1052         return ret;
1053 }
1054 EXPORT_SYMBOL_GPL(nvmet_fc_register_targetport);
1055
1056
1057 static void
1058 nvmet_fc_free_tgtport(struct kref *ref)
1059 {
1060         struct nvmet_fc_tgtport *tgtport =
1061                 container_of(ref, struct nvmet_fc_tgtport, ref);
1062         struct device *dev = tgtport->dev;
1063         unsigned long flags;
1064
1065         spin_lock_irqsave(&nvmet_fc_tgtlock, flags);
1066         list_del(&tgtport->tgt_list);
1067         spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags);
1068
1069         nvmet_fc_free_ls_iodlist(tgtport);
1070
1071         /* let the LLDD know we've finished tearing it down */
1072         tgtport->ops->targetport_delete(&tgtport->fc_target_port);
1073
1074         ida_simple_remove(&nvmet_fc_tgtport_cnt,
1075                         tgtport->fc_target_port.port_num);
1076
1077         ida_destroy(&tgtport->assoc_cnt);
1078
1079         kfree(tgtport);
1080
1081         put_device(dev);
1082 }
1083
1084 static void
1085 nvmet_fc_tgtport_put(struct nvmet_fc_tgtport *tgtport)
1086 {
1087         kref_put(&tgtport->ref, nvmet_fc_free_tgtport);
1088 }
1089
1090 static int
1091 nvmet_fc_tgtport_get(struct nvmet_fc_tgtport *tgtport)
1092 {
1093         return kref_get_unless_zero(&tgtport->ref);
1094 }
1095
1096 static void
1097 __nvmet_fc_free_assocs(struct nvmet_fc_tgtport *tgtport)
1098 {
1099         struct nvmet_fc_tgt_assoc *assoc, *next;
1100         unsigned long flags;
1101
1102         spin_lock_irqsave(&tgtport->lock, flags);
1103         list_for_each_entry_safe(assoc, next,
1104                                 &tgtport->assoc_list, a_list) {
1105                 if (!nvmet_fc_tgt_a_get(assoc))
1106                         continue;
1107                 spin_unlock_irqrestore(&tgtport->lock, flags);
1108                 nvmet_fc_delete_target_assoc(assoc);
1109                 nvmet_fc_tgt_a_put(assoc);
1110                 spin_lock_irqsave(&tgtport->lock, flags);
1111         }
1112         spin_unlock_irqrestore(&tgtport->lock, flags);
1113 }
1114
1115 /*
1116  * nvmet layer has called to terminate an association
1117  */
1118 static void
1119 nvmet_fc_delete_ctrl(struct nvmet_ctrl *ctrl)
1120 {
1121         struct nvmet_fc_tgtport *tgtport, *next;
1122         struct nvmet_fc_tgt_assoc *assoc;
1123         struct nvmet_fc_tgt_queue *queue;
1124         unsigned long flags;
1125         bool found_ctrl = false;
1126
1127         /* this is a bit ugly, but don't want to make locks layered */
1128         spin_lock_irqsave(&nvmet_fc_tgtlock, flags);
1129         list_for_each_entry_safe(tgtport, next, &nvmet_fc_target_list,
1130                         tgt_list) {
1131                 if (!nvmet_fc_tgtport_get(tgtport))
1132                         continue;
1133                 spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags);
1134
1135                 spin_lock_irqsave(&tgtport->lock, flags);
1136                 list_for_each_entry(assoc, &tgtport->assoc_list, a_list) {
1137                         queue = assoc->queues[0];
1138                         if (queue && queue->nvme_sq.ctrl == ctrl) {
1139                                 if (nvmet_fc_tgt_a_get(assoc))
1140                                         found_ctrl = true;
1141                                 break;
1142                         }
1143                 }
1144                 spin_unlock_irqrestore(&tgtport->lock, flags);
1145
1146                 nvmet_fc_tgtport_put(tgtport);
1147
1148                 if (found_ctrl) {
1149                         schedule_work(&assoc->del_work);
1150                         return;
1151                 }
1152
1153                 spin_lock_irqsave(&nvmet_fc_tgtlock, flags);
1154         }
1155         spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags);
1156 }
1157
1158 /**
1159  * nvme_fc_unregister_targetport - transport entry point called by an
1160  *                              LLDD to deregister/remove a previously
1161  *                              registered a local NVME subsystem FC port.
1162  * @tgtport: pointer to the (registered) target port that is to be
1163  *           deregistered.
1164  *
1165  * Returns:
1166  * a completion status. Must be 0 upon success; a negative errno
1167  * (ex: -ENXIO) upon failure.
1168  */
1169 int
1170 nvmet_fc_unregister_targetport(struct nvmet_fc_target_port *target_port)
1171 {
1172         struct nvmet_fc_tgtport *tgtport = targetport_to_tgtport(target_port);
1173
1174         /* terminate any outstanding associations */
1175         __nvmet_fc_free_assocs(tgtport);
1176
1177         nvmet_fc_tgtport_put(tgtport);
1178
1179         return 0;
1180 }
1181 EXPORT_SYMBOL_GPL(nvmet_fc_unregister_targetport);
1182
1183
1184 /* *********************** FC-NVME LS Handling **************************** */
1185
1186
1187 static void
1188 nvmet_fc_format_rsp_hdr(void *buf, u8 ls_cmd, __be32 desc_len, u8 rqst_ls_cmd)
1189 {
1190         struct fcnvme_ls_acc_hdr *acc = buf;
1191
1192         acc->w0.ls_cmd = ls_cmd;
1193         acc->desc_list_len = desc_len;
1194         acc->rqst.desc_tag = cpu_to_be32(FCNVME_LSDESC_RQST);
1195         acc->rqst.desc_len =
1196                         fcnvme_lsdesc_len(sizeof(struct fcnvme_lsdesc_rqst));
1197         acc->rqst.w0.ls_cmd = rqst_ls_cmd;
1198 }
1199
1200 static int
1201 nvmet_fc_format_rjt(void *buf, u16 buflen, u8 ls_cmd,
1202                         u8 reason, u8 explanation, u8 vendor)
1203 {
1204         struct fcnvme_ls_rjt *rjt = buf;
1205
1206         nvmet_fc_format_rsp_hdr(buf, FCNVME_LSDESC_RQST,
1207                         fcnvme_lsdesc_len(sizeof(struct fcnvme_ls_rjt)),
1208                         ls_cmd);
1209         rjt->rjt.desc_tag = cpu_to_be32(FCNVME_LSDESC_RJT);
1210         rjt->rjt.desc_len = fcnvme_lsdesc_len(sizeof(struct fcnvme_lsdesc_rjt));
1211         rjt->rjt.reason_code = reason;
1212         rjt->rjt.reason_explanation = explanation;
1213         rjt->rjt.vendor = vendor;
1214
1215         return sizeof(struct fcnvme_ls_rjt);
1216 }
1217
1218 /* Validation Error indexes into the string table below */
1219 enum {
1220         VERR_NO_ERROR           = 0,
1221         VERR_CR_ASSOC_LEN       = 1,
1222         VERR_CR_ASSOC_RQST_LEN  = 2,
1223         VERR_CR_ASSOC_CMD       = 3,
1224         VERR_CR_ASSOC_CMD_LEN   = 4,
1225         VERR_ERSP_RATIO         = 5,
1226         VERR_ASSOC_ALLOC_FAIL   = 6,
1227         VERR_QUEUE_ALLOC_FAIL   = 7,
1228         VERR_CR_CONN_LEN        = 8,
1229         VERR_CR_CONN_RQST_LEN   = 9,
1230         VERR_ASSOC_ID           = 10,
1231         VERR_ASSOC_ID_LEN       = 11,
1232         VERR_NO_ASSOC           = 12,
1233         VERR_CONN_ID            = 13,
1234         VERR_CONN_ID_LEN        = 14,
1235         VERR_NO_CONN            = 15,
1236         VERR_CR_CONN_CMD        = 16,
1237         VERR_CR_CONN_CMD_LEN    = 17,
1238         VERR_DISCONN_LEN        = 18,
1239         VERR_DISCONN_RQST_LEN   = 19,
1240         VERR_DISCONN_CMD        = 20,
1241         VERR_DISCONN_CMD_LEN    = 21,
1242         VERR_DISCONN_SCOPE      = 22,
1243         VERR_RS_LEN             = 23,
1244         VERR_RS_RQST_LEN        = 24,
1245         VERR_RS_CMD             = 25,
1246         VERR_RS_CMD_LEN         = 26,
1247         VERR_RS_RCTL            = 27,
1248         VERR_RS_RO              = 28,
1249 };
1250
1251 static char *validation_errors[] = {
1252         "OK",
1253         "Bad CR_ASSOC Length",
1254         "Bad CR_ASSOC Rqst Length",
1255         "Not CR_ASSOC Cmd",
1256         "Bad CR_ASSOC Cmd Length",
1257         "Bad Ersp Ratio",
1258         "Association Allocation Failed",
1259         "Queue Allocation Failed",
1260         "Bad CR_CONN Length",
1261         "Bad CR_CONN Rqst Length",
1262         "Not Association ID",
1263         "Bad Association ID Length",
1264         "No Association",
1265         "Not Connection ID",
1266         "Bad Connection ID Length",
1267         "No Connection",
1268         "Not CR_CONN Cmd",
1269         "Bad CR_CONN Cmd Length",
1270         "Bad DISCONN Length",
1271         "Bad DISCONN Rqst Length",
1272         "Not DISCONN Cmd",
1273         "Bad DISCONN Cmd Length",
1274         "Bad Disconnect Scope",
1275         "Bad RS Length",
1276         "Bad RS Rqst Length",
1277         "Not RS Cmd",
1278         "Bad RS Cmd Length",
1279         "Bad RS R_CTL",
1280         "Bad RS Relative Offset",
1281 };
1282
1283 static void
1284 nvmet_fc_ls_create_association(struct nvmet_fc_tgtport *tgtport,
1285                         struct nvmet_fc_ls_iod *iod)
1286 {
1287         struct fcnvme_ls_cr_assoc_rqst *rqst =
1288                                 (struct fcnvme_ls_cr_assoc_rqst *)iod->rqstbuf;
1289         struct fcnvme_ls_cr_assoc_acc *acc =
1290                                 (struct fcnvme_ls_cr_assoc_acc *)iod->rspbuf;
1291         struct nvmet_fc_tgt_queue *queue;
1292         int ret = 0;
1293
1294         memset(acc, 0, sizeof(*acc));
1295
1296         /*
1297          * FC-NVME spec changes. There are initiators sending different
1298          * lengths as padding sizes for Create Association Cmd descriptor
1299          * was incorrect.
1300          * Accept anything of "minimum" length. Assume format per 1.15
1301          * spec (with HOSTID reduced to 16 bytes), ignore how long the
1302          * trailing pad length is.
1303          */
1304         if (iod->rqstdatalen < FCNVME_LSDESC_CRA_RQST_MINLEN)
1305                 ret = VERR_CR_ASSOC_LEN;
1306         else if (be32_to_cpu(rqst->desc_list_len) <
1307                         FCNVME_LSDESC_CRA_RQST_MIN_LISTLEN)
1308                 ret = VERR_CR_ASSOC_RQST_LEN;
1309         else if (rqst->assoc_cmd.desc_tag !=
1310                         cpu_to_be32(FCNVME_LSDESC_CREATE_ASSOC_CMD))
1311                 ret = VERR_CR_ASSOC_CMD;
1312         else if (be32_to_cpu(rqst->assoc_cmd.desc_len) <
1313                         FCNVME_LSDESC_CRA_CMD_DESC_MIN_DESCLEN)
1314                 ret = VERR_CR_ASSOC_CMD_LEN;
1315         else if (!rqst->assoc_cmd.ersp_ratio ||
1316                  (be16_to_cpu(rqst->assoc_cmd.ersp_ratio) >=
1317                                 be16_to_cpu(rqst->assoc_cmd.sqsize)))
1318                 ret = VERR_ERSP_RATIO;
1319
1320         else {
1321                 /* new association w/ admin queue */
1322                 iod->assoc = nvmet_fc_alloc_target_assoc(tgtport);
1323                 if (!iod->assoc)
1324                         ret = VERR_ASSOC_ALLOC_FAIL;
1325                 else {
1326                         queue = nvmet_fc_alloc_target_queue(iod->assoc, 0,
1327                                         be16_to_cpu(rqst->assoc_cmd.sqsize));
1328                         if (!queue)
1329                                 ret = VERR_QUEUE_ALLOC_FAIL;
1330                 }
1331         }
1332
1333         if (ret) {
1334                 dev_err(tgtport->dev,
1335                         "Create Association LS failed: %s\n",
1336                         validation_errors[ret]);
1337                 iod->lsreq->rsplen = nvmet_fc_format_rjt(acc,
1338                                 NVME_FC_MAX_LS_BUFFER_SIZE, rqst->w0.ls_cmd,
1339                                 FCNVME_RJT_RC_LOGIC,
1340                                 FCNVME_RJT_EXP_NONE, 0);
1341                 return;
1342         }
1343
1344         queue->ersp_ratio = be16_to_cpu(rqst->assoc_cmd.ersp_ratio);
1345         atomic_set(&queue->connected, 1);
1346         queue->sqhd = 0;        /* best place to init value */
1347
1348         /* format a response */
1349
1350         iod->lsreq->rsplen = sizeof(*acc);
1351
1352         nvmet_fc_format_rsp_hdr(acc, FCNVME_LS_ACC,
1353                         fcnvme_lsdesc_len(
1354                                 sizeof(struct fcnvme_ls_cr_assoc_acc)),
1355                         FCNVME_LS_CREATE_ASSOCIATION);
1356         acc->associd.desc_tag = cpu_to_be32(FCNVME_LSDESC_ASSOC_ID);
1357         acc->associd.desc_len =
1358                         fcnvme_lsdesc_len(
1359                                 sizeof(struct fcnvme_lsdesc_assoc_id));
1360         acc->associd.association_id =
1361                         cpu_to_be64(nvmet_fc_makeconnid(iod->assoc, 0));
1362         acc->connectid.desc_tag = cpu_to_be32(FCNVME_LSDESC_CONN_ID);
1363         acc->connectid.desc_len =
1364                         fcnvme_lsdesc_len(
1365                                 sizeof(struct fcnvme_lsdesc_conn_id));
1366         acc->connectid.connection_id = acc->associd.association_id;
1367 }
1368
1369 static void
1370 nvmet_fc_ls_create_connection(struct nvmet_fc_tgtport *tgtport,
1371                         struct nvmet_fc_ls_iod *iod)
1372 {
1373         struct fcnvme_ls_cr_conn_rqst *rqst =
1374                                 (struct fcnvme_ls_cr_conn_rqst *)iod->rqstbuf;
1375         struct fcnvme_ls_cr_conn_acc *acc =
1376                                 (struct fcnvme_ls_cr_conn_acc *)iod->rspbuf;
1377         struct nvmet_fc_tgt_queue *queue;
1378         int ret = 0;
1379
1380         memset(acc, 0, sizeof(*acc));
1381
1382         if (iod->rqstdatalen < sizeof(struct fcnvme_ls_cr_conn_rqst))
1383                 ret = VERR_CR_CONN_LEN;
1384         else if (rqst->desc_list_len !=
1385                         fcnvme_lsdesc_len(
1386                                 sizeof(struct fcnvme_ls_cr_conn_rqst)))
1387                 ret = VERR_CR_CONN_RQST_LEN;
1388         else if (rqst->associd.desc_tag != cpu_to_be32(FCNVME_LSDESC_ASSOC_ID))
1389                 ret = VERR_ASSOC_ID;
1390         else if (rqst->associd.desc_len !=
1391                         fcnvme_lsdesc_len(
1392                                 sizeof(struct fcnvme_lsdesc_assoc_id)))
1393                 ret = VERR_ASSOC_ID_LEN;
1394         else if (rqst->connect_cmd.desc_tag !=
1395                         cpu_to_be32(FCNVME_LSDESC_CREATE_CONN_CMD))
1396                 ret = VERR_CR_CONN_CMD;
1397         else if (rqst->connect_cmd.desc_len !=
1398                         fcnvme_lsdesc_len(
1399                                 sizeof(struct fcnvme_lsdesc_cr_conn_cmd)))
1400                 ret = VERR_CR_CONN_CMD_LEN;
1401         else if (!rqst->connect_cmd.ersp_ratio ||
1402                  (be16_to_cpu(rqst->connect_cmd.ersp_ratio) >=
1403                                 be16_to_cpu(rqst->connect_cmd.sqsize)))
1404                 ret = VERR_ERSP_RATIO;
1405
1406         else {
1407                 /* new io queue */
1408                 iod->assoc = nvmet_fc_find_target_assoc(tgtport,
1409                                 be64_to_cpu(rqst->associd.association_id));
1410                 if (!iod->assoc)
1411                         ret = VERR_NO_ASSOC;
1412                 else {
1413                         queue = nvmet_fc_alloc_target_queue(iod->assoc,
1414                                         be16_to_cpu(rqst->connect_cmd.qid),
1415                                         be16_to_cpu(rqst->connect_cmd.sqsize));
1416                         if (!queue)
1417                                 ret = VERR_QUEUE_ALLOC_FAIL;
1418
1419                         /* release get taken in nvmet_fc_find_target_assoc */
1420                         nvmet_fc_tgt_a_put(iod->assoc);
1421                 }
1422         }
1423
1424         if (ret) {
1425                 dev_err(tgtport->dev,
1426                         "Create Connection LS failed: %s\n",
1427                         validation_errors[ret]);
1428                 iod->lsreq->rsplen = nvmet_fc_format_rjt(acc,
1429                                 NVME_FC_MAX_LS_BUFFER_SIZE, rqst->w0.ls_cmd,
1430                                 (ret == VERR_NO_ASSOC) ?
1431                                         FCNVME_RJT_RC_INV_ASSOC :
1432                                         FCNVME_RJT_RC_LOGIC,
1433                                 FCNVME_RJT_EXP_NONE, 0);
1434                 return;
1435         }
1436
1437         queue->ersp_ratio = be16_to_cpu(rqst->connect_cmd.ersp_ratio);
1438         atomic_set(&queue->connected, 1);
1439         queue->sqhd = 0;        /* best place to init value */
1440
1441         /* format a response */
1442
1443         iod->lsreq->rsplen = sizeof(*acc);
1444
1445         nvmet_fc_format_rsp_hdr(acc, FCNVME_LS_ACC,
1446                         fcnvme_lsdesc_len(sizeof(struct fcnvme_ls_cr_conn_acc)),
1447                         FCNVME_LS_CREATE_CONNECTION);
1448         acc->connectid.desc_tag = cpu_to_be32(FCNVME_LSDESC_CONN_ID);
1449         acc->connectid.desc_len =
1450                         fcnvme_lsdesc_len(
1451                                 sizeof(struct fcnvme_lsdesc_conn_id));
1452         acc->connectid.connection_id =
1453                         cpu_to_be64(nvmet_fc_makeconnid(iod->assoc,
1454                                 be16_to_cpu(rqst->connect_cmd.qid)));
1455 }
1456
1457 static void
1458 nvmet_fc_ls_disconnect(struct nvmet_fc_tgtport *tgtport,
1459                         struct nvmet_fc_ls_iod *iod)
1460 {
1461         struct fcnvme_ls_disconnect_rqst *rqst =
1462                         (struct fcnvme_ls_disconnect_rqst *)iod->rqstbuf;
1463         struct fcnvme_ls_disconnect_acc *acc =
1464                         (struct fcnvme_ls_disconnect_acc *)iod->rspbuf;
1465         struct nvmet_fc_tgt_queue *queue = NULL;
1466         struct nvmet_fc_tgt_assoc *assoc;
1467         int ret = 0;
1468         bool del_assoc = false;
1469
1470         memset(acc, 0, sizeof(*acc));
1471
1472         if (iod->rqstdatalen < sizeof(struct fcnvme_ls_disconnect_rqst))
1473                 ret = VERR_DISCONN_LEN;
1474         else if (rqst->desc_list_len !=
1475                         fcnvme_lsdesc_len(
1476                                 sizeof(struct fcnvme_ls_disconnect_rqst)))
1477                 ret = VERR_DISCONN_RQST_LEN;
1478         else if (rqst->associd.desc_tag != cpu_to_be32(FCNVME_LSDESC_ASSOC_ID))
1479                 ret = VERR_ASSOC_ID;
1480         else if (rqst->associd.desc_len !=
1481                         fcnvme_lsdesc_len(
1482                                 sizeof(struct fcnvme_lsdesc_assoc_id)))
1483                 ret = VERR_ASSOC_ID_LEN;
1484         else if (rqst->discon_cmd.desc_tag !=
1485                         cpu_to_be32(FCNVME_LSDESC_DISCONN_CMD))
1486                 ret = VERR_DISCONN_CMD;
1487         else if (rqst->discon_cmd.desc_len !=
1488                         fcnvme_lsdesc_len(
1489                                 sizeof(struct fcnvme_lsdesc_disconn_cmd)))
1490                 ret = VERR_DISCONN_CMD_LEN;
1491         else if ((rqst->discon_cmd.scope != FCNVME_DISCONN_ASSOCIATION) &&
1492                         (rqst->discon_cmd.scope != FCNVME_DISCONN_CONNECTION))
1493                 ret = VERR_DISCONN_SCOPE;
1494         else {
1495                 /* match an active association */
1496                 assoc = nvmet_fc_find_target_assoc(tgtport,
1497                                 be64_to_cpu(rqst->associd.association_id));
1498                 iod->assoc = assoc;
1499                 if (assoc) {
1500                         if (rqst->discon_cmd.scope ==
1501                                         FCNVME_DISCONN_CONNECTION) {
1502                                 queue = nvmet_fc_find_target_queue(tgtport,
1503                                                 be64_to_cpu(
1504                                                         rqst->discon_cmd.id));
1505                                 if (!queue) {
1506                                         nvmet_fc_tgt_a_put(assoc);
1507                                         ret = VERR_NO_CONN;
1508                                 }
1509                         }
1510                 } else
1511                         ret = VERR_NO_ASSOC;
1512         }
1513
1514         if (ret) {
1515                 dev_err(tgtport->dev,
1516                         "Disconnect LS failed: %s\n",
1517                         validation_errors[ret]);
1518                 iod->lsreq->rsplen = nvmet_fc_format_rjt(acc,
1519                                 NVME_FC_MAX_LS_BUFFER_SIZE, rqst->w0.ls_cmd,
1520                                 (ret == VERR_NO_ASSOC) ?
1521                                         FCNVME_RJT_RC_INV_ASSOC :
1522                                         (ret == VERR_NO_CONN) ?
1523                                                 FCNVME_RJT_RC_INV_CONN :
1524                                                 FCNVME_RJT_RC_LOGIC,
1525                                 FCNVME_RJT_EXP_NONE, 0);
1526                 return;
1527         }
1528
1529         /* format a response */
1530
1531         iod->lsreq->rsplen = sizeof(*acc);
1532
1533         nvmet_fc_format_rsp_hdr(acc, FCNVME_LS_ACC,
1534                         fcnvme_lsdesc_len(
1535                                 sizeof(struct fcnvme_ls_disconnect_acc)),
1536                         FCNVME_LS_DISCONNECT);
1537
1538
1539         /* are we to delete a Connection ID (queue) */
1540         if (queue) {
1541                 int qid = queue->qid;
1542
1543                 nvmet_fc_delete_target_queue(queue);
1544
1545                 /* release the get taken by find_target_queue */
1546                 nvmet_fc_tgt_q_put(queue);
1547
1548                 /* tear association down if io queue terminated */
1549                 if (!qid)
1550                         del_assoc = true;
1551         }
1552
1553         /* release get taken in nvmet_fc_find_target_assoc */
1554         nvmet_fc_tgt_a_put(iod->assoc);
1555
1556         if (del_assoc)
1557                 nvmet_fc_delete_target_assoc(iod->assoc);
1558 }
1559
1560
1561 /* *********************** NVME Ctrl Routines **************************** */
1562
1563
1564 static void nvmet_fc_fcp_nvme_cmd_done(struct nvmet_req *nvme_req);
1565
1566 static const struct nvmet_fabrics_ops nvmet_fc_tgt_fcp_ops;
1567
1568 static void
1569 nvmet_fc_xmt_ls_rsp_done(struct nvmefc_tgt_ls_req *lsreq)
1570 {
1571         struct nvmet_fc_ls_iod *iod = lsreq->nvmet_fc_private;
1572         struct nvmet_fc_tgtport *tgtport = iod->tgtport;
1573
1574         fc_dma_sync_single_for_cpu(tgtport->dev, iod->rspdma,
1575                                 NVME_FC_MAX_LS_BUFFER_SIZE, DMA_TO_DEVICE);
1576         nvmet_fc_free_ls_iod(tgtport, iod);
1577         nvmet_fc_tgtport_put(tgtport);
1578 }
1579
1580 static void
1581 nvmet_fc_xmt_ls_rsp(struct nvmet_fc_tgtport *tgtport,
1582                                 struct nvmet_fc_ls_iod *iod)
1583 {
1584         int ret;
1585
1586         fc_dma_sync_single_for_device(tgtport->dev, iod->rspdma,
1587                                   NVME_FC_MAX_LS_BUFFER_SIZE, DMA_TO_DEVICE);
1588
1589         ret = tgtport->ops->xmt_ls_rsp(&tgtport->fc_target_port, iod->lsreq);
1590         if (ret)
1591                 nvmet_fc_xmt_ls_rsp_done(iod->lsreq);
1592 }
1593
1594 /*
1595  * Actual processing routine for received FC-NVME LS Requests from the LLD
1596  */
1597 static void
1598 nvmet_fc_handle_ls_rqst(struct nvmet_fc_tgtport *tgtport,
1599                         struct nvmet_fc_ls_iod *iod)
1600 {
1601         struct fcnvme_ls_rqst_w0 *w0 =
1602                         (struct fcnvme_ls_rqst_w0 *)iod->rqstbuf;
1603
1604         iod->lsreq->nvmet_fc_private = iod;
1605         iod->lsreq->rspbuf = iod->rspbuf;
1606         iod->lsreq->rspdma = iod->rspdma;
1607         iod->lsreq->done = nvmet_fc_xmt_ls_rsp_done;
1608         /* Be preventative. handlers will later set to valid length */
1609         iod->lsreq->rsplen = 0;
1610
1611         iod->assoc = NULL;
1612
1613         /*
1614          * handlers:
1615          *   parse request input, execute the request, and format the
1616          *   LS response
1617          */
1618         switch (w0->ls_cmd) {
1619         case FCNVME_LS_CREATE_ASSOCIATION:
1620                 /* Creates Association and initial Admin Queue/Connection */
1621                 nvmet_fc_ls_create_association(tgtport, iod);
1622                 break;
1623         case FCNVME_LS_CREATE_CONNECTION:
1624                 /* Creates an IO Queue/Connection */
1625                 nvmet_fc_ls_create_connection(tgtport, iod);
1626                 break;
1627         case FCNVME_LS_DISCONNECT:
1628                 /* Terminate a Queue/Connection or the Association */
1629                 nvmet_fc_ls_disconnect(tgtport, iod);
1630                 break;
1631         default:
1632                 iod->lsreq->rsplen = nvmet_fc_format_rjt(iod->rspbuf,
1633                                 NVME_FC_MAX_LS_BUFFER_SIZE, w0->ls_cmd,
1634                                 FCNVME_RJT_RC_INVAL, FCNVME_RJT_EXP_NONE, 0);
1635         }
1636
1637         nvmet_fc_xmt_ls_rsp(tgtport, iod);
1638 }
1639
1640 /*
1641  * Actual processing routine for received FC-NVME LS Requests from the LLD
1642  */
1643 static void
1644 nvmet_fc_handle_ls_rqst_work(struct work_struct *work)
1645 {
1646         struct nvmet_fc_ls_iod *iod =
1647                 container_of(work, struct nvmet_fc_ls_iod, work);
1648         struct nvmet_fc_tgtport *tgtport = iod->tgtport;
1649
1650         nvmet_fc_handle_ls_rqst(tgtport, iod);
1651 }
1652
1653
1654 /**
1655  * nvmet_fc_rcv_ls_req - transport entry point called by an LLDD
1656  *                       upon the reception of a NVME LS request.
1657  *
1658  * The nvmet-fc layer will copy payload to an internal structure for
1659  * processing.  As such, upon completion of the routine, the LLDD may
1660  * immediately free/reuse the LS request buffer passed in the call.
1661  *
1662  * If this routine returns error, the LLDD should abort the exchange.
1663  *
1664  * @tgtport:    pointer to the (registered) target port the LS was
1665  *              received on.
1666  * @lsreq:      pointer to a lsreq request structure to be used to reference
1667  *              the exchange corresponding to the LS.
1668  * @lsreqbuf:   pointer to the buffer containing the LS Request
1669  * @lsreqbuf_len: length, in bytes, of the received LS request
1670  */
1671 int
1672 nvmet_fc_rcv_ls_req(struct nvmet_fc_target_port *target_port,
1673                         struct nvmefc_tgt_ls_req *lsreq,
1674                         void *lsreqbuf, u32 lsreqbuf_len)
1675 {
1676         struct nvmet_fc_tgtport *tgtport = targetport_to_tgtport(target_port);
1677         struct nvmet_fc_ls_iod *iod;
1678
1679         if (lsreqbuf_len > NVME_FC_MAX_LS_BUFFER_SIZE)
1680                 return -E2BIG;
1681
1682         if (!nvmet_fc_tgtport_get(tgtport))
1683                 return -ESHUTDOWN;
1684
1685         iod = nvmet_fc_alloc_ls_iod(tgtport);
1686         if (!iod) {
1687                 nvmet_fc_tgtport_put(tgtport);
1688                 return -ENOENT;
1689         }
1690
1691         iod->lsreq = lsreq;
1692         iod->fcpreq = NULL;
1693         memcpy(iod->rqstbuf, lsreqbuf, lsreqbuf_len);
1694         iod->rqstdatalen = lsreqbuf_len;
1695
1696         schedule_work(&iod->work);
1697
1698         return 0;
1699 }
1700 EXPORT_SYMBOL_GPL(nvmet_fc_rcv_ls_req);
1701
1702
1703 /*
1704  * **********************
1705  * Start of FCP handling
1706  * **********************
1707  */
1708
1709 static int
1710 nvmet_fc_alloc_tgt_pgs(struct nvmet_fc_fcp_iod *fod)
1711 {
1712         struct scatterlist *sg;
1713         unsigned int nent;
1714
1715         sg = sgl_alloc(fod->req.transfer_len, GFP_KERNEL, &nent);
1716         if (!sg)
1717                 goto out;
1718
1719         fod->data_sg = sg;
1720         fod->data_sg_cnt = nent;
1721         fod->data_sg_cnt = fc_dma_map_sg(fod->tgtport->dev, sg, nent,
1722                                 ((fod->io_dir == NVMET_FCP_WRITE) ?
1723                                         DMA_FROM_DEVICE : DMA_TO_DEVICE));
1724                                 /* note: write from initiator perspective */
1725         fod->next_sg = fod->data_sg;
1726
1727         return 0;
1728
1729 out:
1730         return NVME_SC_INTERNAL;
1731 }
1732
1733 static void
1734 nvmet_fc_free_tgt_pgs(struct nvmet_fc_fcp_iod *fod)
1735 {
1736         if (!fod->data_sg || !fod->data_sg_cnt)
1737                 return;
1738
1739         fc_dma_unmap_sg(fod->tgtport->dev, fod->data_sg, fod->data_sg_cnt,
1740                                 ((fod->io_dir == NVMET_FCP_WRITE) ?
1741                                         DMA_FROM_DEVICE : DMA_TO_DEVICE));
1742         sgl_free(fod->data_sg);
1743         fod->data_sg = NULL;
1744         fod->data_sg_cnt = 0;
1745 }
1746
1747
1748 static bool
1749 queue_90percent_full(struct nvmet_fc_tgt_queue *q, u32 sqhd)
1750 {
1751         u32 sqtail, used;
1752
1753         /* egad, this is ugly. And sqtail is just a best guess */
1754         sqtail = atomic_read(&q->sqtail) % q->sqsize;
1755
1756         used = (sqtail < sqhd) ? (sqtail + q->sqsize - sqhd) : (sqtail - sqhd);
1757         return ((used * 10) >= (((u32)(q->sqsize - 1) * 9)));
1758 }
1759
1760 /*
1761  * Prep RSP payload.
1762  * May be a NVMET_FCOP_RSP or NVMET_FCOP_READDATA_RSP op
1763  */
1764 static void
1765 nvmet_fc_prep_fcp_rsp(struct nvmet_fc_tgtport *tgtport,
1766                                 struct nvmet_fc_fcp_iod *fod)
1767 {
1768         struct nvme_fc_ersp_iu *ersp = &fod->rspiubuf;
1769         struct nvme_common_command *sqe = &fod->cmdiubuf.sqe.common;
1770         struct nvme_completion *cqe = &ersp->cqe;
1771         u32 *cqewd = (u32 *)cqe;
1772         bool send_ersp = false;
1773         u32 rsn, rspcnt, xfr_length;
1774
1775         if (fod->fcpreq->op == NVMET_FCOP_READDATA_RSP)
1776                 xfr_length = fod->req.transfer_len;
1777         else
1778                 xfr_length = fod->offset;
1779
1780         /*
1781          * check to see if we can send a 0's rsp.
1782          *   Note: to send a 0's response, the NVME-FC host transport will
1783          *   recreate the CQE. The host transport knows: sq id, SQHD (last
1784          *   seen in an ersp), and command_id. Thus it will create a
1785          *   zero-filled CQE with those known fields filled in. Transport
1786          *   must send an ersp for any condition where the cqe won't match
1787          *   this.
1788          *
1789          * Here are the FC-NVME mandated cases where we must send an ersp:
1790          *  every N responses, where N=ersp_ratio
1791          *  force fabric commands to send ersp's (not in FC-NVME but good
1792          *    practice)
1793          *  normal cmds: any time status is non-zero, or status is zero
1794          *     but words 0 or 1 are non-zero.
1795          *  the SQ is 90% or more full
1796          *  the cmd is a fused command
1797          *  transferred data length not equal to cmd iu length
1798          */
1799         rspcnt = atomic_inc_return(&fod->queue->zrspcnt);
1800         if (!(rspcnt % fod->queue->ersp_ratio) ||
1801             sqe->opcode == nvme_fabrics_command ||
1802             xfr_length != fod->req.transfer_len ||
1803             (le16_to_cpu(cqe->status) & 0xFFFE) || cqewd[0] || cqewd[1] ||
1804             (sqe->flags & (NVME_CMD_FUSE_FIRST | NVME_CMD_FUSE_SECOND)) ||
1805             queue_90percent_full(fod->queue, le16_to_cpu(cqe->sq_head)))
1806                 send_ersp = true;
1807
1808         /* re-set the fields */
1809         fod->fcpreq->rspaddr = ersp;
1810         fod->fcpreq->rspdma = fod->rspdma;
1811
1812         if (!send_ersp) {
1813                 memset(ersp, 0, NVME_FC_SIZEOF_ZEROS_RSP);
1814                 fod->fcpreq->rsplen = NVME_FC_SIZEOF_ZEROS_RSP;
1815         } else {
1816                 ersp->iu_len = cpu_to_be16(sizeof(*ersp)/sizeof(u32));
1817                 rsn = atomic_inc_return(&fod->queue->rsn);
1818                 ersp->rsn = cpu_to_be32(rsn);
1819                 ersp->xfrd_len = cpu_to_be32(xfr_length);
1820                 fod->fcpreq->rsplen = sizeof(*ersp);
1821         }
1822
1823         fc_dma_sync_single_for_device(tgtport->dev, fod->rspdma,
1824                                   sizeof(fod->rspiubuf), DMA_TO_DEVICE);
1825 }
1826
1827 static void nvmet_fc_xmt_fcp_op_done(struct nvmefc_tgt_fcp_req *fcpreq);
1828
1829 static void
1830 nvmet_fc_abort_op(struct nvmet_fc_tgtport *tgtport,
1831                                 struct nvmet_fc_fcp_iod *fod)
1832 {
1833         struct nvmefc_tgt_fcp_req *fcpreq = fod->fcpreq;
1834
1835         /* data no longer needed */
1836         nvmet_fc_free_tgt_pgs(fod);
1837
1838         /*
1839          * if an ABTS was received or we issued the fcp_abort early
1840          * don't call abort routine again.
1841          */
1842         /* no need to take lock - lock was taken earlier to get here */
1843         if (!fod->aborted)
1844                 tgtport->ops->fcp_abort(&tgtport->fc_target_port, fcpreq);
1845
1846         nvmet_fc_free_fcp_iod(fod->queue, fod);
1847 }
1848
1849 static void
1850 nvmet_fc_xmt_fcp_rsp(struct nvmet_fc_tgtport *tgtport,
1851                                 struct nvmet_fc_fcp_iod *fod)
1852 {
1853         int ret;
1854
1855         fod->fcpreq->op = NVMET_FCOP_RSP;
1856         fod->fcpreq->timeout = 0;
1857
1858         nvmet_fc_prep_fcp_rsp(tgtport, fod);
1859
1860         ret = tgtport->ops->fcp_op(&tgtport->fc_target_port, fod->fcpreq);
1861         if (ret)
1862                 nvmet_fc_abort_op(tgtport, fod);
1863 }
1864
1865 static void
1866 nvmet_fc_transfer_fcp_data(struct nvmet_fc_tgtport *tgtport,
1867                                 struct nvmet_fc_fcp_iod *fod, u8 op)
1868 {
1869         struct nvmefc_tgt_fcp_req *fcpreq = fod->fcpreq;
1870         struct scatterlist *sg = fod->next_sg;
1871         unsigned long flags;
1872         u32 remaininglen = fod->req.transfer_len - fod->offset;
1873         u32 tlen = 0;
1874         int ret;
1875
1876         fcpreq->op = op;
1877         fcpreq->offset = fod->offset;
1878         fcpreq->timeout = NVME_FC_TGTOP_TIMEOUT_SEC;
1879
1880         /*
1881          * for next sequence:
1882          *  break at a sg element boundary
1883          *  attempt to keep sequence length capped at
1884          *    NVMET_FC_MAX_SEQ_LENGTH but allow sequence to
1885          *    be longer if a single sg element is larger
1886          *    than that amount. This is done to avoid creating
1887          *    a new sg list to use for the tgtport api.
1888          */
1889         fcpreq->sg = sg;
1890         fcpreq->sg_cnt = 0;
1891         while (tlen < remaininglen &&
1892                fcpreq->sg_cnt < tgtport->max_sg_cnt &&
1893                tlen + sg_dma_len(sg) < NVMET_FC_MAX_SEQ_LENGTH) {
1894                 fcpreq->sg_cnt++;
1895                 tlen += sg_dma_len(sg);
1896                 sg = sg_next(sg);
1897         }
1898         if (tlen < remaininglen && fcpreq->sg_cnt == 0) {
1899                 fcpreq->sg_cnt++;
1900                 tlen += min_t(u32, sg_dma_len(sg), remaininglen);
1901                 sg = sg_next(sg);
1902         }
1903         if (tlen < remaininglen)
1904                 fod->next_sg = sg;
1905         else
1906                 fod->next_sg = NULL;
1907
1908         fcpreq->transfer_length = tlen;
1909         fcpreq->transferred_length = 0;
1910         fcpreq->fcp_error = 0;
1911         fcpreq->rsplen = 0;
1912
1913         /*
1914          * If the last READDATA request: check if LLDD supports
1915          * combined xfr with response.
1916          */
1917         if ((op == NVMET_FCOP_READDATA) &&
1918             ((fod->offset + fcpreq->transfer_length) == fod->req.transfer_len) &&
1919             (tgtport->ops->target_features & NVMET_FCTGTFEAT_READDATA_RSP)) {
1920                 fcpreq->op = NVMET_FCOP_READDATA_RSP;
1921                 nvmet_fc_prep_fcp_rsp(tgtport, fod);
1922         }
1923
1924         ret = tgtport->ops->fcp_op(&tgtport->fc_target_port, fod->fcpreq);
1925         if (ret) {
1926                 /*
1927                  * should be ok to set w/o lock as its in the thread of
1928                  * execution (not an async timer routine) and doesn't
1929                  * contend with any clearing action
1930                  */
1931                 fod->abort = true;
1932
1933                 if (op == NVMET_FCOP_WRITEDATA) {
1934                         spin_lock_irqsave(&fod->flock, flags);
1935                         fod->writedataactive = false;
1936                         spin_unlock_irqrestore(&fod->flock, flags);
1937                         nvmet_req_complete(&fod->req, NVME_SC_INTERNAL);
1938                 } else /* NVMET_FCOP_READDATA or NVMET_FCOP_READDATA_RSP */ {
1939                         fcpreq->fcp_error = ret;
1940                         fcpreq->transferred_length = 0;
1941                         nvmet_fc_xmt_fcp_op_done(fod->fcpreq);
1942                 }
1943         }
1944 }
1945
1946 static inline bool
1947 __nvmet_fc_fod_op_abort(struct nvmet_fc_fcp_iod *fod, bool abort)
1948 {
1949         struct nvmefc_tgt_fcp_req *fcpreq = fod->fcpreq;
1950         struct nvmet_fc_tgtport *tgtport = fod->tgtport;
1951
1952         /* if in the middle of an io and we need to tear down */
1953         if (abort) {
1954                 if (fcpreq->op == NVMET_FCOP_WRITEDATA) {
1955                         nvmet_req_complete(&fod->req, NVME_SC_INTERNAL);
1956                         return true;
1957                 }
1958
1959                 nvmet_fc_abort_op(tgtport, fod);
1960                 return true;
1961         }
1962
1963         return false;
1964 }
1965
1966 /*
1967  * actual done handler for FCP operations when completed by the lldd
1968  */
1969 static void
1970 nvmet_fc_fod_op_done(struct nvmet_fc_fcp_iod *fod)
1971 {
1972         struct nvmefc_tgt_fcp_req *fcpreq = fod->fcpreq;
1973         struct nvmet_fc_tgtport *tgtport = fod->tgtport;
1974         unsigned long flags;
1975         bool abort;
1976
1977         spin_lock_irqsave(&fod->flock, flags);
1978         abort = fod->abort;
1979         fod->writedataactive = false;
1980         spin_unlock_irqrestore(&fod->flock, flags);
1981
1982         switch (fcpreq->op) {
1983
1984         case NVMET_FCOP_WRITEDATA:
1985                 if (__nvmet_fc_fod_op_abort(fod, abort))
1986                         return;
1987                 if (fcpreq->fcp_error ||
1988                     fcpreq->transferred_length != fcpreq->transfer_length) {
1989                         spin_lock_irqsave(&fod->flock, flags);
1990                         fod->abort = true;
1991                         spin_unlock_irqrestore(&fod->flock, flags);
1992
1993                         nvmet_req_complete(&fod->req, NVME_SC_INTERNAL);
1994                         return;
1995                 }
1996
1997                 fod->offset += fcpreq->transferred_length;
1998                 if (fod->offset != fod->req.transfer_len) {
1999                         spin_lock_irqsave(&fod->flock, flags);
2000                         fod->writedataactive = true;
2001                         spin_unlock_irqrestore(&fod->flock, flags);
2002
2003                         /* transfer the next chunk */
2004                         nvmet_fc_transfer_fcp_data(tgtport, fod,
2005                                                 NVMET_FCOP_WRITEDATA);
2006                         return;
2007                 }
2008
2009                 /* data transfer complete, resume with nvmet layer */
2010                 nvmet_req_execute(&fod->req);
2011                 break;
2012
2013         case NVMET_FCOP_READDATA:
2014         case NVMET_FCOP_READDATA_RSP:
2015                 if (__nvmet_fc_fod_op_abort(fod, abort))
2016                         return;
2017                 if (fcpreq->fcp_error ||
2018                     fcpreq->transferred_length != fcpreq->transfer_length) {
2019                         nvmet_fc_abort_op(tgtport, fod);
2020                         return;
2021                 }
2022
2023                 /* success */
2024
2025                 if (fcpreq->op == NVMET_FCOP_READDATA_RSP) {
2026                         /* data no longer needed */
2027                         nvmet_fc_free_tgt_pgs(fod);
2028                         nvmet_fc_free_fcp_iod(fod->queue, fod);
2029                         return;
2030                 }
2031
2032                 fod->offset += fcpreq->transferred_length;
2033                 if (fod->offset != fod->req.transfer_len) {
2034                         /* transfer the next chunk */
2035                         nvmet_fc_transfer_fcp_data(tgtport, fod,
2036                                                 NVMET_FCOP_READDATA);
2037                         return;
2038                 }
2039
2040                 /* data transfer complete, send response */
2041
2042                 /* data no longer needed */
2043                 nvmet_fc_free_tgt_pgs(fod);
2044
2045                 nvmet_fc_xmt_fcp_rsp(tgtport, fod);
2046
2047                 break;
2048
2049         case NVMET_FCOP_RSP:
2050                 if (__nvmet_fc_fod_op_abort(fod, abort))
2051                         return;
2052                 nvmet_fc_free_fcp_iod(fod->queue, fod);
2053                 break;
2054
2055         default:
2056                 break;
2057         }
2058 }
2059
2060 static void
2061 nvmet_fc_fcp_rqst_op_done_work(struct work_struct *work)
2062 {
2063         struct nvmet_fc_fcp_iod *fod =
2064                 container_of(work, struct nvmet_fc_fcp_iod, done_work);
2065
2066         nvmet_fc_fod_op_done(fod);
2067 }
2068
2069 static void
2070 nvmet_fc_xmt_fcp_op_done(struct nvmefc_tgt_fcp_req *fcpreq)
2071 {
2072         struct nvmet_fc_fcp_iod *fod = fcpreq->nvmet_fc_private;
2073         struct nvmet_fc_tgt_queue *queue = fod->queue;
2074
2075         if (fod->tgtport->ops->target_features & NVMET_FCTGTFEAT_OPDONE_IN_ISR)
2076                 /* context switch so completion is not in ISR context */
2077                 queue_work_on(queue->cpu, queue->work_q, &fod->done_work);
2078         else
2079                 nvmet_fc_fod_op_done(fod);
2080 }
2081
2082 /*
2083  * actual completion handler after execution by the nvmet layer
2084  */
2085 static void
2086 __nvmet_fc_fcp_nvme_cmd_done(struct nvmet_fc_tgtport *tgtport,
2087                         struct nvmet_fc_fcp_iod *fod, int status)
2088 {
2089         struct nvme_common_command *sqe = &fod->cmdiubuf.sqe.common;
2090         struct nvme_completion *cqe = &fod->rspiubuf.cqe;
2091         unsigned long flags;
2092         bool abort;
2093
2094         spin_lock_irqsave(&fod->flock, flags);
2095         abort = fod->abort;
2096         spin_unlock_irqrestore(&fod->flock, flags);
2097
2098         /* if we have a CQE, snoop the last sq_head value */
2099         if (!status)
2100                 fod->queue->sqhd = cqe->sq_head;
2101
2102         if (abort) {
2103                 nvmet_fc_abort_op(tgtport, fod);
2104                 return;
2105         }
2106
2107         /* if an error handling the cmd post initial parsing */
2108         if (status) {
2109                 /* fudge up a failed CQE status for our transport error */
2110                 memset(cqe, 0, sizeof(*cqe));
2111                 cqe->sq_head = fod->queue->sqhd;        /* echo last cqe sqhd */
2112                 cqe->sq_id = cpu_to_le16(fod->queue->qid);
2113                 cqe->command_id = sqe->command_id;
2114                 cqe->status = cpu_to_le16(status);
2115         } else {
2116
2117                 /*
2118                  * try to push the data even if the SQE status is non-zero.
2119                  * There may be a status where data still was intended to
2120                  * be moved
2121                  */
2122                 if ((fod->io_dir == NVMET_FCP_READ) && (fod->data_sg_cnt)) {
2123                         /* push the data over before sending rsp */
2124                         nvmet_fc_transfer_fcp_data(tgtport, fod,
2125                                                 NVMET_FCOP_READDATA);
2126                         return;
2127                 }
2128
2129                 /* writes & no data - fall thru */
2130         }
2131
2132         /* data no longer needed */
2133         nvmet_fc_free_tgt_pgs(fod);
2134
2135         nvmet_fc_xmt_fcp_rsp(tgtport, fod);
2136 }
2137
2138
2139 static void
2140 nvmet_fc_fcp_nvme_cmd_done(struct nvmet_req *nvme_req)
2141 {
2142         struct nvmet_fc_fcp_iod *fod = nvmet_req_to_fod(nvme_req);
2143         struct nvmet_fc_tgtport *tgtport = fod->tgtport;
2144
2145         __nvmet_fc_fcp_nvme_cmd_done(tgtport, fod, 0);
2146 }
2147
2148
2149 /*
2150  * Actual processing routine for received FC-NVME LS Requests from the LLD
2151  */
2152 static void
2153 nvmet_fc_handle_fcp_rqst(struct nvmet_fc_tgtport *tgtport,
2154                         struct nvmet_fc_fcp_iod *fod)
2155 {
2156         struct nvme_fc_cmd_iu *cmdiu = &fod->cmdiubuf;
2157         u32 xfrlen = be32_to_cpu(cmdiu->data_len);
2158         int ret;
2159
2160         /*
2161          * Fused commands are currently not supported in the linux
2162          * implementation.
2163          *
2164          * As such, the implementation of the FC transport does not
2165          * look at the fused commands and order delivery to the upper
2166          * layer until we have both based on csn.
2167          */
2168
2169         fod->fcpreq->done = nvmet_fc_xmt_fcp_op_done;
2170
2171         if (cmdiu->flags & FCNVME_CMD_FLAGS_WRITE) {
2172                 fod->io_dir = NVMET_FCP_WRITE;
2173                 if (!nvme_is_write(&cmdiu->sqe))
2174                         goto transport_error;
2175         } else if (cmdiu->flags & FCNVME_CMD_FLAGS_READ) {
2176                 fod->io_dir = NVMET_FCP_READ;
2177                 if (nvme_is_write(&cmdiu->sqe))
2178                         goto transport_error;
2179         } else {
2180                 fod->io_dir = NVMET_FCP_NODATA;
2181                 if (xfrlen)
2182                         goto transport_error;
2183         }
2184
2185         fod->req.cmd = &fod->cmdiubuf.sqe;
2186         fod->req.rsp = &fod->rspiubuf.cqe;
2187         fod->req.port = fod->queue->port;
2188
2189         /* clear any response payload */
2190         memset(&fod->rspiubuf, 0, sizeof(fod->rspiubuf));
2191
2192         fod->data_sg = NULL;
2193         fod->data_sg_cnt = 0;
2194
2195         ret = nvmet_req_init(&fod->req,
2196                                 &fod->queue->nvme_cq,
2197                                 &fod->queue->nvme_sq,
2198                                 &nvmet_fc_tgt_fcp_ops);
2199         if (!ret) {
2200                 /* bad SQE content or invalid ctrl state */
2201                 /* nvmet layer has already called op done to send rsp. */
2202                 return;
2203         }
2204
2205         fod->req.transfer_len = xfrlen;
2206
2207         /* keep a running counter of tail position */
2208         atomic_inc(&fod->queue->sqtail);
2209
2210         if (fod->req.transfer_len) {
2211                 ret = nvmet_fc_alloc_tgt_pgs(fod);
2212                 if (ret) {
2213                         nvmet_req_complete(&fod->req, ret);
2214                         return;
2215                 }
2216         }
2217         fod->req.sg = fod->data_sg;
2218         fod->req.sg_cnt = fod->data_sg_cnt;
2219         fod->offset = 0;
2220
2221         if (fod->io_dir == NVMET_FCP_WRITE) {
2222                 /* pull the data over before invoking nvmet layer */
2223                 nvmet_fc_transfer_fcp_data(tgtport, fod, NVMET_FCOP_WRITEDATA);
2224                 return;
2225         }
2226
2227         /*
2228          * Reads or no data:
2229          *
2230          * can invoke the nvmet_layer now. If read data, cmd completion will
2231          * push the data
2232          */
2233         nvmet_req_execute(&fod->req);
2234         return;
2235
2236 transport_error:
2237         nvmet_fc_abort_op(tgtport, fod);
2238 }
2239
2240 /*
2241  * Actual processing routine for received FC-NVME LS Requests from the LLD
2242  */
2243 static void
2244 nvmet_fc_handle_fcp_rqst_work(struct work_struct *work)
2245 {
2246         struct nvmet_fc_fcp_iod *fod =
2247                 container_of(work, struct nvmet_fc_fcp_iod, work);
2248         struct nvmet_fc_tgtport *tgtport = fod->tgtport;
2249
2250         nvmet_fc_handle_fcp_rqst(tgtport, fod);
2251 }
2252
2253 /**
2254  * nvmet_fc_rcv_fcp_req - transport entry point called by an LLDD
2255  *                       upon the reception of a NVME FCP CMD IU.
2256  *
2257  * Pass a FC-NVME FCP CMD IU received from the FC link to the nvmet-fc
2258  * layer for processing.
2259  *
2260  * The nvmet_fc layer allocates a local job structure (struct
2261  * nvmet_fc_fcp_iod) from the queue for the io and copies the
2262  * CMD IU buffer to the job structure. As such, on a successful
2263  * completion (returns 0), the LLDD may immediately free/reuse
2264  * the CMD IU buffer passed in the call.
2265  *
2266  * However, in some circumstances, due to the packetized nature of FC
2267  * and the api of the FC LLDD which may issue a hw command to send the
2268  * response, but the LLDD may not get the hw completion for that command
2269  * and upcall the nvmet_fc layer before a new command may be
2270  * asynchronously received - its possible for a command to be received
2271  * before the LLDD and nvmet_fc have recycled the job structure. It gives
2272  * the appearance of more commands received than fits in the sq.
2273  * To alleviate this scenario, a temporary queue is maintained in the
2274  * transport for pending LLDD requests waiting for a queue job structure.
2275  * In these "overrun" cases, a temporary queue element is allocated
2276  * the LLDD request and CMD iu buffer information remembered, and the
2277  * routine returns a -EOVERFLOW status. Subsequently, when a queue job
2278  * structure is freed, it is immediately reallocated for anything on the
2279  * pending request list. The LLDDs defer_rcv() callback is called,
2280  * informing the LLDD that it may reuse the CMD IU buffer, and the io
2281  * is then started normally with the transport.
2282  *
2283  * The LLDD, when receiving an -EOVERFLOW completion status, is to treat
2284  * the completion as successful but must not reuse the CMD IU buffer
2285  * until the LLDD's defer_rcv() callback has been called for the
2286  * corresponding struct nvmefc_tgt_fcp_req pointer.
2287  *
2288  * If there is any other condition in which an error occurs, the
2289  * transport will return a non-zero status indicating the error.
2290  * In all cases other than -EOVERFLOW, the transport has not accepted the
2291  * request and the LLDD should abort the exchange.
2292  *
2293  * @target_port: pointer to the (registered) target port the FCP CMD IU
2294  *              was received on.
2295  * @fcpreq:     pointer to a fcpreq request structure to be used to reference
2296  *              the exchange corresponding to the FCP Exchange.
2297  * @cmdiubuf:   pointer to the buffer containing the FCP CMD IU
2298  * @cmdiubuf_len: length, in bytes, of the received FCP CMD IU
2299  */
2300 int
2301 nvmet_fc_rcv_fcp_req(struct nvmet_fc_target_port *target_port,
2302                         struct nvmefc_tgt_fcp_req *fcpreq,
2303                         void *cmdiubuf, u32 cmdiubuf_len)
2304 {
2305         struct nvmet_fc_tgtport *tgtport = targetport_to_tgtport(target_port);
2306         struct nvme_fc_cmd_iu *cmdiu = cmdiubuf;
2307         struct nvmet_fc_tgt_queue *queue;
2308         struct nvmet_fc_fcp_iod *fod;
2309         struct nvmet_fc_defer_fcp_req *deferfcp;
2310         unsigned long flags;
2311
2312         /* validate iu, so the connection id can be used to find the queue */
2313         if ((cmdiubuf_len != sizeof(*cmdiu)) ||
2314                         (cmdiu->scsi_id != NVME_CMD_SCSI_ID) ||
2315                         (cmdiu->fc_id != NVME_CMD_FC_ID) ||
2316                         (be16_to_cpu(cmdiu->iu_len) != (sizeof(*cmdiu)/4)))
2317                 return -EIO;
2318
2319         queue = nvmet_fc_find_target_queue(tgtport,
2320                                 be64_to_cpu(cmdiu->connection_id));
2321         if (!queue)
2322                 return -ENOTCONN;
2323
2324         /*
2325          * note: reference taken by find_target_queue
2326          * After successful fod allocation, the fod will inherit the
2327          * ownership of that reference and will remove the reference
2328          * when the fod is freed.
2329          */
2330
2331         spin_lock_irqsave(&queue->qlock, flags);
2332
2333         fod = nvmet_fc_alloc_fcp_iod(queue);
2334         if (fod) {
2335                 spin_unlock_irqrestore(&queue->qlock, flags);
2336
2337                 fcpreq->nvmet_fc_private = fod;
2338                 fod->fcpreq = fcpreq;
2339
2340                 memcpy(&fod->cmdiubuf, cmdiubuf, cmdiubuf_len);
2341
2342                 nvmet_fc_queue_fcp_req(tgtport, queue, fcpreq);
2343
2344                 return 0;
2345         }
2346
2347         if (!tgtport->ops->defer_rcv) {
2348                 spin_unlock_irqrestore(&queue->qlock, flags);
2349                 /* release the queue lookup reference */
2350                 nvmet_fc_tgt_q_put(queue);
2351                 return -ENOENT;
2352         }
2353
2354         deferfcp = list_first_entry_or_null(&queue->avail_defer_list,
2355                         struct nvmet_fc_defer_fcp_req, req_list);
2356         if (deferfcp) {
2357                 /* Just re-use one that was previously allocated */
2358                 list_del(&deferfcp->req_list);
2359         } else {
2360                 spin_unlock_irqrestore(&queue->qlock, flags);
2361
2362                 /* Now we need to dynamically allocate one */
2363                 deferfcp = kmalloc(sizeof(*deferfcp), GFP_KERNEL);
2364                 if (!deferfcp) {
2365                         /* release the queue lookup reference */
2366                         nvmet_fc_tgt_q_put(queue);
2367                         return -ENOMEM;
2368                 }
2369                 spin_lock_irqsave(&queue->qlock, flags);
2370         }
2371
2372         /* For now, use rspaddr / rsplen to save payload information */
2373         fcpreq->rspaddr = cmdiubuf;
2374         fcpreq->rsplen  = cmdiubuf_len;
2375         deferfcp->fcp_req = fcpreq;
2376
2377         /* defer processing till a fod becomes available */
2378         list_add_tail(&deferfcp->req_list, &queue->pending_cmd_list);
2379
2380         /* NOTE: the queue lookup reference is still valid */
2381
2382         spin_unlock_irqrestore(&queue->qlock, flags);
2383
2384         return -EOVERFLOW;
2385 }
2386 EXPORT_SYMBOL_GPL(nvmet_fc_rcv_fcp_req);
2387
2388 /**
2389  * nvmet_fc_rcv_fcp_abort - transport entry point called by an LLDD
2390  *                       upon the reception of an ABTS for a FCP command
2391  *
2392  * Notify the transport that an ABTS has been received for a FCP command
2393  * that had been given to the transport via nvmet_fc_rcv_fcp_req(). The
2394  * LLDD believes the command is still being worked on
2395  * (template_ops->fcp_req_release() has not been called).
2396  *
2397  * The transport will wait for any outstanding work (an op to the LLDD,
2398  * which the lldd should complete with error due to the ABTS; or the
2399  * completion from the nvmet layer of the nvme command), then will
2400  * stop processing and call the nvmet_fc_rcv_fcp_req() callback to
2401  * return the i/o context to the LLDD.  The LLDD may send the BA_ACC
2402  * to the ABTS either after return from this function (assuming any
2403  * outstanding op work has been terminated) or upon the callback being
2404  * called.
2405  *
2406  * @target_port: pointer to the (registered) target port the FCP CMD IU
2407  *              was received on.
2408  * @fcpreq:     pointer to the fcpreq request structure that corresponds
2409  *              to the exchange that received the ABTS.
2410  */
2411 void
2412 nvmet_fc_rcv_fcp_abort(struct nvmet_fc_target_port *target_port,
2413                         struct nvmefc_tgt_fcp_req *fcpreq)
2414 {
2415         struct nvmet_fc_fcp_iod *fod = fcpreq->nvmet_fc_private;
2416         struct nvmet_fc_tgt_queue *queue;
2417         unsigned long flags;
2418
2419         if (!fod || fod->fcpreq != fcpreq)
2420                 /* job appears to have already completed, ignore abort */
2421                 return;
2422
2423         queue = fod->queue;
2424
2425         spin_lock_irqsave(&queue->qlock, flags);
2426         if (fod->active) {
2427                 /*
2428                  * mark as abort. The abort handler, invoked upon completion
2429                  * of any work, will detect the aborted status and do the
2430                  * callback.
2431                  */
2432                 spin_lock(&fod->flock);
2433                 fod->abort = true;
2434                 fod->aborted = true;
2435                 spin_unlock(&fod->flock);
2436         }
2437         spin_unlock_irqrestore(&queue->qlock, flags);
2438 }
2439 EXPORT_SYMBOL_GPL(nvmet_fc_rcv_fcp_abort);
2440
2441
2442 struct nvmet_fc_traddr {
2443         u64     nn;
2444         u64     pn;
2445 };
2446
2447 static int
2448 __nvme_fc_parse_u64(substring_t *sstr, u64 *val)
2449 {
2450         u64 token64;
2451
2452         if (match_u64(sstr, &token64))
2453                 return -EINVAL;
2454         *val = token64;
2455
2456         return 0;
2457 }
2458
2459 /*
2460  * This routine validates and extracts the WWN's from the TRADDR string.
2461  * As kernel parsers need the 0x to determine number base, universally
2462  * build string to parse with 0x prefix before parsing name strings.
2463  */
2464 static int
2465 nvme_fc_parse_traddr(struct nvmet_fc_traddr *traddr, char *buf, size_t blen)
2466 {
2467         char name[2 + NVME_FC_TRADDR_HEXNAMELEN + 1];
2468         substring_t wwn = { name, &name[sizeof(name)-1] };
2469         int nnoffset, pnoffset;
2470
2471         /* validate it string one of the 2 allowed formats */
2472         if (strnlen(buf, blen) == NVME_FC_TRADDR_MAXLENGTH &&
2473                         !strncmp(buf, "nn-0x", NVME_FC_TRADDR_OXNNLEN) &&
2474                         !strncmp(&buf[NVME_FC_TRADDR_MAX_PN_OFFSET],
2475                                 "pn-0x", NVME_FC_TRADDR_OXNNLEN)) {
2476                 nnoffset = NVME_FC_TRADDR_OXNNLEN;
2477                 pnoffset = NVME_FC_TRADDR_MAX_PN_OFFSET +
2478                                                 NVME_FC_TRADDR_OXNNLEN;
2479         } else if ((strnlen(buf, blen) == NVME_FC_TRADDR_MINLENGTH &&
2480                         !strncmp(buf, "nn-", NVME_FC_TRADDR_NNLEN) &&
2481                         !strncmp(&buf[NVME_FC_TRADDR_MIN_PN_OFFSET],
2482                                 "pn-", NVME_FC_TRADDR_NNLEN))) {
2483                 nnoffset = NVME_FC_TRADDR_NNLEN;
2484                 pnoffset = NVME_FC_TRADDR_MIN_PN_OFFSET + NVME_FC_TRADDR_NNLEN;
2485         } else
2486                 goto out_einval;
2487
2488         name[0] = '0';
2489         name[1] = 'x';
2490         name[2 + NVME_FC_TRADDR_HEXNAMELEN] = 0;
2491
2492         memcpy(&name[2], &buf[nnoffset], NVME_FC_TRADDR_HEXNAMELEN);
2493         if (__nvme_fc_parse_u64(&wwn, &traddr->nn))
2494                 goto out_einval;
2495
2496         memcpy(&name[2], &buf[pnoffset], NVME_FC_TRADDR_HEXNAMELEN);
2497         if (__nvme_fc_parse_u64(&wwn, &traddr->pn))
2498                 goto out_einval;
2499
2500         return 0;
2501
2502 out_einval:
2503         pr_warn("%s: bad traddr string\n", __func__);
2504         return -EINVAL;
2505 }
2506
2507 static int
2508 nvmet_fc_add_port(struct nvmet_port *port)
2509 {
2510         struct nvmet_fc_tgtport *tgtport;
2511         struct nvmet_fc_traddr traddr = { 0L, 0L };
2512         unsigned long flags;
2513         int ret;
2514
2515         /* validate the address info */
2516         if ((port->disc_addr.trtype != NVMF_TRTYPE_FC) ||
2517             (port->disc_addr.adrfam != NVMF_ADDR_FAMILY_FC))
2518                 return -EINVAL;
2519
2520         /* map the traddr address info to a target port */
2521
2522         ret = nvme_fc_parse_traddr(&traddr, port->disc_addr.traddr,
2523                         sizeof(port->disc_addr.traddr));
2524         if (ret)
2525                 return ret;
2526
2527         ret = -ENXIO;
2528         spin_lock_irqsave(&nvmet_fc_tgtlock, flags);
2529         list_for_each_entry(tgtport, &nvmet_fc_target_list, tgt_list) {
2530                 if ((tgtport->fc_target_port.node_name == traddr.nn) &&
2531                     (tgtport->fc_target_port.port_name == traddr.pn)) {
2532                         tgtport->port = port;
2533                         ret = 0;
2534                         break;
2535                 }
2536         }
2537         spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags);
2538         return ret;
2539 }
2540
2541 static void
2542 nvmet_fc_remove_port(struct nvmet_port *port)
2543 {
2544         /* nothing to do */
2545 }
2546
2547 static const struct nvmet_fabrics_ops nvmet_fc_tgt_fcp_ops = {
2548         .owner                  = THIS_MODULE,
2549         .type                   = NVMF_TRTYPE_FC,
2550         .msdbd                  = 1,
2551         .add_port               = nvmet_fc_add_port,
2552         .remove_port            = nvmet_fc_remove_port,
2553         .queue_response         = nvmet_fc_fcp_nvme_cmd_done,
2554         .delete_ctrl            = nvmet_fc_delete_ctrl,
2555 };
2556
2557 static int __init nvmet_fc_init_module(void)
2558 {
2559         return nvmet_register_transport(&nvmet_fc_tgt_fcp_ops);
2560 }
2561
2562 static void __exit nvmet_fc_exit_module(void)
2563 {
2564         /* sanity check - all lports should be removed */
2565         if (!list_empty(&nvmet_fc_target_list))
2566                 pr_warn("%s: targetport list not empty\n", __func__);
2567
2568         nvmet_unregister_transport(&nvmet_fc_tgt_fcp_ops);
2569
2570         ida_destroy(&nvmet_fc_tgtport_cnt);
2571 }
2572
2573 module_init(nvmet_fc_init_module);
2574 module_exit(nvmet_fc_exit_module);
2575
2576 MODULE_LICENSE("GPL v2");