drivers/scsi/lpfc/lpfc_nvmet.c

   1 /*******************************************************************
   2  * This file is part of the Emulex Linux Device Driver for         *
   3  * Fibre Channsel Host Bus Adapters.                               *
   4  * Copyright (C) 2017-2018 Broadcom. All Rights Reserved. The term *
   5  * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries.     *
   6  * Copyright (C) 2004-2016 Emulex.  All rights reserved.           *
   7  * EMULEX and SLI are trademarks of Emulex.                        *
   8  * www.broadcom.com                                                *
   9  * Portions Copyright (C) 2004-2005 Christoph Hellwig              *
  10  *                                                                 *
  11  * This program is free software; you can redistribute it and/or   *
  12  * modify it under the terms of version 2 of the GNU General       *
  13  * Public License as published by the Free Software Foundation.    *
  14  * This program is distributed in the hope that it will be useful. *
  15  * ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND          *
  16  * WARRANTIES, INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY,  *
  17  * FITNESS FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT, ARE      *
  18  * DISCLAIMED, EXCEPT TO THE EXTENT THAT SUCH DISCLAIMERS ARE HELD *
  19  * TO BE LEGALLY INVALID.  See the GNU General Public License for  *
  20  * more details, a copy of which can be found in the file COPYING  *
  21  * included with this package.                                     *
  22  ********************************************************************/
  23 #include <linux/pci.h>
  24 #include <linux/slab.h>
  25 #include <linux/interrupt.h>
  26 #include <linux/delay.h>
  27 #include <asm/unaligned.h>
  28 #include <linux/crc-t10dif.h>
  29 #include <net/checksum.h>
  30
  31 #include <scsi/scsi.h>
  32 #include <scsi/scsi_device.h>
  33 #include <scsi/scsi_eh.h>
  34 #include <scsi/scsi_host.h>
  35 #include <scsi/scsi_tcq.h>
  36 #include <scsi/scsi_transport_fc.h>
  37 #include <scsi/fc/fc_fs.h>
  38
  39 #include <linux/nvme.h>
  40 #include <linux/nvme-fc-driver.h>
  41 #include <linux/nvme-fc.h>
  42
  43 #include "lpfc_version.h"
  44 #include "lpfc_hw4.h"
  45 #include "lpfc_hw.h"
  46 #include "lpfc_sli.h"
  47 #include "lpfc_sli4.h"
  48 #include "lpfc_nl.h"
  49 #include "lpfc_disc.h"
  50 #include "lpfc.h"
  51 #include "lpfc_scsi.h"
  52 #include "lpfc_nvme.h"
  53 #include "lpfc_nvmet.h"
  54 #include "lpfc_logmsg.h"
  55 #include "lpfc_crtn.h"
  56 #include "lpfc_vport.h"
  57 #include "lpfc_debugfs.h"
  58
  59 static struct lpfc_iocbq *lpfc_nvmet_prep_ls_wqe(struct lpfc_hba *,
  60                                                  struct lpfc_nvmet_rcv_ctx *,
  61                                                  dma_addr_t rspbuf,
  62                                                  uint16_t rspsize);
  63 static struct lpfc_iocbq *lpfc_nvmet_prep_fcp_wqe(struct lpfc_hba *,
  64                                                   struct lpfc_nvmet_rcv_ctx *);
  65 static int lpfc_nvmet_sol_fcp_issue_abort(struct lpfc_hba *,
  66                                           struct lpfc_nvmet_rcv_ctx *,
  67                                           uint32_t, uint16_t);
  68 static int lpfc_nvmet_unsol_fcp_issue_abort(struct lpfc_hba *,
  69                                             struct lpfc_nvmet_rcv_ctx *,
  70                                             uint32_t, uint16_t);
  71 static int lpfc_nvmet_unsol_ls_issue_abort(struct lpfc_hba *,
  72                                            struct lpfc_nvmet_rcv_ctx *,
  73                                            uint32_t, uint16_t);
  74 static void lpfc_nvmet_wqfull_flush(struct lpfc_hba *, struct lpfc_queue *,
  75                                     struct lpfc_nvmet_rcv_ctx *);
  76
  77 static union lpfc_wqe128 lpfc_tsend_cmd_template;
  78 static union lpfc_wqe128 lpfc_treceive_cmd_template;
  79 static union lpfc_wqe128 lpfc_trsp_cmd_template;
  80
  81 /* Setup WQE templates for NVME IOs */
  82 void
  83 lpfc_nvmet_cmd_template(void)
  84 {
  85         union lpfc_wqe128 *wqe;
  86
  87         /* TSEND template */
  88         wqe = &lpfc_tsend_cmd_template;
  89         memset(wqe, 0, sizeof(union lpfc_wqe128));
  90
  91         /* Word 0, 1, 2 - BDE is variable */
  92
  93         /* Word 3 - payload_offset_len is zero */
  94
  95         /* Word 4 - relative_offset is variable */
  96
  97         /* Word 5 - is zero */
  98
  99         /* Word 6 - ctxt_tag, xri_tag is variable */
 100
 101         /* Word 7 - wqe_ar is variable */
 102         bf_set(wqe_cmnd, &wqe->fcp_tsend.wqe_com, CMD_FCP_TSEND64_WQE);
 103         bf_set(wqe_pu, &wqe->fcp_tsend.wqe_com, PARM_REL_OFF);
 104         bf_set(wqe_class, &wqe->fcp_tsend.wqe_com, CLASS3);
 105         bf_set(wqe_ct, &wqe->fcp_tsend.wqe_com, SLI4_CT_RPI);
 106         bf_set(wqe_ar, &wqe->fcp_tsend.wqe_com, 1);
 107
 108         /* Word 8 - abort_tag is variable */
 109
 110         /* Word 9  - reqtag, rcvoxid is variable */
 111
 112         /* Word 10 - wqes, xc is variable */
 113         bf_set(wqe_nvme, &wqe->fcp_tsend.wqe_com, 1);
 114         bf_set(wqe_dbde, &wqe->fcp_tsend.wqe_com, 1);
 115         bf_set(wqe_wqes, &wqe->fcp_tsend.wqe_com, 0);
 116         bf_set(wqe_xc, &wqe->fcp_tsend.wqe_com, 1);
 117         bf_set(wqe_iod, &wqe->fcp_tsend.wqe_com, LPFC_WQE_IOD_WRITE);
 118         bf_set(wqe_lenloc, &wqe->fcp_tsend.wqe_com, LPFC_WQE_LENLOC_WORD12);
 119
 120         /* Word 11 - sup, irsp, irsplen is variable */
 121         bf_set(wqe_cmd_type, &wqe->fcp_tsend.wqe_com, FCP_COMMAND_TSEND);
 122         bf_set(wqe_cqid, &wqe->fcp_tsend.wqe_com, LPFC_WQE_CQ_ID_DEFAULT);
 123         bf_set(wqe_sup, &wqe->fcp_tsend.wqe_com, 0);
 124         bf_set(wqe_irsp, &wqe->fcp_tsend.wqe_com, 0);
 125         bf_set(wqe_irsplen, &wqe->fcp_tsend.wqe_com, 0);
 126         bf_set(wqe_pbde, &wqe->fcp_tsend.wqe_com, 0);
 127
 128         /* Word 12 - fcp_data_len is variable */
 129
 130         /* Word 13, 14, 15 - PBDE is zero */
 131
 132         /* TRECEIVE template */
 133         wqe = &lpfc_treceive_cmd_template;
 134         memset(wqe, 0, sizeof(union lpfc_wqe128));
 135
 136         /* Word 0, 1, 2 - BDE is variable */
 137
 138         /* Word 3 */
 139         wqe->fcp_treceive.payload_offset_len = TXRDY_PAYLOAD_LEN;
 140
 141         /* Word 4 - relative_offset is variable */
 142
 143         /* Word 5 - is zero */
 144
 145         /* Word 6 - ctxt_tag, xri_tag is variable */
 146
 147         /* Word 7 */
 148         bf_set(wqe_cmnd, &wqe->fcp_treceive.wqe_com, CMD_FCP_TRECEIVE64_WQE);
 149         bf_set(wqe_pu, &wqe->fcp_treceive.wqe_com, PARM_REL_OFF);
 150         bf_set(wqe_class, &wqe->fcp_treceive.wqe_com, CLASS3);
 151         bf_set(wqe_ct, &wqe->fcp_treceive.wqe_com, SLI4_CT_RPI);
 152         bf_set(wqe_ar, &wqe->fcp_treceive.wqe_com, 0);
 153
 154         /* Word 8 - abort_tag is variable */
 155
 156         /* Word 9  - reqtag, rcvoxid is variable */
 157
 158         /* Word 10 - xc is variable */
 159         bf_set(wqe_dbde, &wqe->fcp_treceive.wqe_com, 1);
 160         bf_set(wqe_wqes, &wqe->fcp_treceive.wqe_com, 0);
 161         bf_set(wqe_nvme, &wqe->fcp_treceive.wqe_com, 1);
 162         bf_set(wqe_iod, &wqe->fcp_treceive.wqe_com, LPFC_WQE_IOD_READ);
 163         bf_set(wqe_lenloc, &wqe->fcp_treceive.wqe_com, LPFC_WQE_LENLOC_WORD12);
 164         bf_set(wqe_xc, &wqe->fcp_tsend.wqe_com, 1);
 165
 166         /* Word 11 - pbde is variable */
 167         bf_set(wqe_cmd_type, &wqe->fcp_treceive.wqe_com, FCP_COMMAND_TRECEIVE);
 168         bf_set(wqe_cqid, &wqe->fcp_treceive.wqe_com, LPFC_WQE_CQ_ID_DEFAULT);
 169         bf_set(wqe_sup, &wqe->fcp_treceive.wqe_com, 0);
 170         bf_set(wqe_irsp, &wqe->fcp_treceive.wqe_com, 0);
 171         bf_set(wqe_irsplen, &wqe->fcp_treceive.wqe_com, 0);
 172         bf_set(wqe_pbde, &wqe->fcp_treceive.wqe_com, 1);
 173
 174         /* Word 12 - fcp_data_len is variable */
 175
 176         /* Word 13, 14, 15 - PBDE is variable */
 177
 178         /* TRSP template */
 179         wqe = &lpfc_trsp_cmd_template;
 180         memset(wqe, 0, sizeof(union lpfc_wqe128));
 181
 182         /* Word 0, 1, 2 - BDE is variable */
 183
 184         /* Word 3 - response_len is variable */
 185
 186         /* Word 4, 5 - is zero */
 187
 188         /* Word 6 - ctxt_tag, xri_tag is variable */
 189
 190         /* Word 7 */
 191         bf_set(wqe_cmnd, &wqe->fcp_trsp.wqe_com, CMD_FCP_TRSP64_WQE);
 192         bf_set(wqe_pu, &wqe->fcp_trsp.wqe_com, PARM_UNUSED);
 193         bf_set(wqe_class, &wqe->fcp_trsp.wqe_com, CLASS3);
 194         bf_set(wqe_ct, &wqe->fcp_trsp.wqe_com, SLI4_CT_RPI);
 195         bf_set(wqe_ag, &wqe->fcp_trsp.wqe_com, 1); /* wqe_ar */
 196
 197         /* Word 8 - abort_tag is variable */
 198
 199         /* Word 9  - reqtag is variable */
 200
 201         /* Word 10 wqes, xc is variable */
 202         bf_set(wqe_dbde, &wqe->fcp_trsp.wqe_com, 1);
 203         bf_set(wqe_nvme, &wqe->fcp_trsp.wqe_com, 1);
 204         bf_set(wqe_wqes, &wqe->fcp_trsp.wqe_com, 0);
 205         bf_set(wqe_xc, &wqe->fcp_trsp.wqe_com, 0);
 206         bf_set(wqe_iod, &wqe->fcp_trsp.wqe_com, LPFC_WQE_IOD_NONE);
 207         bf_set(wqe_lenloc, &wqe->fcp_trsp.wqe_com, LPFC_WQE_LENLOC_WORD3);
 208
 209         /* Word 11 irsp, irsplen is variable */
 210         bf_set(wqe_cmd_type, &wqe->fcp_trsp.wqe_com, FCP_COMMAND_TRSP);
 211         bf_set(wqe_cqid, &wqe->fcp_trsp.wqe_com, LPFC_WQE_CQ_ID_DEFAULT);
 212         bf_set(wqe_sup, &wqe->fcp_trsp.wqe_com, 0);
 213         bf_set(wqe_irsp, &wqe->fcp_trsp.wqe_com, 0);
 214         bf_set(wqe_irsplen, &wqe->fcp_trsp.wqe_com, 0);
 215         bf_set(wqe_pbde, &wqe->fcp_trsp.wqe_com, 0);
 216
 217         /* Word 12, 13, 14, 15 - is zero */
 218 }
 219
 220 void
 221 lpfc_nvmet_defer_release(struct lpfc_hba *phba, struct lpfc_nvmet_rcv_ctx *ctxp)
 222 {
 223         unsigned long iflag;
 224
 225         lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
 226                         "6313 NVMET Defer ctx release xri x%x flg x%x\n",
 227                         ctxp->oxid, ctxp->flag);
 228
 229         spin_lock_irqsave(&phba->sli4_hba.abts_nvme_buf_list_lock, iflag);
 230         if (ctxp->flag & LPFC_NVMET_CTX_RLS) {
 231                 spin_unlock_irqrestore(&phba->sli4_hba.abts_nvme_buf_list_lock,
 232                                        iflag);
 233                 return;
 234         }
 235         ctxp->flag |= LPFC_NVMET_CTX_RLS;
 236         list_add_tail(&ctxp->list, &phba->sli4_hba.lpfc_abts_nvmet_ctx_list);
 237         spin_unlock_irqrestore(&phba->sli4_hba.abts_nvme_buf_list_lock, iflag);
 238 }
 239
 240 /**
 241  * lpfc_nvmet_xmt_ls_rsp_cmp - Completion handler for LS Response
 242  * @phba: Pointer to HBA context object.
 243  * @cmdwqe: Pointer to driver command WQE object.
 244  * @wcqe: Pointer to driver response CQE object.
 245  *
 246  * The function is called from SLI ring event handler with no
 247  * lock held. This function is the completion handler for NVME LS commands
 248  * The function frees memory resources used for the NVME commands.
 249  **/
 250 static void
 251 lpfc_nvmet_xmt_ls_rsp_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe,
 252                           struct lpfc_wcqe_complete *wcqe)
 253 {
 254         struct lpfc_nvmet_tgtport *tgtp;
 255         struct nvmefc_tgt_ls_req *rsp;
 256         struct lpfc_nvmet_rcv_ctx *ctxp;
 257         uint32_t status, result;
 258
 259         status = bf_get(lpfc_wcqe_c_status, wcqe);
 260         result = wcqe->parameter;
 261         ctxp = cmdwqe->context2;
 262
 263         if (ctxp->state != LPFC_NVMET_STE_LS_RSP || ctxp->entry_cnt != 2) {
 264                 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
 265                                 "6410 NVMET LS cmpl state mismatch IO x%x: "
 266                                 "%d %d\n",
 267                                 ctxp->oxid, ctxp->state, ctxp->entry_cnt);
 268         }
 269
 270         if (!phba->targetport)
 271                 goto out;
 272
 273         tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
 274
 275         if (tgtp) {
 276                 if (status) {
 277                         atomic_inc(&tgtp->xmt_ls_rsp_error);
 278                         if (result == IOERR_ABORT_REQUESTED)
 279                                 atomic_inc(&tgtp->xmt_ls_rsp_aborted);
 280                         if (bf_get(lpfc_wcqe_c_xb, wcqe))
 281                                 atomic_inc(&tgtp->xmt_ls_rsp_xb_set);
 282                 } else {
 283                         atomic_inc(&tgtp->xmt_ls_rsp_cmpl);
 284                 }
 285         }
 286
 287 out:
 288         rsp = &ctxp->ctx.ls_req;
 289
 290         lpfc_nvmeio_data(phba, "NVMET LS  CMPL: xri x%x stat x%x result x%x\n",
 291                          ctxp->oxid, status, result);
 292
 293         lpfc_printf_log(phba, KERN_INFO, LOG_NVME_DISC,
 294                         "6038 NVMET LS rsp cmpl: %d %d oxid x%x\n",
 295                         status, result, ctxp->oxid);
 296
 297         lpfc_nlp_put(cmdwqe->context1);
 298         cmdwqe->context2 = NULL;
 299         cmdwqe->context3 = NULL;
 300         lpfc_sli_release_iocbq(phba, cmdwqe);
 301         rsp->done(rsp);
 302         kfree(ctxp);
 303 }
 304
 305 /**
 306  * lpfc_nvmet_ctxbuf_post - Repost a NVMET RQ DMA buffer and clean up context
 307  * @phba: HBA buffer is associated with
 308  * @ctxp: context to clean up
 309  * @mp: Buffer to free
 310  *
 311  * Description: Frees the given DMA buffer in the appropriate way given by
 312  * reposting it to its associated RQ so it can be reused.
 313  *
 314  * Notes: Takes phba->hbalock.  Can be called with or without other locks held.
 315  *
 316  * Returns: None
 317  **/
 318 void
 319 lpfc_nvmet_ctxbuf_post(struct lpfc_hba *phba, struct lpfc_nvmet_ctxbuf *ctx_buf)
 320 {
 321 #if (IS_ENABLED(CONFIG_NVME_TARGET_FC))
 322         struct lpfc_nvmet_rcv_ctx *ctxp = ctx_buf->context;
 323         struct lpfc_nvmet_tgtport *tgtp;
 324         struct fc_frame_header *fc_hdr;
 325         struct rqb_dmabuf *nvmebuf;
 326         struct lpfc_nvmet_ctx_info *infop;
 327         uint32_t *payload;
 328         uint32_t size, oxid, sid, rc;
 329         int cpu;
 330         unsigned long iflag;
 331
 332         if (ctxp->txrdy) {
 333                 dma_pool_free(phba->txrdy_payload_pool, ctxp->txrdy,
 334                               ctxp->txrdy_phys);
 335                 ctxp->txrdy = NULL;
 336                 ctxp->txrdy_phys = 0;
 337         }
 338
 339         if (ctxp->state == LPFC_NVMET_STE_FREE) {
 340                 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
 341                                 "6411 NVMET free, already free IO x%x: %d %d\n",
 342                                 ctxp->oxid, ctxp->state, ctxp->entry_cnt);
 343         }
 344         ctxp->state = LPFC_NVMET_STE_FREE;
 345
 346         spin_lock_irqsave(&phba->sli4_hba.nvmet_io_wait_lock, iflag);
 347         if (phba->sli4_hba.nvmet_io_wait_cnt) {
 348                 list_remove_head(&phba->sli4_hba.lpfc_nvmet_io_wait_list,
 349                                  nvmebuf, struct rqb_dmabuf,
 350                                  hbuf.list);
 351                 phba->sli4_hba.nvmet_io_wait_cnt--;
 352                 spin_unlock_irqrestore(&phba->sli4_hba.nvmet_io_wait_lock,
 353                                        iflag);
 354
 355                 fc_hdr = (struct fc_frame_header *)(nvmebuf->hbuf.virt);
 356                 oxid = be16_to_cpu(fc_hdr->fh_ox_id);
 357                 tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
 358                 payload = (uint32_t *)(nvmebuf->dbuf.virt);
 359                 size = nvmebuf->bytes_recv;
 360                 sid = sli4_sid_from_fc_hdr(fc_hdr);
 361
 362                 ctxp = (struct lpfc_nvmet_rcv_ctx *)ctx_buf->context;
 363                 ctxp->wqeq = NULL;
 364                 ctxp->txrdy = NULL;
 365                 ctxp->offset = 0;
 366                 ctxp->phba = phba;
 367                 ctxp->size = size;
 368                 ctxp->oxid = oxid;
 369                 ctxp->sid = sid;
 370                 ctxp->state = LPFC_NVMET_STE_RCV;
 371                 ctxp->entry_cnt = 1;
 372                 ctxp->flag = 0;
 373                 ctxp->ctxbuf = ctx_buf;
 374                 ctxp->rqb_buffer = (void *)nvmebuf;
 375                 spin_lock_init(&ctxp->ctxlock);
 376
 377 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
 378                 if (ctxp->ts_cmd_nvme) {
 379                         ctxp->ts_cmd_nvme = ktime_get_ns();
 380                         ctxp->ts_nvme_data = 0;
 381                         ctxp->ts_data_wqput = 0;
 382                         ctxp->ts_isr_data = 0;
 383                         ctxp->ts_data_nvme = 0;
 384                         ctxp->ts_nvme_status = 0;
 385                         ctxp->ts_status_wqput = 0;
 386                         ctxp->ts_isr_status = 0;
 387                         ctxp->ts_status_nvme = 0;
 388                 }
 389 #endif
 390                 atomic_inc(&tgtp->rcv_fcp_cmd_in);
 391                 /*
 392                  * The calling sequence should be:
 393                  * nvmet_fc_rcv_fcp_req->lpfc_nvmet_xmt_fcp_op/cmp- req->done
 394                  * lpfc_nvmet_xmt_fcp_op_cmp should free the allocated ctxp.
 395                  * When we return from nvmet_fc_rcv_fcp_req, all relevant info
 396                  * the NVME command / FC header is stored.
 397                  * A buffer has already been reposted for this IO, so just free
 398                  * the nvmebuf.
 399                  */
 400                 rc = nvmet_fc_rcv_fcp_req(phba->targetport, &ctxp->ctx.fcp_req,
 401                                           payload, size);
 402
 403                 /* Process FCP command */
 404                 if (rc == 0) {
 405                         ctxp->rqb_buffer = NULL;
 406                         atomic_inc(&tgtp->rcv_fcp_cmd_out);
 407                         nvmebuf->hrq->rqbp->rqb_free_buffer(phba, nvmebuf);
 408                         return;
 409                 }
 410
 411                 /* Processing of FCP command is deferred */
 412                 if (rc == -EOVERFLOW) {
 413                         lpfc_nvmeio_data(phba,
 414                                          "NVMET RCV BUSY: xri x%x sz %d "
 415                                          "from %06x\n",
 416                                          oxid, size, sid);
 417                         atomic_inc(&tgtp->rcv_fcp_cmd_out);
 418                         return;
 419                 }
 420                 atomic_inc(&tgtp->rcv_fcp_cmd_drop);
 421                 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
 422                                 "2582 FCP Drop IO x%x: err x%x: x%x x%x x%x\n",
 423                                 ctxp->oxid, rc,
 424                                 atomic_read(&tgtp->rcv_fcp_cmd_in),
 425                                 atomic_read(&tgtp->rcv_fcp_cmd_out),
 426                                 atomic_read(&tgtp->xmt_fcp_release));
 427
 428                 lpfc_nvmet_defer_release(phba, ctxp);
 429                 lpfc_nvmet_unsol_fcp_issue_abort(phba, ctxp, sid, oxid);
 430                 nvmebuf->hrq->rqbp->rqb_free_buffer(phba, nvmebuf);
 431                 return;
 432         }
 433         spin_unlock_irqrestore(&phba->sli4_hba.nvmet_io_wait_lock, iflag);
 434
 435         /*
 436          * Use the CPU context list, from the MRQ the IO was received on
 437          * (ctxp->idx), to save context structure.
 438          */
 439         cpu = smp_processor_id();
 440         infop = lpfc_get_ctx_list(phba, cpu, ctxp->idx);
 441         spin_lock_irqsave(&infop->nvmet_ctx_list_lock, iflag);
 442         list_add_tail(&ctx_buf->list, &infop->nvmet_ctx_list);
 443         infop->nvmet_ctx_list_cnt++;
 444         spin_unlock_irqrestore(&infop->nvmet_ctx_list_lock, iflag);
 445 #endif
 446 }
 447
 448 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
 449 static void
 450 lpfc_nvmet_ktime(struct lpfc_hba *phba,
 451                  struct lpfc_nvmet_rcv_ctx *ctxp)
 452 {
 453         uint64_t seg1, seg2, seg3, seg4, seg5;
 454         uint64_t seg6, seg7, seg8, seg9, seg10;
 455         uint64_t segsum;
 456
 457         if (!ctxp->ts_isr_cmd || !ctxp->ts_cmd_nvme ||
 458             !ctxp->ts_nvme_data || !ctxp->ts_data_wqput ||
 459             !ctxp->ts_isr_data || !ctxp->ts_data_nvme ||
 460             !ctxp->ts_nvme_status || !ctxp->ts_status_wqput ||
 461             !ctxp->ts_isr_status || !ctxp->ts_status_nvme)
 462                 return;
 463
 464         if (ctxp->ts_status_nvme < ctxp->ts_isr_cmd)
 465                 return;
 466         if (ctxp->ts_isr_cmd  > ctxp->ts_cmd_nvme)
 467                 return;
 468         if (ctxp->ts_cmd_nvme > ctxp->ts_nvme_data)
 469                 return;
 470         if (ctxp->ts_nvme_data > ctxp->ts_data_wqput)
 471                 return;
 472         if (ctxp->ts_data_wqput > ctxp->ts_isr_data)
 473                 return;
 474         if (ctxp->ts_isr_data > ctxp->ts_data_nvme)
 475                 return;
 476         if (ctxp->ts_data_nvme > ctxp->ts_nvme_status)
 477                 return;
 478         if (ctxp->ts_nvme_status > ctxp->ts_status_wqput)
 479                 return;
 480         if (ctxp->ts_status_wqput > ctxp->ts_isr_status)
 481                 return;
 482         if (ctxp->ts_isr_status > ctxp->ts_status_nvme)
 483                 return;
 484         /*
 485          * Segment 1 - Time from FCP command received by MSI-X ISR
 486          * to FCP command is passed to NVME Layer.
 487          * Segment 2 - Time from FCP command payload handed
 488          * off to NVME Layer to Driver receives a Command op
 489          * from NVME Layer.
 490          * Segment 3 - Time from Driver receives a Command op
 491          * from NVME Layer to Command is put on WQ.
 492          * Segment 4 - Time from Driver WQ put is done
 493          * to MSI-X ISR for Command cmpl.
 494          * Segment 5 - Time from MSI-X ISR for Command cmpl to
 495          * Command cmpl is passed to NVME Layer.
 496          * Segment 6 - Time from Command cmpl is passed to NVME
 497          * Layer to Driver receives a RSP op from NVME Layer.
 498          * Segment 7 - Time from Driver receives a RSP op from
 499          * NVME Layer to WQ put is done on TRSP FCP Status.
 500          * Segment 8 - Time from Driver WQ put is done on TRSP
 501          * FCP Status to MSI-X ISR for TRSP cmpl.
 502          * Segment 9 - Time from MSI-X ISR for TRSP cmpl to
 503          * TRSP cmpl is passed to NVME Layer.
 504          * Segment 10 - Time from FCP command received by
 505          * MSI-X ISR to command is completed on wire.
 506          * (Segments 1 thru 8) for READDATA / WRITEDATA
 507          * (Segments 1 thru 4) for READDATA_RSP
 508          */
 509         seg1 = ctxp->ts_cmd_nvme - ctxp->ts_isr_cmd;
 510         segsum = seg1;
 511
 512         seg2 = ctxp->ts_nvme_data - ctxp->ts_isr_cmd;
 513         if (segsum > seg2)
 514                 return;
 515         seg2 -= segsum;
 516         segsum += seg2;
 517
 518         seg3 = ctxp->ts_data_wqput - ctxp->ts_isr_cmd;
 519         if (segsum > seg3)
 520                 return;
 521         seg3 -= segsum;
 522         segsum += seg3;
 523
 524         seg4 = ctxp->ts_isr_data - ctxp->ts_isr_cmd;
 525         if (segsum > seg4)
 526                 return;
 527         seg4 -= segsum;
 528         segsum += seg4;
 529
 530         seg5 = ctxp->ts_data_nvme - ctxp->ts_isr_cmd;
 531         if (segsum > seg5)
 532                 return;
 533         seg5 -= segsum;
 534         segsum += seg5;
 535
 536
 537         /* For auto rsp commands seg6 thru seg10 will be 0 */
 538         if (ctxp->ts_nvme_status > ctxp->ts_data_nvme) {
 539                 seg6 = ctxp->ts_nvme_status - ctxp->ts_isr_cmd;
 540                 if (segsum > seg6)
 541                         return;
 542                 seg6 -= segsum;
 543                 segsum += seg6;
 544
 545                 seg7 = ctxp->ts_status_wqput - ctxp->ts_isr_cmd;
 546                 if (segsum > seg7)
 547                         return;
 548                 seg7 -= segsum;
 549                 segsum += seg7;
 550
 551                 seg8 = ctxp->ts_isr_status - ctxp->ts_isr_cmd;
 552                 if (segsum > seg8)
 553                         return;
 554                 seg8 -= segsum;
 555                 segsum += seg8;
 556
 557                 seg9 = ctxp->ts_status_nvme - ctxp->ts_isr_cmd;
 558                 if (segsum > seg9)
 559                         return;
 560                 seg9 -= segsum;
 561                 segsum += seg9;
 562
 563                 if (ctxp->ts_isr_status < ctxp->ts_isr_cmd)
 564                         return;
 565                 seg10 = (ctxp->ts_isr_status -
 566                         ctxp->ts_isr_cmd);
 567         } else {
 568                 if (ctxp->ts_isr_data < ctxp->ts_isr_cmd)
 569                         return;
 570                 seg6 =  0;
 571                 seg7 =  0;
 572                 seg8 =  0;
 573                 seg9 =  0;
 574                 seg10 = (ctxp->ts_isr_data - ctxp->ts_isr_cmd);
 575         }
 576
 577         phba->ktime_seg1_total += seg1;
 578         if (seg1 < phba->ktime_seg1_min)
 579                 phba->ktime_seg1_min = seg1;
 580         else if (seg1 > phba->ktime_seg1_max)
 581                 phba->ktime_seg1_max = seg1;
 582
 583         phba->ktime_seg2_total += seg2;
 584         if (seg2 < phba->ktime_seg2_min)
 585                 phba->ktime_seg2_min = seg2;
 586         else if (seg2 > phba->ktime_seg2_max)
 587                 phba->ktime_seg2_max = seg2;
 588
 589         phba->ktime_seg3_total += seg3;
 590         if (seg3 < phba->ktime_seg3_min)
 591                 phba->ktime_seg3_min = seg3;
 592         else if (seg3 > phba->ktime_seg3_max)
 593                 phba->ktime_seg3_max = seg3;
 594
 595         phba->ktime_seg4_total += seg4;
 596         if (seg4 < phba->ktime_seg4_min)
 597                 phba->ktime_seg4_min = seg4;
 598         else if (seg4 > phba->ktime_seg4_max)
 599                 phba->ktime_seg4_max = seg4;
 600
 601         phba->ktime_seg5_total += seg5;
 602         if (seg5 < phba->ktime_seg5_min)
 603                 phba->ktime_seg5_min = seg5;
 604         else if (seg5 > phba->ktime_seg5_max)
 605                 phba->ktime_seg5_max = seg5;
 606
 607         phba->ktime_data_samples++;
 608         if (!seg6)
 609                 goto out;
 610
 611         phba->ktime_seg6_total += seg6;
 612         if (seg6 < phba->ktime_seg6_min)
 613                 phba->ktime_seg6_min = seg6;
 614         else if (seg6 > phba->ktime_seg6_max)
 615                 phba->ktime_seg6_max = seg6;
 616
 617         phba->ktime_seg7_total += seg7;
 618         if (seg7 < phba->ktime_seg7_min)
 619                 phba->ktime_seg7_min = seg7;
 620         else if (seg7 > phba->ktime_seg7_max)
 621                 phba->ktime_seg7_max = seg7;
 622
 623         phba->ktime_seg8_total += seg8;
 624         if (seg8 < phba->ktime_seg8_min)
 625                 phba->ktime_seg8_min = seg8;
 626         else if (seg8 > phba->ktime_seg8_max)
 627                 phba->ktime_seg8_max = seg8;
 628
 629         phba->ktime_seg9_total += seg9;
 630         if (seg9 < phba->ktime_seg9_min)
 631                 phba->ktime_seg9_min = seg9;
 632         else if (seg9 > phba->ktime_seg9_max)
 633                 phba->ktime_seg9_max = seg9;
 634 out:
 635         phba->ktime_seg10_total += seg10;
 636         if (seg10 < phba->ktime_seg10_min)
 637                 phba->ktime_seg10_min = seg10;
 638         else if (seg10 > phba->ktime_seg10_max)
 639                 phba->ktime_seg10_max = seg10;
 640         phba->ktime_status_samples++;
 641 }
 642 #endif
 643
 644 /**
 645  * lpfc_nvmet_xmt_fcp_op_cmp - Completion handler for FCP Response
 646  * @phba: Pointer to HBA context object.
 647  * @cmdwqe: Pointer to driver command WQE object.
 648  * @wcqe: Pointer to driver response CQE object.
 649  *
 650  * The function is called from SLI ring event handler with no
 651  * lock held. This function is the completion handler for NVME FCP commands
 652  * The function frees memory resources used for the NVME commands.
 653  **/
 654 static void
 655 lpfc_nvmet_xmt_fcp_op_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe,
 656                           struct lpfc_wcqe_complete *wcqe)
 657 {
 658         struct lpfc_nvmet_tgtport *tgtp;
 659         struct nvmefc_tgt_fcp_req *rsp;
 660         struct lpfc_nvmet_rcv_ctx *ctxp;
 661         uint32_t status, result, op, start_clean, logerr;
 662 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
 663         uint32_t id;
 664 #endif
 665
 666         ctxp = cmdwqe->context2;
 667         ctxp->flag &= ~LPFC_NVMET_IO_INP;
 668
 669         rsp = &ctxp->ctx.fcp_req;
 670         op = rsp->op;
 671
 672         status = bf_get(lpfc_wcqe_c_status, wcqe);
 673         result = wcqe->parameter;
 674
 675         if (phba->targetport)
 676                 tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
 677         else
 678                 tgtp = NULL;
 679
 680         lpfc_nvmeio_data(phba, "NVMET FCP CMPL: xri x%x op x%x status x%x\n",
 681                          ctxp->oxid, op, status);
 682
 683         if (status) {
 684                 rsp->fcp_error = NVME_SC_DATA_XFER_ERROR;
 685                 rsp->transferred_length = 0;
 686                 if (tgtp) {
 687                         atomic_inc(&tgtp->xmt_fcp_rsp_error);
 688                         if (result == IOERR_ABORT_REQUESTED)
 689                                 atomic_inc(&tgtp->xmt_fcp_rsp_aborted);
 690                 }
 691
 692                 logerr = LOG_NVME_IOERR;
 693
 694                 /* pick up SLI4 exhange busy condition */
 695                 if (bf_get(lpfc_wcqe_c_xb, wcqe)) {
 696                         ctxp->flag |= LPFC_NVMET_XBUSY;
 697                         logerr |= LOG_NVME_ABTS;
 698                         if (tgtp)
 699                                 atomic_inc(&tgtp->xmt_fcp_rsp_xb_set);
 700
 701                 } else {
 702                         ctxp->flag &= ~LPFC_NVMET_XBUSY;
 703                 }
 704
 705                 lpfc_printf_log(phba, KERN_INFO, logerr,
 706                                 "6315 IO Error Cmpl xri x%x: %x/%x XBUSY:x%x\n",
 707                                 ctxp->oxid, status, result, ctxp->flag);
 708
 709         } else {
 710                 rsp->fcp_error = NVME_SC_SUCCESS;
 711                 if (op == NVMET_FCOP_RSP)
 712                         rsp->transferred_length = rsp->rsplen;
 713                 else
 714                         rsp->transferred_length = rsp->transfer_length;
 715                 if (tgtp)
 716                         atomic_inc(&tgtp->xmt_fcp_rsp_cmpl);
 717         }
 718
 719         if ((op == NVMET_FCOP_READDATA_RSP) ||
 720             (op == NVMET_FCOP_RSP)) {
 721                 /* Sanity check */
 722                 ctxp->state = LPFC_NVMET_STE_DONE;
 723                 ctxp->entry_cnt++;
 724
 725 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
 726                 if (ctxp->ts_cmd_nvme) {
 727                         if (rsp->op == NVMET_FCOP_READDATA_RSP) {
 728                                 ctxp->ts_isr_data =
 729                                         cmdwqe->isr_timestamp;
 730                                 ctxp->ts_data_nvme =
 731                                         ktime_get_ns();
 732                                 ctxp->ts_nvme_status =
 733                                         ctxp->ts_data_nvme;
 734                                 ctxp->ts_status_wqput =
 735                                         ctxp->ts_data_nvme;
 736                                 ctxp->ts_isr_status =
 737                                         ctxp->ts_data_nvme;
 738                                 ctxp->ts_status_nvme =
 739                                         ctxp->ts_data_nvme;
 740                         } else {
 741                                 ctxp->ts_isr_status =
 742                                         cmdwqe->isr_timestamp;
 743                                 ctxp->ts_status_nvme =
 744                                         ktime_get_ns();
 745                         }
 746                 }
 747                 if (phba->cpucheck_on & LPFC_CHECK_NVMET_IO) {
 748                         id = smp_processor_id();
 749                         if (ctxp->cpu != id)
 750                                 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
 751                                                 "6703 CPU Check cmpl: "
 752                                                 "cpu %d expect %d\n",
 753                                                 id, ctxp->cpu);
 754                         if (ctxp->cpu < LPFC_CHECK_CPU_CNT)
 755                                 phba->cpucheck_cmpl_io[id]++;
 756                 }
 757 #endif
 758                 rsp->done(rsp);
 759 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
 760                 if (ctxp->ts_cmd_nvme)
 761                         lpfc_nvmet_ktime(phba, ctxp);
 762 #endif
 763                 /* lpfc_nvmet_xmt_fcp_release() will recycle the context */
 764         } else {
 765                 ctxp->entry_cnt++;
 766                 start_clean = offsetof(struct lpfc_iocbq, iocb_flag);
 767                 memset(((char *)cmdwqe) + start_clean, 0,
 768                        (sizeof(struct lpfc_iocbq) - start_clean));
 769 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
 770                 if (ctxp->ts_cmd_nvme) {
 771                         ctxp->ts_isr_data = cmdwqe->isr_timestamp;
 772                         ctxp->ts_data_nvme = ktime_get_ns();
 773                 }
 774                 if (phba->cpucheck_on & LPFC_CHECK_NVMET_IO) {
 775                         id = smp_processor_id();
 776                         if (ctxp->cpu != id)
 777                                 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
 778                                                 "6704 CPU Check cmdcmpl: "
 779                                                 "cpu %d expect %d\n",
 780                                                 id, ctxp->cpu);
 781                         if (ctxp->cpu < LPFC_CHECK_CPU_CNT)
 782                                 phba->cpucheck_ccmpl_io[id]++;
 783                 }
 784 #endif
 785                 rsp->done(rsp);
 786         }
 787 }
 788
 789 static int
 790 lpfc_nvmet_xmt_ls_rsp(struct nvmet_fc_target_port *tgtport,
 791                       struct nvmefc_tgt_ls_req *rsp)
 792 {
 793         struct lpfc_nvmet_rcv_ctx *ctxp =
 794                 container_of(rsp, struct lpfc_nvmet_rcv_ctx, ctx.ls_req);
 795         struct lpfc_hba *phba = ctxp->phba;
 796         struct hbq_dmabuf *nvmebuf =
 797                 (struct hbq_dmabuf *)ctxp->rqb_buffer;
 798         struct lpfc_iocbq *nvmewqeq;
 799         struct lpfc_nvmet_tgtport *nvmep = tgtport->private;
 800         struct lpfc_dmabuf dmabuf;
 801         struct ulp_bde64 bpl;
 802         int rc;
 803
 804         if (phba->pport->load_flag & FC_UNLOADING)
 805                 return -ENODEV;
 806
 807         if (phba->pport->load_flag & FC_UNLOADING)
 808                 return -ENODEV;
 809
 810         lpfc_printf_log(phba, KERN_INFO, LOG_NVME_DISC,
 811                         "6023 NVMET LS rsp oxid x%x\n", ctxp->oxid);
 812
 813         if ((ctxp->state != LPFC_NVMET_STE_LS_RCV) ||
 814             (ctxp->entry_cnt != 1)) {
 815                 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
 816                                 "6412 NVMET LS rsp state mismatch "
 817                                 "oxid x%x: %d %d\n",
 818                                 ctxp->oxid, ctxp->state, ctxp->entry_cnt);
 819         }
 820         ctxp->state = LPFC_NVMET_STE_LS_RSP;
 821         ctxp->entry_cnt++;
 822
 823         nvmewqeq = lpfc_nvmet_prep_ls_wqe(phba, ctxp, rsp->rspdma,
 824                                       rsp->rsplen);
 825         if (nvmewqeq == NULL) {
 826                 atomic_inc(&nvmep->xmt_ls_drop);
 827                 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
 828                                 "6150 LS Drop IO x%x: Prep\n",
 829                                 ctxp->oxid);
 830                 lpfc_in_buf_free(phba, &nvmebuf->dbuf);
 831                 atomic_inc(&nvmep->xmt_ls_abort);
 832                 lpfc_nvmet_unsol_ls_issue_abort(phba, ctxp,
 833                                                 ctxp->sid, ctxp->oxid);
 834                 return -ENOMEM;
 835         }
 836
 837         /* Save numBdes for bpl2sgl */
 838         nvmewqeq->rsvd2 = 1;
 839         nvmewqeq->hba_wqidx = 0;
 840         nvmewqeq->context3 = &dmabuf;
 841         dmabuf.virt = &bpl;
 842         bpl.addrLow = nvmewqeq->wqe.xmit_sequence.bde.addrLow;
 843         bpl.addrHigh = nvmewqeq->wqe.xmit_sequence.bde.addrHigh;
 844         bpl.tus.f.bdeSize = rsp->rsplen;
 845         bpl.tus.f.bdeFlags = 0;
 846         bpl.tus.w = le32_to_cpu(bpl.tus.w);
 847
 848         nvmewqeq->wqe_cmpl = lpfc_nvmet_xmt_ls_rsp_cmp;
 849         nvmewqeq->iocb_cmpl = NULL;
 850         nvmewqeq->context2 = ctxp;
 851
 852         lpfc_nvmeio_data(phba, "NVMET LS  RESP: xri x%x wqidx x%x len x%x\n",
 853                          ctxp->oxid, nvmewqeq->hba_wqidx, rsp->rsplen);
 854
 855         rc = lpfc_sli4_issue_wqe(phba, LPFC_ELS_RING, nvmewqeq);
 856         if (rc == WQE_SUCCESS) {
 857                 /*
 858                  * Okay to repost buffer here, but wait till cmpl
 859                  * before freeing ctxp and iocbq.
 860                  */
 861                 lpfc_in_buf_free(phba, &nvmebuf->dbuf);
 862                 ctxp->rqb_buffer = 0;
 863                 atomic_inc(&nvmep->xmt_ls_rsp);
 864                 return 0;
 865         }
 866         /* Give back resources */
 867         atomic_inc(&nvmep->xmt_ls_drop);
 868         lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
 869                         "6151 LS Drop IO x%x: Issue %d\n",
 870                         ctxp->oxid, rc);
 871
 872         lpfc_nlp_put(nvmewqeq->context1);
 873
 874         lpfc_in_buf_free(phba, &nvmebuf->dbuf);
 875         atomic_inc(&nvmep->xmt_ls_abort);
 876         lpfc_nvmet_unsol_ls_issue_abort(phba, ctxp, ctxp->sid, ctxp->oxid);
 877         return -ENXIO;
 878 }
 879
 880 static int
 881 lpfc_nvmet_xmt_fcp_op(struct nvmet_fc_target_port *tgtport,
 882                       struct nvmefc_tgt_fcp_req *rsp)
 883 {
 884         struct lpfc_nvmet_tgtport *lpfc_nvmep = tgtport->private;
 885         struct lpfc_nvmet_rcv_ctx *ctxp =
 886                 container_of(rsp, struct lpfc_nvmet_rcv_ctx, ctx.fcp_req);
 887         struct lpfc_hba *phba = ctxp->phba;
 888         struct lpfc_queue *wq;
 889         struct lpfc_iocbq *nvmewqeq;
 890         struct lpfc_sli_ring *pring;
 891         unsigned long iflags;
 892         int rc;
 893
 894         if (phba->pport->load_flag & FC_UNLOADING) {
 895                 rc = -ENODEV;
 896                 goto aerr;
 897         }
 898
 899         if (phba->pport->load_flag & FC_UNLOADING) {
 900                 rc = -ENODEV;
 901                 goto aerr;
 902         }
 903
 904 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
 905         if (ctxp->ts_cmd_nvme) {
 906                 if (rsp->op == NVMET_FCOP_RSP)
 907                         ctxp->ts_nvme_status = ktime_get_ns();
 908                 else
 909                         ctxp->ts_nvme_data = ktime_get_ns();
 910         }
 911         if (phba->cpucheck_on & LPFC_CHECK_NVMET_IO) {
 912                 int id = smp_processor_id();
 913                 ctxp->cpu = id;
 914                 if (id < LPFC_CHECK_CPU_CNT)
 915                         phba->cpucheck_xmt_io[id]++;
 916                 if (rsp->hwqid != id) {
 917                         lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
 918                                         "6705 CPU Check OP: "
 919                                         "cpu %d expect %d\n",
 920                                         id, rsp->hwqid);
 921                         ctxp->cpu = rsp->hwqid;
 922                 }
 923         }
 924 #endif
 925
 926         /* Sanity check */
 927         if ((ctxp->flag & LPFC_NVMET_ABTS_RCV) ||
 928             (ctxp->state == LPFC_NVMET_STE_ABORT)) {
 929                 atomic_inc(&lpfc_nvmep->xmt_fcp_drop);
 930                 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
 931                                 "6102 IO xri x%x aborted\n",
 932                                 ctxp->oxid);
 933                 rc = -ENXIO;
 934                 goto aerr;
 935         }
 936
 937         nvmewqeq = lpfc_nvmet_prep_fcp_wqe(phba, ctxp);
 938         if (nvmewqeq == NULL) {
 939                 atomic_inc(&lpfc_nvmep->xmt_fcp_drop);
 940                 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
 941                                 "6152 FCP Drop IO x%x: Prep\n",
 942                                 ctxp->oxid);
 943                 rc = -ENXIO;
 944                 goto aerr;
 945         }
 946
 947         nvmewqeq->wqe_cmpl = lpfc_nvmet_xmt_fcp_op_cmp;
 948         nvmewqeq->iocb_cmpl = NULL;
 949         nvmewqeq->context2 = ctxp;
 950         nvmewqeq->iocb_flag |=  LPFC_IO_NVMET;
 951         ctxp->wqeq->hba_wqidx = rsp->hwqid;
 952
 953         lpfc_nvmeio_data(phba, "NVMET FCP CMND: xri x%x op x%x len x%x\n",
 954                          ctxp->oxid, rsp->op, rsp->rsplen);
 955
 956         ctxp->flag |= LPFC_NVMET_IO_INP;
 957         rc = lpfc_sli4_issue_wqe(phba, LPFC_FCP_RING, nvmewqeq);
 958         if (rc == WQE_SUCCESS) {
 959 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
 960                 if (!ctxp->ts_cmd_nvme)
 961                         return 0;
 962                 if (rsp->op == NVMET_FCOP_RSP)
 963                         ctxp->ts_status_wqput = ktime_get_ns();
 964                 else
 965                         ctxp->ts_data_wqput = ktime_get_ns();
 966 #endif
 967                 return 0;
 968         }
 969
 970         if (rc == -EBUSY) {
 971                 /*
 972                  * WQ was full, so queue nvmewqeq to be sent after
 973                  * WQE release CQE
 974                  */
 975                 ctxp->flag |= LPFC_NVMET_DEFER_WQFULL;
 976                 wq = phba->sli4_hba.nvme_wq[rsp->hwqid];
 977                 pring = wq->pring;
 978                 spin_lock_irqsave(&pring->ring_lock, iflags);
 979                 list_add_tail(&nvmewqeq->list, &wq->wqfull_list);
 980                 wq->q_flag |= HBA_NVMET_WQFULL;
 981                 spin_unlock_irqrestore(&pring->ring_lock, iflags);
 982                 atomic_inc(&lpfc_nvmep->defer_wqfull);
 983                 return 0;
 984         }
 985
 986         /* Give back resources */
 987         atomic_inc(&lpfc_nvmep->xmt_fcp_drop);
 988         lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
 989                         "6153 FCP Drop IO x%x: Issue: %d\n",
 990                         ctxp->oxid, rc);
 991
 992         ctxp->wqeq->hba_wqidx = 0;
 993         nvmewqeq->context2 = NULL;
 994         nvmewqeq->context3 = NULL;
 995         rc = -EBUSY;
 996 aerr:
 997         return rc;
 998 }
 999
1000 static void
1001 lpfc_nvmet_targetport_delete(struct nvmet_fc_target_port *targetport)
1002 {
1003         struct lpfc_nvmet_tgtport *tport = targetport->private;
1004
1005         /* release any threads waiting for the unreg to complete */
1006         if (tport->phba->targetport)
1007                 complete(tport->tport_unreg_cmp);
1008 }
1009
1010 static void
1011 lpfc_nvmet_xmt_fcp_abort(struct nvmet_fc_target_port *tgtport,
1012                          struct nvmefc_tgt_fcp_req *req)
1013 {
1014         struct lpfc_nvmet_tgtport *lpfc_nvmep = tgtport->private;
1015         struct lpfc_nvmet_rcv_ctx *ctxp =
1016                 container_of(req, struct lpfc_nvmet_rcv_ctx, ctx.fcp_req);
1017         struct lpfc_hba *phba = ctxp->phba;
1018         struct lpfc_queue *wq;
1019         unsigned long flags;
1020
1021         if (phba->pport->load_flag & FC_UNLOADING)
1022                 return;
1023
1024         if (phba->pport->load_flag & FC_UNLOADING)
1025                 return;
1026
1027         lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
1028                         "6103 NVMET Abort op: oxri x%x flg x%x ste %d\n",
1029                         ctxp->oxid, ctxp->flag, ctxp->state);
1030
1031         lpfc_nvmeio_data(phba, "NVMET FCP ABRT: xri x%x flg x%x ste x%x\n",
1032                          ctxp->oxid, ctxp->flag, ctxp->state);
1033
1034         atomic_inc(&lpfc_nvmep->xmt_fcp_abort);
1035
1036         spin_lock_irqsave(&ctxp->ctxlock, flags);
1037         ctxp->state = LPFC_NVMET_STE_ABORT;
1038
1039         /* Since iaab/iaar are NOT set, we need to check
1040          * if the firmware is in process of aborting IO
1041          */
1042         if (ctxp->flag & LPFC_NVMET_XBUSY) {
1043                 spin_unlock_irqrestore(&ctxp->ctxlock, flags);
1044                 return;
1045         }
1046         ctxp->flag |= LPFC_NVMET_ABORT_OP;
1047
1048         if (ctxp->flag & LPFC_NVMET_DEFER_WQFULL) {
1049                 lpfc_nvmet_unsol_fcp_issue_abort(phba, ctxp, ctxp->sid,
1050                                                  ctxp->oxid);
1051                 wq = phba->sli4_hba.nvme_wq[ctxp->wqeq->hba_wqidx];
1052                 spin_unlock_irqrestore(&ctxp->ctxlock, flags);
1053                 lpfc_nvmet_wqfull_flush(phba, wq, ctxp);
1054                 return;
1055         }
1056
1057         /* An state of LPFC_NVMET_STE_RCV means we have just received
1058          * the NVME command and have not started processing it.
1059          * (by issuing any IO WQEs on this exchange yet)
1060          */
1061         if (ctxp->state == LPFC_NVMET_STE_RCV)
1062                 lpfc_nvmet_unsol_fcp_issue_abort(phba, ctxp, ctxp->sid,
1063                                                  ctxp->oxid);
1064         else
1065                 lpfc_nvmet_sol_fcp_issue_abort(phba, ctxp, ctxp->sid,
1066                                                ctxp->oxid);
1067         spin_unlock_irqrestore(&ctxp->ctxlock, flags);
1068 }
1069
1070 static void
1071 lpfc_nvmet_xmt_fcp_release(struct nvmet_fc_target_port *tgtport,
1072                            struct nvmefc_tgt_fcp_req *rsp)
1073 {
1074         struct lpfc_nvmet_tgtport *lpfc_nvmep = tgtport->private;
1075         struct lpfc_nvmet_rcv_ctx *ctxp =
1076                 container_of(rsp, struct lpfc_nvmet_rcv_ctx, ctx.fcp_req);
1077         struct lpfc_hba *phba = ctxp->phba;
1078         unsigned long flags;
1079         bool aborting = false;
1080
1081         if (ctxp->state != LPFC_NVMET_STE_DONE &&
1082             ctxp->state != LPFC_NVMET_STE_ABORT) {
1083                 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
1084                                 "6413 NVMET release bad state %d %d oxid x%x\n",
1085                                 ctxp->state, ctxp->entry_cnt, ctxp->oxid);
1086         }
1087
1088         spin_lock_irqsave(&ctxp->ctxlock, flags);
1089         if ((ctxp->flag & LPFC_NVMET_ABORT_OP) ||
1090             (ctxp->flag & LPFC_NVMET_XBUSY)) {
1091                 aborting = true;
1092                 /* let the abort path do the real release */
1093                 lpfc_nvmet_defer_release(phba, ctxp);
1094         }
1095         spin_unlock_irqrestore(&ctxp->ctxlock, flags);
1096
1097         lpfc_nvmeio_data(phba, "NVMET FCP FREE: xri x%x ste %d abt %d\n", ctxp->oxid,
1098                          ctxp->state, aborting);
1099
1100         atomic_inc(&lpfc_nvmep->xmt_fcp_release);
1101
1102         if (aborting)
1103                 return;
1104
1105         lpfc_nvmet_ctxbuf_post(phba, ctxp->ctxbuf);
1106 }
1107
1108 static void
1109 lpfc_nvmet_defer_rcv(struct nvmet_fc_target_port *tgtport,
1110                      struct nvmefc_tgt_fcp_req *rsp)
1111 {
1112         struct lpfc_nvmet_tgtport *tgtp;
1113         struct lpfc_nvmet_rcv_ctx *ctxp =
1114                 container_of(rsp, struct lpfc_nvmet_rcv_ctx, ctx.fcp_req);
1115         struct rqb_dmabuf *nvmebuf = ctxp->rqb_buffer;
1116         struct lpfc_hba *phba = ctxp->phba;
1117
1118         lpfc_nvmeio_data(phba, "NVMET DEFERRCV: xri x%x sz %d CPU %02x\n",
1119                          ctxp->oxid, ctxp->size, smp_processor_id());
1120
1121         if (!nvmebuf) {
1122                 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_IOERR,
1123                                 "6425 Defer rcv: no buffer xri x%x: "
1124                                 "flg %x ste %x\n",
1125                                 ctxp->oxid, ctxp->flag, ctxp->state);
1126                 return;
1127         }
1128
1129         tgtp = phba->targetport->private;
1130         if (tgtp)
1131                 atomic_inc(&tgtp->rcv_fcp_cmd_defer);
1132
1133         /* Free the nvmebuf since a new buffer already replaced it */
1134         nvmebuf->hrq->rqbp->rqb_free_buffer(phba, nvmebuf);
1135 }
1136
1137 static struct nvmet_fc_target_template lpfc_tgttemplate = {
1138         .targetport_delete = lpfc_nvmet_targetport_delete,
1139         .xmt_ls_rsp     = lpfc_nvmet_xmt_ls_rsp,
1140         .fcp_op         = lpfc_nvmet_xmt_fcp_op,
1141         .fcp_abort      = lpfc_nvmet_xmt_fcp_abort,
1142         .fcp_req_release = lpfc_nvmet_xmt_fcp_release,
1143         .defer_rcv      = lpfc_nvmet_defer_rcv,
1144
1145         .max_hw_queues  = 1,
1146         .max_sgl_segments = LPFC_NVMET_DEFAULT_SEGS,
1147         .max_dif_sgl_segments = LPFC_NVMET_DEFAULT_SEGS,
1148         .dma_boundary = 0xFFFFFFFF,
1149
1150         /* optional features */
1151         .target_features = 0,
1152         /* sizes of additional private data for data structures */
1153         .target_priv_sz = sizeof(struct lpfc_nvmet_tgtport),
1154 };
1155
1156 static void
1157 __lpfc_nvmet_clean_io_for_cpu(struct lpfc_hba *phba,
1158                 struct lpfc_nvmet_ctx_info *infop)
1159 {
1160         struct lpfc_nvmet_ctxbuf *ctx_buf, *next_ctx_buf;
1161         unsigned long flags;
1162
1163         spin_lock_irqsave(&infop->nvmet_ctx_list_lock, flags);
1164         list_for_each_entry_safe(ctx_buf, next_ctx_buf,
1165                                 &infop->nvmet_ctx_list, list) {
1166                 spin_lock(&phba->sli4_hba.abts_nvme_buf_list_lock);
1167                 list_del_init(&ctx_buf->list);
1168                 spin_unlock(&phba->sli4_hba.abts_nvme_buf_list_lock);
1169
1170                 __lpfc_clear_active_sglq(phba, ctx_buf->sglq->sli4_lxritag);
1171                 ctx_buf->sglq->state = SGL_FREED;
1172                 ctx_buf->sglq->ndlp = NULL;
1173
1174                 spin_lock(&phba->sli4_hba.sgl_list_lock);
1175                 list_add_tail(&ctx_buf->sglq->list,
1176                                 &phba->sli4_hba.lpfc_nvmet_sgl_list);
1177                 spin_unlock(&phba->sli4_hba.sgl_list_lock);
1178
1179                 lpfc_sli_release_iocbq(phba, ctx_buf->iocbq);
1180                 kfree(ctx_buf->context);
1181         }
1182         spin_unlock_irqrestore(&infop->nvmet_ctx_list_lock, flags);
1183 }
1184
1185 static void
1186 lpfc_nvmet_cleanup_io_context(struct lpfc_hba *phba)
1187 {
1188         struct lpfc_nvmet_ctx_info *infop;
1189         int i, j;
1190
1191         /* The first context list, MRQ 0 CPU 0 */
1192         infop = phba->sli4_hba.nvmet_ctx_info;
1193         if (!infop)
1194                 return;
1195
1196         /* Cycle the the entire CPU context list for every MRQ */
1197         for (i = 0; i < phba->cfg_nvmet_mrq; i++) {
1198                 for (j = 0; j < phba->sli4_hba.num_present_cpu; j++) {
1199                         __lpfc_nvmet_clean_io_for_cpu(phba, infop);
1200                         infop++; /* next */
1201                 }
1202         }
1203         kfree(phba->sli4_hba.nvmet_ctx_info);
1204         phba->sli4_hba.nvmet_ctx_info = NULL;
1205 }
1206
1207 static int
1208 lpfc_nvmet_setup_io_context(struct lpfc_hba *phba)
1209 {
1210         struct lpfc_nvmet_ctxbuf *ctx_buf;
1211         struct lpfc_iocbq *nvmewqe;
1212         union lpfc_wqe128 *wqe;
1213         struct lpfc_nvmet_ctx_info *last_infop;
1214         struct lpfc_nvmet_ctx_info *infop;
1215         int i, j, idx;
1216
1217         lpfc_printf_log(phba, KERN_INFO, LOG_NVME,
1218                         "6403 Allocate NVMET resources for %d XRIs\n",
1219                         phba->sli4_hba.nvmet_xri_cnt);
1220
1221         phba->sli4_hba.nvmet_ctx_info = kcalloc(
1222                 phba->sli4_hba.num_present_cpu * phba->cfg_nvmet_mrq,
1223                 sizeof(struct lpfc_nvmet_ctx_info), GFP_KERNEL);
1224         if (!phba->sli4_hba.nvmet_ctx_info) {
1225                 lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
1226                                 "6419 Failed allocate memory for "
1227                                 "nvmet context lists\n");
1228                 return -ENOMEM;
1229         }
1230
1231         /*
1232          * Assuming X CPUs in the system, and Y MRQs, allocate some
1233          * lpfc_nvmet_ctx_info structures as follows:
1234          *
1235          * cpu0/mrq0 cpu1/mrq0 ... cpuX/mrq0
1236          * cpu0/mrq1 cpu1/mrq1 ... cpuX/mrq1
1237          * ...
1238          * cpuX/mrqY cpuX/mrqY ... cpuX/mrqY
1239          *
1240          * Each line represents a MRQ "silo" containing an entry for
1241          * every CPU.
1242          *
1243          * MRQ X is initially assumed to be associated with CPU X, thus
1244          * contexts are initially distributed across all MRQs using
1245          * the MRQ index (N) as follows cpuN/mrqN. When contexts are
1246          * freed, the are freed to the MRQ silo based on the CPU number
1247          * of the IO completion. Thus a context that was allocated for MRQ A
1248          * whose IO completed on CPU B will be freed to cpuB/mrqA.
1249          */
1250         infop = phba->sli4_hba.nvmet_ctx_info;
1251         for (i = 0; i < phba->sli4_hba.num_present_cpu; i++) {
1252                 for (j = 0; j < phba->cfg_nvmet_mrq; j++) {
1253                         INIT_LIST_HEAD(&infop->nvmet_ctx_list);
1254                         spin_lock_init(&infop->nvmet_ctx_list_lock);
1255                         infop->nvmet_ctx_list_cnt = 0;
1256                         infop++;
1257                 }
1258         }
1259
1260         /*
1261          * Setup the next CPU context info ptr for each MRQ.
1262          * MRQ 0 will cycle thru CPUs 0 - X separately from
1263          * MRQ 1 cycling thru CPUs 0 - X, and so on.
1264          */
1265         for (j = 0; j < phba->cfg_nvmet_mrq; j++) {
1266                 last_infop = lpfc_get_ctx_list(phba, 0, j);
1267                 for (i = phba->sli4_hba.num_present_cpu - 1;  i >= 0; i--) {
1268                         infop = lpfc_get_ctx_list(phba, i, j);
1269                         infop->nvmet_ctx_next_cpu = last_infop;
1270                         last_infop = infop;
1271                 }
1272         }
1273
1274         /* For all nvmet xris, allocate resources needed to process a
1275          * received command on a per xri basis.
1276          */
1277         idx = 0;
1278         for (i = 0; i < phba->sli4_hba.nvmet_xri_cnt; i++) {
1279                 ctx_buf = kzalloc(sizeof(*ctx_buf), GFP_KERNEL);
1280                 if (!ctx_buf) {
1281                         lpfc_printf_log(phba, KERN_ERR, LOG_NVME,
1282                                         "6404 Ran out of memory for NVMET\n");
1283                         return -ENOMEM;
1284                 }
1285
1286                 ctx_buf->context = kzalloc(sizeof(*ctx_buf->context),
1287                                            GFP_KERNEL);
1288                 if (!ctx_buf->context) {
1289                         kfree(ctx_buf);
1290                         lpfc_printf_log(phba, KERN_ERR, LOG_NVME,
1291                                         "6405 Ran out of NVMET "
1292                                         "context memory\n");
1293                         return -ENOMEM;
1294                 }
1295                 ctx_buf->context->ctxbuf = ctx_buf;
1296                 ctx_buf->context->state = LPFC_NVMET_STE_FREE;
1297
1298                 ctx_buf->iocbq = lpfc_sli_get_iocbq(phba);
1299                 if (!ctx_buf->iocbq) {
1300                         kfree(ctx_buf->context);
1301                         kfree(ctx_buf);
1302                         lpfc_printf_log(phba, KERN_ERR, LOG_NVME,
1303                                         "6406 Ran out of NVMET iocb/WQEs\n");
1304                         return -ENOMEM;
1305                 }
1306                 ctx_buf->iocbq->iocb_flag = LPFC_IO_NVMET;
1307                 nvmewqe = ctx_buf->iocbq;
1308                 wqe = &nvmewqe->wqe;
1309
1310                 /* Initialize WQE */
1311                 memset(wqe, 0, sizeof(union lpfc_wqe));
1312
1313                 ctx_buf->iocbq->context1 = NULL;
1314                 spin_lock(&phba->sli4_hba.sgl_list_lock);
1315                 ctx_buf->sglq = __lpfc_sli_get_nvmet_sglq(phba, ctx_buf->iocbq);
1316                 spin_unlock(&phba->sli4_hba.sgl_list_lock);
1317                 if (!ctx_buf->sglq) {
1318                         lpfc_sli_release_iocbq(phba, ctx_buf->iocbq);
1319                         kfree(ctx_buf->context);
1320                         kfree(ctx_buf);
1321                         lpfc_printf_log(phba, KERN_ERR, LOG_NVME,
1322                                         "6407 Ran out of NVMET XRIs\n");
1323                         return -ENOMEM;
1324                 }
1325
1326                 /*
1327                  * Add ctx to MRQidx context list. Our initial assumption
1328                  * is MRQidx will be associated with CPUidx. This association
1329                  * can change on the fly.
1330                  */
1331                 infop = lpfc_get_ctx_list(phba, idx, idx);
1332                 spin_lock(&infop->nvmet_ctx_list_lock);
1333                 list_add_tail(&ctx_buf->list, &infop->nvmet_ctx_list);
1334                 infop->nvmet_ctx_list_cnt++;
1335                 spin_unlock(&infop->nvmet_ctx_list_lock);
1336
1337                 /* Spread ctx structures evenly across all MRQs */
1338                 idx++;
1339                 if (idx >= phba->cfg_nvmet_mrq)
1340                         idx = 0;
1341         }
1342
1343         for (i = 0; i < phba->sli4_hba.num_present_cpu; i++) {
1344                 for (j = 0; j < phba->cfg_nvmet_mrq; j++) {
1345                         infop = lpfc_get_ctx_list(phba, i, j);
1346                         lpfc_printf_log(phba, KERN_INFO, LOG_NVME | LOG_INIT,
1347                                         "6408 TOTAL NVMET ctx for CPU %d "
1348                                         "MRQ %d: cnt %d nextcpu %p\n",
1349                                         i, j, infop->nvmet_ctx_list_cnt,
1350                                         infop->nvmet_ctx_next_cpu);
1351                 }
1352         }
1353         return 0;
1354 }
1355
1356 int
1357 lpfc_nvmet_create_targetport(struct lpfc_hba *phba)
1358 {
1359         struct lpfc_vport  *vport = phba->pport;
1360         struct lpfc_nvmet_tgtport *tgtp;
1361         struct nvmet_fc_port_info pinfo;
1362         int error;
1363
1364         if (phba->targetport)
1365                 return 0;
1366
1367         error = lpfc_nvmet_setup_io_context(phba);
1368         if (error)
1369                 return error;
1370
1371         memset(&pinfo, 0, sizeof(struct nvmet_fc_port_info));
1372         pinfo.node_name = wwn_to_u64(vport->fc_nodename.u.wwn);
1373         pinfo.port_name = wwn_to_u64(vport->fc_portname.u.wwn);
1374         pinfo.port_id = vport->fc_myDID;
1375
1376         /* Limit to LPFC_MAX_NVME_SEG_CNT.
1377          * For now need + 1 to get around NVME transport logic.
1378          */
1379         if (phba->cfg_sg_seg_cnt > LPFC_MAX_NVME_SEG_CNT) {
1380                 lpfc_printf_log(phba, KERN_INFO, LOG_NVME | LOG_INIT,
1381                                 "6400 Reducing sg segment cnt to %d\n",
1382                                 LPFC_MAX_NVME_SEG_CNT);
1383                 phba->cfg_nvme_seg_cnt = LPFC_MAX_NVME_SEG_CNT;
1384         } else {
1385                 phba->cfg_nvme_seg_cnt = phba->cfg_sg_seg_cnt;
1386         }
1387         lpfc_tgttemplate.max_sgl_segments = phba->cfg_nvme_seg_cnt + 1;
1388         lpfc_tgttemplate.max_hw_queues = phba->cfg_nvme_io_channel;
1389         lpfc_tgttemplate.target_features = NVMET_FCTGTFEAT_READDATA_RSP;
1390
1391 #if (IS_ENABLED(CONFIG_NVME_TARGET_FC))
1392         error = nvmet_fc_register_targetport(&pinfo, &lpfc_tgttemplate,
1393                                              &phba->pcidev->dev,
1394                                              &phba->targetport);
1395 #else
1396         error = -ENOENT;
1397 #endif
1398         if (error) {
1399                 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_DISC,
1400                                 "6025 Cannot register NVME targetport x%x: "
1401                                 "portnm %llx nodenm %llx segs %d qs %d\n",
1402                                 error,
1403                                 pinfo.port_name, pinfo.node_name,
1404                                 lpfc_tgttemplate.max_sgl_segments,
1405                                 lpfc_tgttemplate.max_hw_queues);
1406                 phba->targetport = NULL;
1407                 phba->nvmet_support = 0;
1408
1409                 lpfc_nvmet_cleanup_io_context(phba);
1410
1411         } else {
1412                 tgtp = (struct lpfc_nvmet_tgtport *)
1413                         phba->targetport->private;
1414                 tgtp->phba = phba;
1415
1416                 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_DISC,
1417                                 "6026 Registered NVME "
1418                                 "targetport: %p, private %p "
1419                                 "portnm %llx nodenm %llx segs %d qs %d\n",
1420                                 phba->targetport, tgtp,
1421                                 pinfo.port_name, pinfo.node_name,
1422                                 lpfc_tgttemplate.max_sgl_segments,
1423                                 lpfc_tgttemplate.max_hw_queues);
1424
1425                 atomic_set(&tgtp->rcv_ls_req_in, 0);
1426                 atomic_set(&tgtp->rcv_ls_req_out, 0);
1427                 atomic_set(&tgtp->rcv_ls_req_drop, 0);
1428                 atomic_set(&tgtp->xmt_ls_abort, 0);
1429                 atomic_set(&tgtp->xmt_ls_abort_cmpl, 0);
1430                 atomic_set(&tgtp->xmt_ls_rsp, 0);
1431                 atomic_set(&tgtp->xmt_ls_drop, 0);
1432                 atomic_set(&tgtp->xmt_ls_rsp_error, 0);
1433                 atomic_set(&tgtp->xmt_ls_rsp_xb_set, 0);
1434                 atomic_set(&tgtp->xmt_ls_rsp_aborted, 0);
1435                 atomic_set(&tgtp->xmt_ls_rsp_cmpl, 0);
1436                 atomic_set(&tgtp->rcv_fcp_cmd_in, 0);
1437                 atomic_set(&tgtp->rcv_fcp_cmd_out, 0);
1438                 atomic_set(&tgtp->rcv_fcp_cmd_drop, 0);
1439                 atomic_set(&tgtp->xmt_fcp_drop, 0);
1440                 atomic_set(&tgtp->xmt_fcp_read_rsp, 0);
1441                 atomic_set(&tgtp->xmt_fcp_read, 0);
1442                 atomic_set(&tgtp->xmt_fcp_write, 0);
1443                 atomic_set(&tgtp->xmt_fcp_rsp, 0);
1444                 atomic_set(&tgtp->xmt_fcp_release, 0);
1445                 atomic_set(&tgtp->xmt_fcp_rsp_cmpl, 0);
1446                 atomic_set(&tgtp->xmt_fcp_rsp_error, 0);
1447                 atomic_set(&tgtp->xmt_fcp_rsp_xb_set, 0);
1448                 atomic_set(&tgtp->xmt_fcp_rsp_aborted, 0);
1449                 atomic_set(&tgtp->xmt_fcp_rsp_drop, 0);
1450                 atomic_set(&tgtp->xmt_fcp_xri_abort_cqe, 0);
1451                 atomic_set(&tgtp->xmt_fcp_abort, 0);
1452                 atomic_set(&tgtp->xmt_fcp_abort_cmpl, 0);
1453                 atomic_set(&tgtp->xmt_abort_unsol, 0);
1454                 atomic_set(&tgtp->xmt_abort_sol, 0);
1455                 atomic_set(&tgtp->xmt_abort_rsp, 0);
1456                 atomic_set(&tgtp->xmt_abort_rsp_error, 0);
1457                 atomic_set(&tgtp->defer_ctx, 0);
1458                 atomic_set(&tgtp->defer_fod, 0);
1459                 atomic_set(&tgtp->defer_wqfull, 0);
1460         }
1461         return error;
1462 }
1463
1464 int
1465 lpfc_nvmet_update_targetport(struct lpfc_hba *phba)
1466 {
1467         struct lpfc_vport  *vport = phba->pport;
1468
1469         if (!phba->targetport)
1470                 return 0;
1471
1472         lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME,
1473                          "6007 Update NVMET port %p did x%x\n",
1474                          phba->targetport, vport->fc_myDID);
1475
1476         phba->targetport->port_id = vport->fc_myDID;
1477         return 0;
1478 }
1479
1480 /**
1481  * lpfc_sli4_nvmet_xri_aborted - Fast-path process of nvmet xri abort
1482  * @phba: pointer to lpfc hba data structure.
1483  * @axri: pointer to the nvmet xri abort wcqe structure.
1484  *
1485  * This routine is invoked by the worker thread to process a SLI4 fast-path
1486  * NVMET aborted xri.
1487  **/
1488 void
1489 lpfc_sli4_nvmet_xri_aborted(struct lpfc_hba *phba,
1490                             struct sli4_wcqe_xri_aborted *axri)
1491 {
1492         uint16_t xri = bf_get(lpfc_wcqe_xa_xri, axri);
1493         uint16_t rxid = bf_get(lpfc_wcqe_xa_remote_xid, axri);
1494         struct lpfc_nvmet_rcv_ctx *ctxp, *next_ctxp;
1495         struct lpfc_nvmet_tgtport *tgtp;
1496         struct lpfc_nodelist *ndlp;
1497         unsigned long iflag = 0;
1498         int rrq_empty = 0;
1499         bool released = false;
1500
1501         lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
1502                         "6317 XB aborted xri x%x rxid x%x\n", xri, rxid);
1503
1504         if (!(phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME))
1505                 return;
1506
1507         if (phba->targetport) {
1508                 tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
1509                 atomic_inc(&tgtp->xmt_fcp_xri_abort_cqe);
1510         }
1511
1512         spin_lock_irqsave(&phba->hbalock, iflag);
1513         spin_lock(&phba->sli4_hba.abts_nvme_buf_list_lock);
1514         list_for_each_entry_safe(ctxp, next_ctxp,
1515                                  &phba->sli4_hba.lpfc_abts_nvmet_ctx_list,
1516                                  list) {
1517                 if (ctxp->ctxbuf->sglq->sli4_xritag != xri)
1518                         continue;
1519
1520                 /* Check if we already received a free context call
1521                  * and we have completed processing an abort situation.
1522                  */
1523                 if (ctxp->flag & LPFC_NVMET_CTX_RLS &&
1524                     !(ctxp->flag & LPFC_NVMET_ABORT_OP)) {
1525                         list_del(&ctxp->list);
1526                         released = true;
1527                 }
1528                 ctxp->flag &= ~LPFC_NVMET_XBUSY;
1529                 spin_unlock(&phba->sli4_hba.abts_nvme_buf_list_lock);
1530
1531                 rrq_empty = list_empty(&phba->active_rrq_list);
1532                 spin_unlock_irqrestore(&phba->hbalock, iflag);
1533                 ndlp = lpfc_findnode_did(phba->pport, ctxp->sid);
1534                 if (ndlp && NLP_CHK_NODE_ACT(ndlp) &&
1535                     (ndlp->nlp_state == NLP_STE_UNMAPPED_NODE ||
1536                      ndlp->nlp_state == NLP_STE_MAPPED_NODE)) {
1537                         lpfc_set_rrq_active(phba, ndlp,
1538                                 ctxp->ctxbuf->sglq->sli4_lxritag,
1539                                 rxid, 1);
1540                         lpfc_sli4_abts_err_handler(phba, ndlp, axri);
1541                 }
1542
1543                 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
1544                                 "6318 XB aborted oxid %x flg x%x (%x)\n",
1545                                 ctxp->oxid, ctxp->flag, released);
1546                 if (released)
1547                         lpfc_nvmet_ctxbuf_post(phba, ctxp->ctxbuf);
1548
1549                 if (rrq_empty)
1550                         lpfc_worker_wake_up(phba);
1551                 return;
1552         }
1553         spin_unlock(&phba->sli4_hba.abts_nvme_buf_list_lock);
1554         spin_unlock_irqrestore(&phba->hbalock, iflag);
1555 }
1556
1557 int
1558 lpfc_nvmet_rcv_unsol_abort(struct lpfc_vport *vport,
1559                            struct fc_frame_header *fc_hdr)
1560
1561 {
1562 #if (IS_ENABLED(CONFIG_NVME_TARGET_FC))
1563         struct lpfc_hba *phba = vport->phba;
1564         struct lpfc_nvmet_rcv_ctx *ctxp, *next_ctxp;
1565         struct nvmefc_tgt_fcp_req *rsp;
1566         uint16_t xri;
1567         unsigned long iflag = 0;
1568
1569         xri = be16_to_cpu(fc_hdr->fh_ox_id);
1570
1571         spin_lock_irqsave(&phba->hbalock, iflag);
1572         spin_lock(&phba->sli4_hba.abts_nvme_buf_list_lock);
1573         list_for_each_entry_safe(ctxp, next_ctxp,
1574                                  &phba->sli4_hba.lpfc_abts_nvmet_ctx_list,
1575                                  list) {
1576                 if (ctxp->ctxbuf->sglq->sli4_xritag != xri)
1577                         continue;
1578
1579                 spin_unlock(&phba->sli4_hba.abts_nvme_buf_list_lock);
1580                 spin_unlock_irqrestore(&phba->hbalock, iflag);
1581
1582                 spin_lock_irqsave(&ctxp->ctxlock, iflag);
1583                 ctxp->flag |= LPFC_NVMET_ABTS_RCV;
1584                 spin_unlock_irqrestore(&ctxp->ctxlock, iflag);
1585
1586                 lpfc_nvmeio_data(phba,
1587                         "NVMET ABTS RCV: xri x%x CPU %02x rjt %d\n",
1588                         xri, smp_processor_id(), 0);
1589
1590                 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
1591                                 "6319 NVMET Rcv ABTS:acc xri x%x\n", xri);
1592
1593                 rsp = &ctxp->ctx.fcp_req;
1594                 nvmet_fc_rcv_fcp_abort(phba->targetport, rsp);
1595
1596                 /* Respond with BA_ACC accordingly */
1597                 lpfc_sli4_seq_abort_rsp(vport, fc_hdr, 1);
1598                 return 0;
1599         }
1600         spin_unlock(&phba->sli4_hba.abts_nvme_buf_list_lock);
1601         spin_unlock_irqrestore(&phba->hbalock, iflag);
1602
1603         lpfc_nvmeio_data(phba, "NVMET ABTS RCV: xri x%x CPU %02x rjt %d\n",
1604                          xri, smp_processor_id(), 1);
1605
1606         lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
1607                         "6320 NVMET Rcv ABTS:rjt xri x%x\n", xri);
1608
1609         /* Respond with BA_RJT accordingly */
1610         lpfc_sli4_seq_abort_rsp(vport, fc_hdr, 0);
1611 #endif
1612         return 0;
1613 }
1614
1615 static void
1616 lpfc_nvmet_wqfull_flush(struct lpfc_hba *phba, struct lpfc_queue *wq,
1617                         struct lpfc_nvmet_rcv_ctx *ctxp)
1618 {
1619         struct lpfc_sli_ring *pring;
1620         struct lpfc_iocbq *nvmewqeq;
1621         struct lpfc_iocbq *next_nvmewqeq;
1622         unsigned long iflags;
1623         struct lpfc_wcqe_complete wcqe;
1624         struct lpfc_wcqe_complete *wcqep;
1625
1626         pring = wq->pring;
1627         wcqep = &wcqe;
1628
1629         /* Fake an ABORT error code back to cmpl routine */
1630         memset(wcqep, 0, sizeof(struct lpfc_wcqe_complete));
1631         bf_set(lpfc_wcqe_c_status, wcqep, IOSTAT_LOCAL_REJECT);
1632         wcqep->parameter = IOERR_ABORT_REQUESTED;
1633
1634         spin_lock_irqsave(&pring->ring_lock, iflags);
1635         list_for_each_entry_safe(nvmewqeq, next_nvmewqeq,
1636                                  &wq->wqfull_list, list) {
1637                 if (ctxp) {
1638                         /* Checking for a specific IO to flush */
1639                         if (nvmewqeq->context2 == ctxp) {
1640                                 list_del(&nvmewqeq->list);
1641                                 spin_unlock_irqrestore(&pring->ring_lock,
1642                                                        iflags);
1643                                 lpfc_nvmet_xmt_fcp_op_cmp(phba, nvmewqeq,
1644                                                           wcqep);
1645                                 return;
1646                         }
1647                         continue;
1648                 } else {
1649                         /* Flush all IOs */
1650                         list_del(&nvmewqeq->list);
1651                         spin_unlock_irqrestore(&pring->ring_lock, iflags);
1652                         lpfc_nvmet_xmt_fcp_op_cmp(phba, nvmewqeq, wcqep);
1653                         spin_lock_irqsave(&pring->ring_lock, iflags);
1654                 }
1655         }
1656         if (!ctxp)
1657                 wq->q_flag &= ~HBA_NVMET_WQFULL;
1658         spin_unlock_irqrestore(&pring->ring_lock, iflags);
1659 }
1660
1661 void
1662 lpfc_nvmet_wqfull_process(struct lpfc_hba *phba,
1663                           struct lpfc_queue *wq)
1664 {
1665 #if (IS_ENABLED(CONFIG_NVME_TARGET_FC))
1666         struct lpfc_sli_ring *pring;
1667         struct lpfc_iocbq *nvmewqeq;
1668         unsigned long iflags;
1669         int rc;
1670
1671         /*
1672          * Some WQE slots are available, so try to re-issue anything
1673          * on the WQ wqfull_list.
1674          */
1675         pring = wq->pring;
1676         spin_lock_irqsave(&pring->ring_lock, iflags);
1677         while (!list_empty(&wq->wqfull_list)) {
1678                 list_remove_head(&wq->wqfull_list, nvmewqeq, struct lpfc_iocbq,
1679                                  list);
1680                 spin_unlock_irqrestore(&pring->ring_lock, iflags);
1681                 rc = lpfc_sli4_issue_wqe(phba, LPFC_FCP_RING, nvmewqeq);
1682                 spin_lock_irqsave(&pring->ring_lock, iflags);
1683                 if (rc == -EBUSY) {
1684                         /* WQ was full again, so put it back on the list */
1685                         list_add(&nvmewqeq->list, &wq->wqfull_list);
1686                         spin_unlock_irqrestore(&pring->ring_lock, iflags);
1687                         return;
1688                 }
1689         }
1690         wq->q_flag &= ~HBA_NVMET_WQFULL;
1691         spin_unlock_irqrestore(&pring->ring_lock, iflags);
1692
1693 #endif
1694 }
1695
1696 void
1697 lpfc_nvmet_destroy_targetport(struct lpfc_hba *phba)
1698 {
1699 #if (IS_ENABLED(CONFIG_NVME_TARGET_FC))
1700         struct lpfc_nvmet_tgtport *tgtp;
1701         struct lpfc_queue *wq;
1702         uint32_t qidx;
1703         DECLARE_COMPLETION_ONSTACK(tport_unreg_cmp);
1704
1705         if (phba->nvmet_support == 0)
1706                 return;
1707         if (phba->targetport) {
1708                 tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
1709                 for (qidx = 0; qidx < phba->cfg_nvme_io_channel; qidx++) {
1710                         wq = phba->sli4_hba.nvme_wq[qidx];
1711                         lpfc_nvmet_wqfull_flush(phba, wq, NULL);
1712                 }
1713                 tgtp->tport_unreg_cmp = &tport_unreg_cmp;
1714                 nvmet_fc_unregister_targetport(phba->targetport);
1715                 if (!wait_for_completion_timeout(&tport_unreg_cmp,
1716                                         msecs_to_jiffies(LPFC_NVMET_WAIT_TMO)))
1717                         lpfc_printf_log(phba, KERN_ERR, LOG_NVME,
1718                                         "6179 Unreg targetport %p timeout "
1719                                         "reached.\n", phba->targetport);
1720                 lpfc_nvmet_cleanup_io_context(phba);
1721         }
1722         phba->targetport = NULL;
1723 #endif
1724 }
1725
1726 /**
1727  * lpfc_nvmet_unsol_ls_buffer - Process an unsolicited event data buffer
1728  * @phba: pointer to lpfc hba data structure.
1729  * @pring: pointer to a SLI ring.
1730  * @nvmebuf: pointer to lpfc nvme command HBQ data structure.
1731  *
1732  * This routine is used for processing the WQE associated with a unsolicited
1733  * event. It first determines whether there is an existing ndlp that matches
1734  * the DID from the unsolicited WQE. If not, it will create a new one with
1735  * the DID from the unsolicited WQE. The ELS command from the unsolicited
1736  * WQE is then used to invoke the proper routine and to set up proper state
1737  * of the discovery state machine.
1738  **/
1739 static void
1740 lpfc_nvmet_unsol_ls_buffer(struct lpfc_hba *phba, struct lpfc_sli_ring *pring,
1741                            struct hbq_dmabuf *nvmebuf)
1742 {
1743 #if (IS_ENABLED(CONFIG_NVME_TARGET_FC))
1744         struct lpfc_nvmet_tgtport *tgtp;
1745         struct fc_frame_header *fc_hdr;
1746         struct lpfc_nvmet_rcv_ctx *ctxp;
1747         uint32_t *payload;
1748         uint32_t size, oxid, sid, rc;
1749
1750         fc_hdr = (struct fc_frame_header *)(nvmebuf->hbuf.virt);
1751         oxid = be16_to_cpu(fc_hdr->fh_ox_id);
1752
1753         if (!phba->targetport) {
1754                 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
1755                                 "6154 LS Drop IO x%x\n", oxid);
1756                 oxid = 0;
1757                 size = 0;
1758                 sid = 0;
1759                 ctxp = NULL;
1760                 goto dropit;
1761         }
1762
1763         tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
1764         payload = (uint32_t *)(nvmebuf->dbuf.virt);
1765         size = bf_get(lpfc_rcqe_length,  &nvmebuf->cq_event.cqe.rcqe_cmpl);
1766         sid = sli4_sid_from_fc_hdr(fc_hdr);
1767
1768         ctxp = kzalloc(sizeof(struct lpfc_nvmet_rcv_ctx), GFP_ATOMIC);
1769         if (ctxp == NULL) {
1770                 atomic_inc(&tgtp->rcv_ls_req_drop);
1771                 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
1772                                 "6155 LS Drop IO x%x: Alloc\n",
1773                                 oxid);
1774 dropit:
1775                 lpfc_nvmeio_data(phba, "NVMET LS  DROP: "
1776                                  "xri x%x sz %d from %06x\n",
1777                                  oxid, size, sid);
1778                 lpfc_in_buf_free(phba, &nvmebuf->dbuf);
1779                 return;
1780         }
1781         ctxp->phba = phba;
1782         ctxp->size = size;
1783         ctxp->oxid = oxid;
1784         ctxp->sid = sid;
1785         ctxp->wqeq = NULL;
1786         ctxp->state = LPFC_NVMET_STE_LS_RCV;
1787         ctxp->entry_cnt = 1;
1788         ctxp->rqb_buffer = (void *)nvmebuf;
1789
1790         lpfc_nvmeio_data(phba, "NVMET LS   RCV: xri x%x sz %d from %06x\n",
1791                          oxid, size, sid);
1792         /*
1793          * The calling sequence should be:
1794          * nvmet_fc_rcv_ls_req -> lpfc_nvmet_xmt_ls_rsp/cmp ->_req->done
1795          * lpfc_nvmet_xmt_ls_rsp_cmp should free the allocated ctxp.
1796          */
1797         atomic_inc(&tgtp->rcv_ls_req_in);
1798         rc = nvmet_fc_rcv_ls_req(phba->targetport, &ctxp->ctx.ls_req,
1799                                  payload, size);
1800
1801         lpfc_printf_log(phba, KERN_INFO, LOG_NVME_DISC,
1802                         "6037 NVMET Unsol rcv: sz %d rc %d: %08x %08x %08x "
1803                         "%08x %08x %08x\n", size, rc,
1804                         *payload, *(payload+1), *(payload+2),
1805                         *(payload+3), *(payload+4), *(payload+5));
1806
1807         if (rc == 0) {
1808                 atomic_inc(&tgtp->rcv_ls_req_out);
1809                 return;
1810         }
1811
1812         lpfc_nvmeio_data(phba, "NVMET LS  DROP: xri x%x sz %d from %06x\n",
1813                          oxid, size, sid);
1814
1815         atomic_inc(&tgtp->rcv_ls_req_drop);
1816         lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
1817                         "6156 LS Drop IO x%x: nvmet_fc_rcv_ls_req %d\n",
1818                         ctxp->oxid, rc);
1819
1820         /* We assume a rcv'ed cmd ALWAYs fits into 1 buffer */
1821         lpfc_in_buf_free(phba, &nvmebuf->dbuf);
1822
1823         atomic_inc(&tgtp->xmt_ls_abort);
1824         lpfc_nvmet_unsol_ls_issue_abort(phba, ctxp, sid, oxid);
1825 #endif
1826 }
1827
1828 static struct lpfc_nvmet_ctxbuf *
1829 lpfc_nvmet_replenish_context(struct lpfc_hba *phba,
1830                              struct lpfc_nvmet_ctx_info *current_infop)
1831 {
1832 #if (IS_ENABLED(CONFIG_NVME_TARGET_FC))
1833         struct lpfc_nvmet_ctxbuf *ctx_buf = NULL;
1834         struct lpfc_nvmet_ctx_info *get_infop;
1835         int i;
1836
1837         /*
1838          * The current_infop for the MRQ a NVME command IU was received
1839          * on is empty. Our goal is to replenish this MRQs context
1840          * list from a another CPUs.
1841          *
1842          * First we need to pick a context list to start looking on.
1843          * nvmet_ctx_start_cpu has available context the last time
1844          * we needed to replenish this CPU where nvmet_ctx_next_cpu
1845          * is just the next sequential CPU for this MRQ.
1846          */
1847         if (current_infop->nvmet_ctx_start_cpu)
1848                 get_infop = current_infop->nvmet_ctx_start_cpu;
1849         else
1850                 get_infop = current_infop->nvmet_ctx_next_cpu;
1851
1852         for (i = 0; i < phba->sli4_hba.num_present_cpu; i++) {
1853                 if (get_infop == current_infop) {
1854                         get_infop = get_infop->nvmet_ctx_next_cpu;
1855                         continue;
1856                 }
1857                 spin_lock(&get_infop->nvmet_ctx_list_lock);
1858
1859                 /* Just take the entire context list, if there are any */
1860                 if (get_infop->nvmet_ctx_list_cnt) {
1861                         list_splice_init(&get_infop->nvmet_ctx_list,
1862                                     &current_infop->nvmet_ctx_list);
1863                         current_infop->nvmet_ctx_list_cnt =
1864                                 get_infop->nvmet_ctx_list_cnt - 1;
1865                         get_infop->nvmet_ctx_list_cnt = 0;
1866                         spin_unlock(&get_infop->nvmet_ctx_list_lock);
1867
1868                         current_infop->nvmet_ctx_start_cpu = get_infop;
1869                         list_remove_head(&current_infop->nvmet_ctx_list,
1870                                          ctx_buf, struct lpfc_nvmet_ctxbuf,
1871                                          list);
1872                         return ctx_buf;
1873                 }
1874
1875                 /* Otherwise, move on to the next CPU for this MRQ */
1876                 spin_unlock(&get_infop->nvmet_ctx_list_lock);
1877                 get_infop = get_infop->nvmet_ctx_next_cpu;
1878         }
1879
1880 #endif
1881         /* Nothing found, all contexts for the MRQ are in-flight */
1882         return NULL;
1883 }
1884
1885 /**
1886  * lpfc_nvmet_unsol_fcp_buffer - Process an unsolicited event data buffer
1887  * @phba: pointer to lpfc hba data structure.
1888  * @idx: relative index of MRQ vector
1889  * @nvmebuf: pointer to lpfc nvme command HBQ data structure.
1890  *
1891  * This routine is used for processing the WQE associated with a unsolicited
1892  * event. It first determines whether there is an existing ndlp that matches
1893  * the DID from the unsolicited WQE. If not, it will create a new one with
1894  * the DID from the unsolicited WQE. The ELS command from the unsolicited
1895  * WQE is then used to invoke the proper routine and to set up proper state
1896  * of the discovery state machine.
1897  **/
1898 static void
1899 lpfc_nvmet_unsol_fcp_buffer(struct lpfc_hba *phba,
1900                             uint32_t idx,
1901                             struct rqb_dmabuf *nvmebuf,
1902                             uint64_t isr_timestamp)
1903 {
1904         struct lpfc_nvmet_rcv_ctx *ctxp;
1905         struct lpfc_nvmet_tgtport *tgtp;
1906         struct fc_frame_header *fc_hdr;
1907         struct lpfc_nvmet_ctxbuf *ctx_buf;
1908         struct lpfc_nvmet_ctx_info *current_infop;
1909         uint32_t *payload;
1910         uint32_t size, oxid, sid, rc, qno;
1911         unsigned long iflag;
1912         int current_cpu;
1913 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1914         uint32_t id;
1915 #endif
1916
1917         if (!IS_ENABLED(CONFIG_NVME_TARGET_FC))
1918                 return;
1919
1920         ctx_buf = NULL;
1921         if (!nvmebuf || !phba->targetport) {
1922                 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
1923                                 "6157 NVMET FCP Drop IO\n");
1924                 oxid = 0;
1925                 size = 0;
1926                 sid = 0;
1927                 ctxp = NULL;
1928                 goto dropit;
1929         }
1930
1931         /*
1932          * Get a pointer to the context list for this MRQ based on
1933          * the CPU this MRQ IRQ is associated with. If the CPU association
1934          * changes from our initial assumption, the context list could
1935          * be empty, thus it would need to be replenished with the
1936          * context list from another CPU for this MRQ.
1937          */
1938         current_cpu = smp_processor_id();
1939         current_infop = lpfc_get_ctx_list(phba, current_cpu, idx);
1940         spin_lock_irqsave(&current_infop->nvmet_ctx_list_lock, iflag);
1941         if (current_infop->nvmet_ctx_list_cnt) {
1942                 list_remove_head(&current_infop->nvmet_ctx_list,
1943                                  ctx_buf, struct lpfc_nvmet_ctxbuf, list);
1944                 current_infop->nvmet_ctx_list_cnt--;
1945         } else {
1946                 ctx_buf = lpfc_nvmet_replenish_context(phba, current_infop);
1947         }
1948         spin_unlock_irqrestore(&current_infop->nvmet_ctx_list_lock, iflag);
1949
1950         fc_hdr = (struct fc_frame_header *)(nvmebuf->hbuf.virt);
1951         oxid = be16_to_cpu(fc_hdr->fh_ox_id);
1952         size = nvmebuf->bytes_recv;
1953
1954 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1955         if (phba->cpucheck_on & LPFC_CHECK_NVMET_RCV) {
1956                 id = smp_processor_id();
1957                 if (id < LPFC_CHECK_CPU_CNT)
1958                         phba->cpucheck_rcv_io[id]++;
1959         }
1960 #endif
1961
1962         lpfc_nvmeio_data(phba, "NVMET FCP  RCV: xri x%x sz %d CPU %02x\n",
1963                          oxid, size, smp_processor_id());
1964
1965         tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
1966
1967         if (!ctx_buf) {
1968                 /* Queue this NVME IO to process later */
1969                 spin_lock_irqsave(&phba->sli4_hba.nvmet_io_wait_lock, iflag);
1970                 list_add_tail(&nvmebuf->hbuf.list,
1971                               &phba->sli4_hba.lpfc_nvmet_io_wait_list);
1972                 phba->sli4_hba.nvmet_io_wait_cnt++;
1973                 phba->sli4_hba.nvmet_io_wait_total++;
1974                 spin_unlock_irqrestore(&phba->sli4_hba.nvmet_io_wait_lock,
1975                                        iflag);
1976
1977                 /* Post a brand new DMA buffer to RQ */
1978                 qno = nvmebuf->idx;
1979                 lpfc_post_rq_buffer(
1980                         phba, phba->sli4_hba.nvmet_mrq_hdr[qno],
1981                         phba->sli4_hba.nvmet_mrq_data[qno], 1, qno);
1982
1983                 atomic_inc(&tgtp->defer_ctx);
1984                 return;
1985         }
1986
1987         payload = (uint32_t *)(nvmebuf->dbuf.virt);
1988         sid = sli4_sid_from_fc_hdr(fc_hdr);
1989
1990         ctxp = (struct lpfc_nvmet_rcv_ctx *)ctx_buf->context;
1991         if (ctxp->state != LPFC_NVMET_STE_FREE) {
1992                 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
1993                                 "6414 NVMET Context corrupt %d %d oxid x%x\n",
1994                                 ctxp->state, ctxp->entry_cnt, ctxp->oxid);
1995         }
1996         ctxp->wqeq = NULL;
1997         ctxp->txrdy = NULL;
1998         ctxp->offset = 0;
1999         ctxp->phba = phba;
2000         ctxp->size = size;
2001         ctxp->oxid = oxid;
2002         ctxp->sid = sid;
2003         ctxp->idx = idx;
2004         ctxp->state = LPFC_NVMET_STE_RCV;
2005         ctxp->entry_cnt = 1;
2006         ctxp->flag = 0;
2007         ctxp->ctxbuf = ctx_buf;
2008         ctxp->rqb_buffer = (void *)nvmebuf;
2009         spin_lock_init(&ctxp->ctxlock);
2010
2011 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
2012         if (isr_timestamp) {
2013                 ctxp->ts_isr_cmd = isr_timestamp;
2014                 ctxp->ts_cmd_nvme = ktime_get_ns();
2015                 ctxp->ts_nvme_data = 0;
2016                 ctxp->ts_data_wqput = 0;
2017                 ctxp->ts_isr_data = 0;
2018                 ctxp->ts_data_nvme = 0;
2019                 ctxp->ts_nvme_status = 0;
2020                 ctxp->ts_status_wqput = 0;
2021                 ctxp->ts_isr_status = 0;
2022                 ctxp->ts_status_nvme = 0;
2023         } else {
2024                 ctxp->ts_cmd_nvme = 0;
2025         }
2026 #endif
2027
2028         atomic_inc(&tgtp->rcv_fcp_cmd_in);
2029         /*
2030          * The calling sequence should be:
2031          * nvmet_fc_rcv_fcp_req -> lpfc_nvmet_xmt_fcp_op/cmp -> req->done
2032          * lpfc_nvmet_xmt_fcp_op_cmp should free the allocated ctxp.
2033          * When we return from nvmet_fc_rcv_fcp_req, all relevant info in
2034          * the NVME command / FC header is stored, so we are free to repost
2035          * the buffer.
2036          */
2037         rc = nvmet_fc_rcv_fcp_req(phba->targetport, &ctxp->ctx.fcp_req,
2038                                   payload, size);
2039
2040         /* Process FCP command */
2041         if (rc == 0) {
2042                 ctxp->rqb_buffer = NULL;
2043                 atomic_inc(&tgtp->rcv_fcp_cmd_out);
2044                 lpfc_rq_buf_free(phba, &nvmebuf->hbuf); /* repost */
2045                 return;
2046         }
2047
2048         /* Processing of FCP command is deferred */
2049         if (rc == -EOVERFLOW) {
2050                 /*
2051                  * Post a brand new DMA buffer to RQ and defer
2052                  * freeing rcv buffer till .defer_rcv callback
2053                  */
2054                 qno = nvmebuf->idx;
2055                 lpfc_post_rq_buffer(
2056                         phba, phba->sli4_hba.nvmet_mrq_hdr[qno],
2057                         phba->sli4_hba.nvmet_mrq_data[qno], 1, qno);
2058
2059                 lpfc_nvmeio_data(phba,
2060                                  "NVMET RCV BUSY: xri x%x sz %d from %06x\n",
2061                                  oxid, size, sid);
2062                 atomic_inc(&tgtp->rcv_fcp_cmd_out);
2063                 atomic_inc(&tgtp->defer_fod);
2064                 return;
2065         }
2066         ctxp->rqb_buffer = nvmebuf;
2067
2068         atomic_inc(&tgtp->rcv_fcp_cmd_drop);
2069         lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
2070                         "6159 FCP Drop IO x%x: err x%x: x%x x%x x%x\n",
2071                         ctxp->oxid, rc,
2072                         atomic_read(&tgtp->rcv_fcp_cmd_in),
2073                         atomic_read(&tgtp->rcv_fcp_cmd_out),
2074                         atomic_read(&tgtp->xmt_fcp_release));
2075 dropit:
2076         lpfc_nvmeio_data(phba, "NVMET FCP DROP: xri x%x sz %d from %06x\n",
2077                          oxid, size, sid);
2078         if (oxid) {
2079                 lpfc_nvmet_defer_release(phba, ctxp);
2080                 lpfc_nvmet_unsol_fcp_issue_abort(phba, ctxp, sid, oxid);
2081                 lpfc_rq_buf_free(phba, &nvmebuf->hbuf); /* repost */
2082                 return;
2083         }
2084
2085         if (ctx_buf)
2086                 lpfc_nvmet_ctxbuf_post(phba, ctx_buf);
2087
2088         if (nvmebuf)
2089                 lpfc_rq_buf_free(phba, &nvmebuf->hbuf); /* repost */
2090 }
2091
2092 /**
2093  * lpfc_nvmet_unsol_ls_event - Process an unsolicited event from an nvme nport
2094  * @phba: pointer to lpfc hba data structure.
2095  * @pring: pointer to a SLI ring.
2096  * @nvmebuf: pointer to received nvme data structure.
2097  *
2098  * This routine is used to process an unsolicited event received from a SLI
2099  * (Service Level Interface) ring. The actual processing of the data buffer
2100  * associated with the unsolicited event is done by invoking the routine
2101  * lpfc_nvmet_unsol_ls_buffer() after properly set up the buffer from the
2102  * SLI RQ on which the unsolicited event was received.
2103  **/
2104 void
2105 lpfc_nvmet_unsol_ls_event(struct lpfc_hba *phba, struct lpfc_sli_ring *pring,
2106                           struct lpfc_iocbq *piocb)
2107 {
2108         struct lpfc_dmabuf *d_buf;
2109         struct hbq_dmabuf *nvmebuf;
2110
2111         d_buf = piocb->context2;
2112         nvmebuf = container_of(d_buf, struct hbq_dmabuf, dbuf);
2113
2114         if (phba->nvmet_support == 0) {
2115                 lpfc_in_buf_free(phba, &nvmebuf->dbuf);
2116                 return;
2117         }
2118         lpfc_nvmet_unsol_ls_buffer(phba, pring, nvmebuf);
2119 }
2120
2121 /**
2122  * lpfc_nvmet_unsol_fcp_event - Process an unsolicited event from an nvme nport
2123  * @phba: pointer to lpfc hba data structure.
2124  * @idx: relative index of MRQ vector
2125  * @nvmebuf: pointer to received nvme data structure.
2126  *
2127  * This routine is used to process an unsolicited event received from a SLI
2128  * (Service Level Interface) ring. The actual processing of the data buffer
2129  * associated with the unsolicited event is done by invoking the routine
2130  * lpfc_nvmet_unsol_fcp_buffer() after properly set up the buffer from the
2131  * SLI RQ on which the unsolicited event was received.
2132  **/
2133 void
2134 lpfc_nvmet_unsol_fcp_event(struct lpfc_hba *phba,
2135                            uint32_t idx,
2136                            struct rqb_dmabuf *nvmebuf,
2137                            uint64_t isr_timestamp)
2138 {
2139         if (phba->nvmet_support == 0) {
2140                 lpfc_rq_buf_free(phba, &nvmebuf->hbuf);
2141                 return;
2142         }
2143         lpfc_nvmet_unsol_fcp_buffer(phba, idx, nvmebuf,
2144                                     isr_timestamp);
2145 }
2146
2147 /**
2148  * lpfc_nvmet_prep_ls_wqe - Allocate and prepare a lpfc wqe data structure
2149  * @phba: pointer to a host N_Port data structure.
2150  * @ctxp: Context info for NVME LS Request
2151  * @rspbuf: DMA buffer of NVME command.
2152  * @rspsize: size of the NVME command.
2153  *
2154  * This routine is used for allocating a lpfc-WQE data structure from
2155  * the driver lpfc-WQE free-list and prepare the WQE with the parameters
2156  * passed into the routine for discovery state machine to issue an Extended
2157  * Link Service (NVME) commands. It is a generic lpfc-WQE allocation
2158  * and preparation routine that is used by all the discovery state machine
2159  * routines and the NVME command-specific fields will be later set up by
2160  * the individual discovery machine routines after calling this routine
2161  * allocating and preparing a generic WQE data structure. It fills in the
2162  * Buffer Descriptor Entries (BDEs), allocates buffers for both command
2163  * payload and response payload (if expected). The reference count on the
2164  * ndlp is incremented by 1 and the reference to the ndlp is put into
2165  * context1 of the WQE data structure for this WQE to hold the ndlp
2166  * reference for the command's callback function to access later.
2167  *
2168  * Return code
2169  *   Pointer to the newly allocated/prepared nvme wqe data structure
2170  *   NULL - when nvme wqe data structure allocation/preparation failed
2171  **/
2172 static struct lpfc_iocbq *
2173 lpfc_nvmet_prep_ls_wqe(struct lpfc_hba *phba,
2174                        struct lpfc_nvmet_rcv_ctx *ctxp,
2175                        dma_addr_t rspbuf, uint16_t rspsize)
2176 {
2177         struct lpfc_nodelist *ndlp;
2178         struct lpfc_iocbq *nvmewqe;
2179         union lpfc_wqe128 *wqe;
2180
2181         if (!lpfc_is_link_up(phba)) {
2182                 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_DISC,
2183                                 "6104 NVMET prep LS wqe: link err: "
2184                                 "NPORT x%x oxid:x%x ste %d\n",
2185                                 ctxp->sid, ctxp->oxid, ctxp->state);
2186                 return NULL;
2187         }
2188
2189         /* Allocate buffer for  command wqe */
2190         nvmewqe = lpfc_sli_get_iocbq(phba);
2191         if (nvmewqe == NULL) {
2192                 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_DISC,
2193                                 "6105 NVMET prep LS wqe: No WQE: "
2194                                 "NPORT x%x oxid x%x ste %d\n",
2195                                 ctxp->sid, ctxp->oxid, ctxp->state);
2196                 return NULL;
2197         }
2198
2199         ndlp = lpfc_findnode_did(phba->pport, ctxp->sid);
2200         if (!ndlp || !NLP_CHK_NODE_ACT(ndlp) ||
2201             ((ndlp->nlp_state != NLP_STE_UNMAPPED_NODE) &&
2202             (ndlp->nlp_state != NLP_STE_MAPPED_NODE))) {
2203                 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_DISC,
2204                                 "6106 NVMET prep LS wqe: No ndlp: "
2205                                 "NPORT x%x oxid x%x ste %d\n",
2206                                 ctxp->sid, ctxp->oxid, ctxp->state);
2207                 goto nvme_wqe_free_wqeq_exit;
2208         }
2209         ctxp->wqeq = nvmewqe;
2210
2211         /* prevent preparing wqe with NULL ndlp reference */
2212         nvmewqe->context1 = lpfc_nlp_get(ndlp);
2213         if (nvmewqe->context1 == NULL)
2214                 goto nvme_wqe_free_wqeq_exit;
2215         nvmewqe->context2 = ctxp;
2216
2217         wqe = &nvmewqe->wqe;
2218         memset(wqe, 0, sizeof(union lpfc_wqe));
2219
2220         /* Words 0 - 2 */
2221         wqe->xmit_sequence.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_64;
2222         wqe->xmit_sequence.bde.tus.f.bdeSize = rspsize;
2223         wqe->xmit_sequence.bde.addrLow = le32_to_cpu(putPaddrLow(rspbuf));
2224         wqe->xmit_sequence.bde.addrHigh = le32_to_cpu(putPaddrHigh(rspbuf));
2225
2226         /* Word 3 */
2227
2228         /* Word 4 */
2229
2230         /* Word 5 */
2231         bf_set(wqe_dfctl, &wqe->xmit_sequence.wge_ctl, 0);
2232         bf_set(wqe_ls, &wqe->xmit_sequence.wge_ctl, 1);
2233         bf_set(wqe_la, &wqe->xmit_sequence.wge_ctl, 0);
2234         bf_set(wqe_rctl, &wqe->xmit_sequence.wge_ctl, FC_RCTL_ELS4_REP);
2235         bf_set(wqe_type, &wqe->xmit_sequence.wge_ctl, FC_TYPE_NVME);
2236
2237         /* Word 6 */
2238         bf_set(wqe_ctxt_tag, &wqe->xmit_sequence.wqe_com,
2239                phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]);
2240         bf_set(wqe_xri_tag, &wqe->xmit_sequence.wqe_com, nvmewqe->sli4_xritag);
2241
2242         /* Word 7 */
2243         bf_set(wqe_cmnd, &wqe->xmit_sequence.wqe_com,
2244                CMD_XMIT_SEQUENCE64_WQE);
2245         bf_set(wqe_ct, &wqe->xmit_sequence.wqe_com, SLI4_CT_RPI);
2246         bf_set(wqe_class, &wqe->xmit_sequence.wqe_com, CLASS3);
2247         bf_set(wqe_pu, &wqe->xmit_sequence.wqe_com, 0);
2248
2249         /* Word 8 */
2250         wqe->xmit_sequence.wqe_com.abort_tag = nvmewqe->iotag;
2251
2252         /* Word 9 */
2253         bf_set(wqe_reqtag, &wqe->xmit_sequence.wqe_com, nvmewqe->iotag);
2254         /* Needs to be set by caller */
2255         bf_set(wqe_rcvoxid, &wqe->xmit_sequence.wqe_com, ctxp->oxid);
2256
2257         /* Word 10 */
2258         bf_set(wqe_dbde, &wqe->xmit_sequence.wqe_com, 1);
2259         bf_set(wqe_iod, &wqe->xmit_sequence.wqe_com, LPFC_WQE_IOD_WRITE);
2260         bf_set(wqe_lenloc, &wqe->xmit_sequence.wqe_com,
2261                LPFC_WQE_LENLOC_WORD12);
2262         bf_set(wqe_ebde_cnt, &wqe->xmit_sequence.wqe_com, 0);
2263
2264         /* Word 11 */
2265         bf_set(wqe_cqid, &wqe->xmit_sequence.wqe_com,
2266                LPFC_WQE_CQ_ID_DEFAULT);
2267         bf_set(wqe_cmd_type, &wqe->xmit_sequence.wqe_com,
2268                OTHER_COMMAND);
2269
2270         /* Word 12 */
2271         wqe->xmit_sequence.xmit_len = rspsize;
2272
2273         nvmewqe->retry = 1;
2274         nvmewqe->vport = phba->pport;
2275         nvmewqe->drvrTimeout = (phba->fc_ratov * 3) + LPFC_DRVR_TIMEOUT;
2276         nvmewqe->iocb_flag |= LPFC_IO_NVME_LS;
2277
2278         /* Xmit NVMET response to remote NPORT <did> */
2279         lpfc_printf_log(phba, KERN_INFO, LOG_NVME_DISC,
2280                         "6039 Xmit NVMET LS response to remote "
2281                         "NPORT x%x iotag:x%x oxid:x%x size:x%x\n",
2282                         ndlp->nlp_DID, nvmewqe->iotag, ctxp->oxid,
2283                         rspsize);
2284         return nvmewqe;
2285
2286 nvme_wqe_free_wqeq_exit:
2287         nvmewqe->context2 = NULL;
2288         nvmewqe->context3 = NULL;
2289         lpfc_sli_release_iocbq(phba, nvmewqe);
2290         return NULL;
2291 }
2292
2293
2294 static struct lpfc_iocbq *
2295 lpfc_nvmet_prep_fcp_wqe(struct lpfc_hba *phba,
2296                         struct lpfc_nvmet_rcv_ctx *ctxp)
2297 {
2298         struct nvmefc_tgt_fcp_req *rsp = &ctxp->ctx.fcp_req;
2299         struct lpfc_nvmet_tgtport *tgtp;
2300         struct sli4_sge *sgl;
2301         struct lpfc_nodelist *ndlp;
2302         struct lpfc_iocbq *nvmewqe;
2303         struct scatterlist *sgel;
2304         union lpfc_wqe128 *wqe;
2305         struct ulp_bde64 *bde;
2306         uint32_t *txrdy;
2307         dma_addr_t physaddr;
2308         int i, cnt;
2309         int do_pbde;
2310         int xc = 1;
2311
2312         if (!lpfc_is_link_up(phba)) {
2313                 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
2314                                 "6107 NVMET prep FCP wqe: link err:"
2315                                 "NPORT x%x oxid x%x ste %d\n",
2316                                 ctxp->sid, ctxp->oxid, ctxp->state);
2317                 return NULL;
2318         }
2319
2320         ndlp = lpfc_findnode_did(phba->pport, ctxp->sid);
2321         if (!ndlp || !NLP_CHK_NODE_ACT(ndlp) ||
2322             ((ndlp->nlp_state != NLP_STE_UNMAPPED_NODE) &&
2323              (ndlp->nlp_state != NLP_STE_MAPPED_NODE))) {
2324                 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
2325                                 "6108 NVMET prep FCP wqe: no ndlp: "
2326                                 "NPORT x%x oxid x%x ste %d\n",
2327                                 ctxp->sid, ctxp->oxid, ctxp->state);
2328                 return NULL;
2329         }
2330
2331         if (rsp->sg_cnt > lpfc_tgttemplate.max_sgl_segments) {
2332                 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
2333                                 "6109 NVMET prep FCP wqe: seg cnt err: "
2334                                 "NPORT x%x oxid x%x ste %d cnt %d\n",
2335                                 ctxp->sid, ctxp->oxid, ctxp->state,
2336                                 phba->cfg_nvme_seg_cnt);
2337                 return NULL;
2338         }
2339
2340         tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
2341         nvmewqe = ctxp->wqeq;
2342         if (nvmewqe == NULL) {
2343                 /* Allocate buffer for  command wqe */
2344                 nvmewqe = ctxp->ctxbuf->iocbq;
2345                 if (nvmewqe == NULL) {
2346                         lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
2347                                         "6110 NVMET prep FCP wqe: No "
2348                                         "WQE: NPORT x%x oxid x%x ste %d\n",
2349                                         ctxp->sid, ctxp->oxid, ctxp->state);
2350                         return NULL;
2351                 }
2352                 ctxp->wqeq = nvmewqe;
2353                 xc = 0; /* create new XRI */
2354                 nvmewqe->sli4_lxritag = NO_XRI;
2355                 nvmewqe->sli4_xritag = NO_XRI;
2356         }
2357
2358         /* Sanity check */
2359         if (((ctxp->state == LPFC_NVMET_STE_RCV) &&
2360             (ctxp->entry_cnt == 1)) ||
2361             (ctxp->state == LPFC_NVMET_STE_DATA)) {
2362                 wqe = &nvmewqe->wqe;
2363         } else {
2364                 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
2365                                 "6111 Wrong state NVMET FCP: %d  cnt %d\n",
2366                                 ctxp->state, ctxp->entry_cnt);
2367                 return NULL;
2368         }
2369
2370         sgl  = (struct sli4_sge *)ctxp->ctxbuf->sglq->sgl;
2371         switch (rsp->op) {
2372         case NVMET_FCOP_READDATA:
2373         case NVMET_FCOP_READDATA_RSP:
2374                 /* From the tsend template, initialize words 7 - 11 */
2375                 memcpy(&wqe->words[7],
2376                        &lpfc_tsend_cmd_template.words[7],
2377                        sizeof(uint32_t) * 5);
2378
2379                 /* Words 0 - 2 : The first sg segment */
2380                 sgel = &rsp->sg[0];
2381                 physaddr = sg_dma_address(sgel);
2382                 wqe->fcp_tsend.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_64;
2383                 wqe->fcp_tsend.bde.tus.f.bdeSize = sg_dma_len(sgel);
2384                 wqe->fcp_tsend.bde.addrLow = cpu_to_le32(putPaddrLow(physaddr));
2385                 wqe->fcp_tsend.bde.addrHigh =
2386                         cpu_to_le32(putPaddrHigh(physaddr));
2387
2388                 /* Word 3 */
2389                 wqe->fcp_tsend.payload_offset_len = 0;
2390
2391                 /* Word 4 */
2392                 wqe->fcp_tsend.relative_offset = ctxp->offset;
2393
2394                 /* Word 5 */
2395                 wqe->fcp_tsend.reserved = 0;
2396
2397                 /* Word 6 */
2398                 bf_set(wqe_ctxt_tag, &wqe->fcp_tsend.wqe_com,
2399                        phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]);
2400                 bf_set(wqe_xri_tag, &wqe->fcp_tsend.wqe_com,
2401                        nvmewqe->sli4_xritag);
2402
2403                 /* Word 7 - set ar later */
2404
2405                 /* Word 8 */
2406                 wqe->fcp_tsend.wqe_com.abort_tag = nvmewqe->iotag;
2407
2408                 /* Word 9 */
2409                 bf_set(wqe_reqtag, &wqe->fcp_tsend.wqe_com, nvmewqe->iotag);
2410                 bf_set(wqe_rcvoxid, &wqe->fcp_tsend.wqe_com, ctxp->oxid);
2411
2412                 /* Word 10 - set wqes later, in template xc=1 */
2413                 if (!xc)
2414                         bf_set(wqe_xc, &wqe->fcp_tsend.wqe_com, 0);
2415
2416                 /* Word 11 - set sup, irsp, irsplen later */
2417                 do_pbde = 0;
2418
2419                 /* Word 12 */
2420                 wqe->fcp_tsend.fcp_data_len = rsp->transfer_length;
2421
2422                 /* Setup 2 SKIP SGEs */
2423                 sgl->addr_hi = 0;
2424                 sgl->addr_lo = 0;
2425                 sgl->word2 = 0;
2426                 bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_SKIP);
2427                 sgl->word2 = cpu_to_le32(sgl->word2);
2428                 sgl->sge_len = 0;
2429                 sgl++;
2430                 sgl->addr_hi = 0;
2431                 sgl->addr_lo = 0;
2432                 sgl->word2 = 0;
2433                 bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_SKIP);
2434                 sgl->word2 = cpu_to_le32(sgl->word2);
2435                 sgl->sge_len = 0;
2436                 sgl++;
2437                 if (rsp->op == NVMET_FCOP_READDATA_RSP) {
2438                         atomic_inc(&tgtp->xmt_fcp_read_rsp);
2439
2440                         /* In template ar=1 wqes=0 sup=0 irsp=0 irsplen=0 */
2441
2442                         if (rsp->rsplen == LPFC_NVMET_SUCCESS_LEN) {
2443                                 if (ndlp->nlp_flag & NLP_SUPPRESS_RSP)
2444                                         bf_set(wqe_sup,
2445                                                &wqe->fcp_tsend.wqe_com, 1);
2446                         } else {
2447                                 bf_set(wqe_wqes, &wqe->fcp_tsend.wqe_com, 1);
2448                                 bf_set(wqe_irsp, &wqe->fcp_tsend.wqe_com, 1);
2449                                 bf_set(wqe_irsplen, &wqe->fcp_tsend.wqe_com,
2450                                        ((rsp->rsplen >> 2) - 1));
2451                                 memcpy(&wqe->words[16], rsp->rspaddr,
2452                                        rsp->rsplen);
2453                         }
2454                 } else {
2455                         atomic_inc(&tgtp->xmt_fcp_read);
2456
2457                         /* In template ar=1 wqes=0 sup=0 irsp=0 irsplen=0 */
2458                         bf_set(wqe_ar, &wqe->fcp_tsend.wqe_com, 0);
2459                 }
2460                 break;
2461
2462         case NVMET_FCOP_WRITEDATA:
2463                 /* From the treceive template, initialize words 3 - 11 */
2464                 memcpy(&wqe->words[3],
2465                        &lpfc_treceive_cmd_template.words[3],
2466                        sizeof(uint32_t) * 9);
2467
2468                 /* Words 0 - 2 : The first sg segment */
2469                 txrdy = dma_pool_alloc(phba->txrdy_payload_pool,
2470                                        GFP_KERNEL, &physaddr);
2471                 if (!txrdy) {
2472                         lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
2473                                         "6041 Bad txrdy buffer: oxid x%x\n",
2474                                         ctxp->oxid);
2475                         return NULL;
2476                 }
2477                 ctxp->txrdy = txrdy;
2478                 ctxp->txrdy_phys = physaddr;
2479                 wqe->fcp_treceive.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_64;
2480                 wqe->fcp_treceive.bde.tus.f.bdeSize = TXRDY_PAYLOAD_LEN;
2481                 wqe->fcp_treceive.bde.addrLow =
2482                         cpu_to_le32(putPaddrLow(physaddr));
2483                 wqe->fcp_treceive.bde.addrHigh =
2484                         cpu_to_le32(putPaddrHigh(physaddr));
2485
2486                 /* Word 4 */
2487                 wqe->fcp_treceive.relative_offset = ctxp->offset;
2488
2489                 /* Word 6 */
2490                 bf_set(wqe_ctxt_tag, &wqe->fcp_treceive.wqe_com,
2491                        phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]);
2492                 bf_set(wqe_xri_tag, &wqe->fcp_treceive.wqe_com,
2493                        nvmewqe->sli4_xritag);
2494
2495                 /* Word 7 */
2496
2497                 /* Word 8 */
2498                 wqe->fcp_treceive.wqe_com.abort_tag = nvmewqe->iotag;
2499
2500                 /* Word 9 */
2501                 bf_set(wqe_reqtag, &wqe->fcp_treceive.wqe_com, nvmewqe->iotag);
2502                 bf_set(wqe_rcvoxid, &wqe->fcp_treceive.wqe_com, ctxp->oxid);
2503
2504                 /* Word 10 - in template xc=1 */
2505                 if (!xc)
2506                         bf_set(wqe_xc, &wqe->fcp_treceive.wqe_com, 0);
2507
2508                 /* Word 11 - set pbde later */
2509                 if (phba->cfg_enable_pbde) {
2510                         do_pbde = 1;
2511                 } else {
2512                         bf_set(wqe_pbde, &wqe->fcp_treceive.wqe_com, 0);
2513                         do_pbde = 0;
2514                 }
2515
2516                 /* Word 12 */
2517                 wqe->fcp_tsend.fcp_data_len = rsp->transfer_length;
2518
2519                 /* Setup 1 TXRDY and 1 SKIP SGE */
2520                 txrdy[0] = 0;
2521                 txrdy[1] = cpu_to_be32(rsp->transfer_length);
2522                 txrdy[2] = 0;
2523
2524                 sgl->addr_hi = putPaddrHigh(physaddr);
2525                 sgl->addr_lo = putPaddrLow(physaddr);
2526                 sgl->word2 = 0;
2527                 bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_DATA);
2528                 sgl->word2 = cpu_to_le32(sgl->word2);
2529                 sgl->sge_len = cpu_to_le32(TXRDY_PAYLOAD_LEN);
2530                 sgl++;
2531                 sgl->addr_hi = 0;
2532                 sgl->addr_lo = 0;
2533                 sgl->word2 = 0;
2534                 bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_SKIP);
2535                 sgl->word2 = cpu_to_le32(sgl->word2);
2536                 sgl->sge_len = 0;
2537                 sgl++;
2538                 atomic_inc(&tgtp->xmt_fcp_write);
2539                 break;
2540
2541         case NVMET_FCOP_RSP:
2542                 /* From the treceive template, initialize words 4 - 11 */
2543                 memcpy(&wqe->words[4],
2544                        &lpfc_trsp_cmd_template.words[4],
2545                        sizeof(uint32_t) * 8);
2546
2547                 /* Words 0 - 2 */
2548                 physaddr = rsp->rspdma;
2549                 wqe->fcp_trsp.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_64;
2550                 wqe->fcp_trsp.bde.tus.f.bdeSize = rsp->rsplen;
2551                 wqe->fcp_trsp.bde.addrLow =
2552                         cpu_to_le32(putPaddrLow(physaddr));
2553                 wqe->fcp_trsp.bde.addrHigh =
2554                         cpu_to_le32(putPaddrHigh(physaddr));
2555
2556                 /* Word 3 */
2557                 wqe->fcp_trsp.response_len = rsp->rsplen;
2558
2559                 /* Word 6 */
2560                 bf_set(wqe_ctxt_tag, &wqe->fcp_trsp.wqe_com,
2561                        phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]);
2562                 bf_set(wqe_xri_tag, &wqe->fcp_trsp.wqe_com,
2563                        nvmewqe->sli4_xritag);
2564
2565                 /* Word 7 */
2566
2567                 /* Word 8 */
2568                 wqe->fcp_trsp.wqe_com.abort_tag = nvmewqe->iotag;
2569
2570                 /* Word 9 */
2571                 bf_set(wqe_reqtag, &wqe->fcp_trsp.wqe_com, nvmewqe->iotag);
2572                 bf_set(wqe_rcvoxid, &wqe->fcp_trsp.wqe_com, ctxp->oxid);
2573
2574                 /* Word 10 */
2575                 if (xc)
2576                         bf_set(wqe_xc, &wqe->fcp_trsp.wqe_com, 1);
2577
2578                 /* Word 11 */
2579                 /* In template wqes=0 irsp=0 irsplen=0 - good response */
2580                 if (rsp->rsplen != LPFC_NVMET_SUCCESS_LEN) {
2581                         /* Bad response - embed it */
2582                         bf_set(wqe_wqes, &wqe->fcp_trsp.wqe_com, 1);
2583                         bf_set(wqe_irsp, &wqe->fcp_trsp.wqe_com, 1);
2584                         bf_set(wqe_irsplen, &wqe->fcp_trsp.wqe_com,
2585                                ((rsp->rsplen >> 2) - 1));
2586                         memcpy(&wqe->words[16], rsp->rspaddr, rsp->rsplen);
2587                 }
2588                 do_pbde = 0;
2589
2590                 /* Word 12 */
2591                 wqe->fcp_trsp.rsvd_12_15[0] = 0;
2592
2593                 /* Use rspbuf, NOT sg list */
2594                 rsp->sg_cnt = 0;
2595                 sgl->word2 = 0;
2596                 atomic_inc(&tgtp->xmt_fcp_rsp);
2597                 break;
2598
2599         default:
2600                 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_IOERR,
2601                                 "6064 Unknown Rsp Op %d\n",
2602                                 rsp->op);
2603                 return NULL;
2604         }
2605
2606         nvmewqe->retry = 1;
2607         nvmewqe->vport = phba->pport;
2608         nvmewqe->drvrTimeout = (phba->fc_ratov * 3) + LPFC_DRVR_TIMEOUT;
2609         nvmewqe->context1 = ndlp;
2610
2611         for (i = 0; i < rsp->sg_cnt; i++) {
2612                 sgel = &rsp->sg[i];
2613                 physaddr = sg_dma_address(sgel);
2614                 cnt = sg_dma_len(sgel);
2615                 sgl->addr_hi = putPaddrHigh(physaddr);
2616                 sgl->addr_lo = putPaddrLow(physaddr);
2617                 sgl->word2 = 0;
2618                 bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_DATA);
2619                 bf_set(lpfc_sli4_sge_offset, sgl, ctxp->offset);
2620                 if ((i+1) == rsp->sg_cnt)
2621                         bf_set(lpfc_sli4_sge_last, sgl, 1);
2622                 sgl->word2 = cpu_to_le32(sgl->word2);
2623                 sgl->sge_len = cpu_to_le32(cnt);
2624                 if (i == 0) {
2625                         bde = (struct ulp_bde64 *)&wqe->words[13];
2626                         if (do_pbde) {
2627                                 /* Words 13-15  (PBDE) */
2628                                 bde->addrLow = sgl->addr_lo;
2629                                 bde->addrHigh = sgl->addr_hi;
2630                                 bde->tus.f.bdeSize =
2631                                         le32_to_cpu(sgl->sge_len);
2632                                 bde->tus.f.bdeFlags = BUFF_TYPE_BDE_64;
2633                                 bde->tus.w = cpu_to_le32(bde->tus.w);
2634                         } else {
2635                                 memset(bde, 0, sizeof(struct ulp_bde64));
2636                         }
2637                 }
2638                 sgl++;
2639                 ctxp->offset += cnt;
2640         }
2641         ctxp->state = LPFC_NVMET_STE_DATA;
2642         ctxp->entry_cnt++;
2643         return nvmewqe;
2644 }
2645
2646 /**
2647  * lpfc_nvmet_sol_fcp_abort_cmp - Completion handler for ABTS
2648  * @phba: Pointer to HBA context object.
2649  * @cmdwqe: Pointer to driver command WQE object.
2650  * @wcqe: Pointer to driver response CQE object.
2651  *
2652  * The function is called from SLI ring event handler with no
2653  * lock held. This function is the completion handler for NVME ABTS for FCP cmds
2654  * The function frees memory resources used for the NVME commands.
2655  **/
2656 static void
2657 lpfc_nvmet_sol_fcp_abort_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe,
2658                              struct lpfc_wcqe_complete *wcqe)
2659 {
2660         struct lpfc_nvmet_rcv_ctx *ctxp;
2661         struct lpfc_nvmet_tgtport *tgtp;
2662         uint32_t status, result;
2663         unsigned long flags;
2664         bool released = false;
2665
2666         ctxp = cmdwqe->context2;
2667         status = bf_get(lpfc_wcqe_c_status, wcqe);
2668         result = wcqe->parameter;
2669
2670         tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
2671         if (ctxp->flag & LPFC_NVMET_ABORT_OP)
2672                 atomic_inc(&tgtp->xmt_fcp_abort_cmpl);
2673
2674         ctxp->state = LPFC_NVMET_STE_DONE;
2675
2676         /* Check if we already received a free context call
2677          * and we have completed processing an abort situation.
2678          */
2679         spin_lock_irqsave(&ctxp->ctxlock, flags);
2680         if ((ctxp->flag & LPFC_NVMET_CTX_RLS) &&
2681             !(ctxp->flag & LPFC_NVMET_XBUSY)) {
2682                 list_del(&ctxp->list);
2683                 released = true;
2684         }
2685         ctxp->flag &= ~LPFC_NVMET_ABORT_OP;
2686         spin_unlock_irqrestore(&ctxp->ctxlock, flags);
2687         atomic_inc(&tgtp->xmt_abort_rsp);
2688
2689         lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
2690                         "6165 ABORT cmpl: xri x%x flg x%x (%d) "
2691                         "WCQE: %08x %08x %08x %08x\n",
2692                         ctxp->oxid, ctxp->flag, released,
2693                         wcqe->word0, wcqe->total_data_placed,
2694                         result, wcqe->word3);
2695
2696         cmdwqe->context2 = NULL;
2697         cmdwqe->context3 = NULL;
2698         /*
2699          * if transport has released ctx, then can reuse it. Otherwise,
2700          * will be recycled by transport release call.
2701          */
2702         if (released)
2703                 lpfc_nvmet_ctxbuf_post(phba, ctxp->ctxbuf);
2704
2705         /* This is the iocbq for the abort, not the command */
2706         lpfc_sli_release_iocbq(phba, cmdwqe);
2707
2708         /* Since iaab/iaar are NOT set, there is no work left.
2709          * For LPFC_NVMET_XBUSY, lpfc_sli4_nvmet_xri_aborted
2710          * should have been called already.
2711          */
2712 }
2713
2714 /**
2715  * lpfc_nvmet_unsol_fcp_abort_cmp - Completion handler for ABTS
2716  * @phba: Pointer to HBA context object.
2717  * @cmdwqe: Pointer to driver command WQE object.
2718  * @wcqe: Pointer to driver response CQE object.
2719  *
2720  * The function is called from SLI ring event handler with no
2721  * lock held. This function is the completion handler for NVME ABTS for FCP cmds
2722  * The function frees memory resources used for the NVME commands.
2723  **/
2724 static void
2725 lpfc_nvmet_unsol_fcp_abort_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe,
2726                                struct lpfc_wcqe_complete *wcqe)
2727 {
2728         struct lpfc_nvmet_rcv_ctx *ctxp;
2729         struct lpfc_nvmet_tgtport *tgtp;
2730         unsigned long flags;
2731         uint32_t status, result;
2732         bool released = false;
2733
2734         ctxp = cmdwqe->context2;
2735         status = bf_get(lpfc_wcqe_c_status, wcqe);
2736         result = wcqe->parameter;
2737
2738         if (!ctxp) {
2739                 /* if context is clear, related io alrady complete */
2740                 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
2741                                 "6070 ABTS cmpl: WCQE: %08x %08x %08x %08x\n",
2742                                 wcqe->word0, wcqe->total_data_placed,
2743                                 result, wcqe->word3);
2744                 return;
2745         }
2746
2747         tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
2748         if (ctxp->flag & LPFC_NVMET_ABORT_OP)
2749                 atomic_inc(&tgtp->xmt_fcp_abort_cmpl);
2750
2751         /* Sanity check */
2752         if (ctxp->state != LPFC_NVMET_STE_ABORT) {
2753                 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_ABTS,
2754                                 "6112 ABTS Wrong state:%d oxid x%x\n",
2755                                 ctxp->state, ctxp->oxid);
2756         }
2757
2758         /* Check if we already received a free context call
2759          * and we have completed processing an abort situation.
2760          */
2761         ctxp->state = LPFC_NVMET_STE_DONE;
2762         spin_lock_irqsave(&ctxp->ctxlock, flags);
2763         if ((ctxp->flag & LPFC_NVMET_CTX_RLS) &&
2764             !(ctxp->flag & LPFC_NVMET_XBUSY)) {
2765                 list_del(&ctxp->list);
2766                 released = true;
2767         }
2768         ctxp->flag &= ~LPFC_NVMET_ABORT_OP;
2769         spin_unlock_irqrestore(&ctxp->ctxlock, flags);
2770         atomic_inc(&tgtp->xmt_abort_rsp);
2771
2772         lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
2773                         "6316 ABTS cmpl xri x%x flg x%x (%x) "
2774                         "WCQE: %08x %08x %08x %08x\n",
2775                         ctxp->oxid, ctxp->flag, released,
2776                         wcqe->word0, wcqe->total_data_placed,
2777                         result, wcqe->word3);
2778
2779         cmdwqe->context2 = NULL;
2780         cmdwqe->context3 = NULL;
2781         /*
2782          * if transport has released ctx, then can reuse it. Otherwise,
2783          * will be recycled by transport release call.
2784          */
2785         if (released)
2786                 lpfc_nvmet_ctxbuf_post(phba, ctxp->ctxbuf);
2787
2788         /* Since iaab/iaar are NOT set, there is no work left.
2789          * For LPFC_NVMET_XBUSY, lpfc_sli4_nvmet_xri_aborted
2790          * should have been called already.
2791          */
2792 }
2793
2794 /**
2795  * lpfc_nvmet_xmt_ls_abort_cmp - Completion handler for ABTS
2796  * @phba: Pointer to HBA context object.
2797  * @cmdwqe: Pointer to driver command WQE object.
2798  * @wcqe: Pointer to driver response CQE object.
2799  *
2800  * The function is called from SLI ring event handler with no
2801  * lock held. This function is the completion handler for NVME ABTS for LS cmds
2802  * The function frees memory resources used for the NVME commands.
2803  **/
2804 static void
2805 lpfc_nvmet_xmt_ls_abort_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe,
2806                             struct lpfc_wcqe_complete *wcqe)
2807 {
2808         struct lpfc_nvmet_rcv_ctx *ctxp;
2809         struct lpfc_nvmet_tgtport *tgtp;
2810         uint32_t status, result;
2811
2812         ctxp = cmdwqe->context2;
2813         status = bf_get(lpfc_wcqe_c_status, wcqe);
2814         result = wcqe->parameter;
2815
2816         tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
2817         atomic_inc(&tgtp->xmt_ls_abort_cmpl);
2818
2819         lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
2820                         "6083 Abort cmpl: ctx %p WCQE:%08x %08x %08x %08x\n",
2821                         ctxp, wcqe->word0, wcqe->total_data_placed,
2822                         result, wcqe->word3);
2823
2824         if (!ctxp) {
2825                 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_ABTS,
2826                                 "6415 NVMET LS Abort No ctx: WCQE: "
2827                                  "%08x %08x %08x %08x\n",
2828                                 wcqe->word0, wcqe->total_data_placed,
2829                                 result, wcqe->word3);
2830
2831                 lpfc_sli_release_iocbq(phba, cmdwqe);
2832                 return;
2833         }
2834
2835         if (ctxp->state != LPFC_NVMET_STE_LS_ABORT) {
2836                 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
2837                                 "6416 NVMET LS abort cmpl state mismatch: "
2838                                 "oxid x%x: %d %d\n",
2839                                 ctxp->oxid, ctxp->state, ctxp->entry_cnt);
2840         }
2841
2842         cmdwqe->context2 = NULL;
2843         cmdwqe->context3 = NULL;
2844         lpfc_sli_release_iocbq(phba, cmdwqe);
2845         kfree(ctxp);
2846 }
2847
2848 static int
2849 lpfc_nvmet_unsol_issue_abort(struct lpfc_hba *phba,
2850                              struct lpfc_nvmet_rcv_ctx *ctxp,
2851                              uint32_t sid, uint16_t xri)
2852 {
2853         struct lpfc_nvmet_tgtport *tgtp;
2854         struct lpfc_iocbq *abts_wqeq;
2855         union lpfc_wqe128 *wqe_abts;
2856         struct lpfc_nodelist *ndlp;
2857
2858         lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
2859                         "6067 ABTS: sid %x xri x%x/x%x\n",
2860                         sid, xri, ctxp->wqeq->sli4_xritag);
2861
2862         tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
2863
2864         ndlp = lpfc_findnode_did(phba->pport, sid);
2865         if (!ndlp || !NLP_CHK_NODE_ACT(ndlp) ||
2866             ((ndlp->nlp_state != NLP_STE_UNMAPPED_NODE) &&
2867             (ndlp->nlp_state != NLP_STE_MAPPED_NODE))) {
2868                 atomic_inc(&tgtp->xmt_abort_rsp_error);
2869                 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_ABTS,
2870                                 "6134 Drop ABTS - wrong NDLP state x%x.\n",
2871                                 (ndlp) ? ndlp->nlp_state : NLP_STE_MAX_STATE);
2872
2873                 /* No failure to an ABTS request. */
2874                 return 0;
2875         }
2876
2877         abts_wqeq = ctxp->wqeq;
2878         wqe_abts = &abts_wqeq->wqe;
2879
2880         /*
2881          * Since we zero the whole WQE, we need to ensure we set the WQE fields
2882          * that were initialized in lpfc_sli4_nvmet_alloc.
2883          */
2884         memset(wqe_abts, 0, sizeof(union lpfc_wqe));
2885
2886         /* Word 5 */
2887         bf_set(wqe_dfctl, &wqe_abts->xmit_sequence.wge_ctl, 0);
2888         bf_set(wqe_ls, &wqe_abts->xmit_sequence.wge_ctl, 1);
2889         bf_set(wqe_la, &wqe_abts->xmit_sequence.wge_ctl, 0);
2890         bf_set(wqe_rctl, &wqe_abts->xmit_sequence.wge_ctl, FC_RCTL_BA_ABTS);
2891         bf_set(wqe_type, &wqe_abts->xmit_sequence.wge_ctl, FC_TYPE_BLS);
2892
2893         /* Word 6 */
2894         bf_set(wqe_ctxt_tag, &wqe_abts->xmit_sequence.wqe_com,
2895                phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]);
2896         bf_set(wqe_xri_tag, &wqe_abts->xmit_sequence.wqe_com,
2897                abts_wqeq->sli4_xritag);
2898
2899         /* Word 7 */
2900         bf_set(wqe_cmnd, &wqe_abts->xmit_sequence.wqe_com,
2901                CMD_XMIT_SEQUENCE64_WQE);
2902         bf_set(wqe_ct, &wqe_abts->xmit_sequence.wqe_com, SLI4_CT_RPI);
2903         bf_set(wqe_class, &wqe_abts->xmit_sequence.wqe_com, CLASS3);
2904         bf_set(wqe_pu, &wqe_abts->xmit_sequence.wqe_com, 0);
2905
2906         /* Word 8 */
2907         wqe_abts->xmit_sequence.wqe_com.abort_tag = abts_wqeq->iotag;
2908
2909         /* Word 9 */
2910         bf_set(wqe_reqtag, &wqe_abts->xmit_sequence.wqe_com, abts_wqeq->iotag);
2911         /* Needs to be set by caller */
2912         bf_set(wqe_rcvoxid, &wqe_abts->xmit_sequence.wqe_com, xri);
2913
2914         /* Word 10 */
2915         bf_set(wqe_iod, &wqe_abts->xmit_sequence.wqe_com, LPFC_WQE_IOD_WRITE);
2916         bf_set(wqe_lenloc, &wqe_abts->xmit_sequence.wqe_com,
2917                LPFC_WQE_LENLOC_WORD12);
2918         bf_set(wqe_ebde_cnt, &wqe_abts->xmit_sequence.wqe_com, 0);
2919         bf_set(wqe_qosd, &wqe_abts->xmit_sequence.wqe_com, 0);
2920
2921         /* Word 11 */
2922         bf_set(wqe_cqid, &wqe_abts->xmit_sequence.wqe_com,
2923                LPFC_WQE_CQ_ID_DEFAULT);
2924         bf_set(wqe_cmd_type, &wqe_abts->xmit_sequence.wqe_com,
2925                OTHER_COMMAND);
2926
2927         abts_wqeq->vport = phba->pport;
2928         abts_wqeq->context1 = ndlp;
2929         abts_wqeq->context2 = ctxp;
2930         abts_wqeq->context3 = NULL;
2931         abts_wqeq->rsvd2 = 0;
2932         /* hba_wqidx should already be setup from command we are aborting */
2933         abts_wqeq->iocb.ulpCommand = CMD_XMIT_SEQUENCE64_CR;
2934         abts_wqeq->iocb.ulpLe = 1;
2935
2936         lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
2937                         "6069 Issue ABTS to xri x%x reqtag x%x\n",
2938                         xri, abts_wqeq->iotag);
2939         return 1;
2940 }
2941
2942 static int
2943 lpfc_nvmet_sol_fcp_issue_abort(struct lpfc_hba *phba,
2944                                struct lpfc_nvmet_rcv_ctx *ctxp,
2945                                uint32_t sid, uint16_t xri)
2946 {
2947         struct lpfc_nvmet_tgtport *tgtp;
2948         struct lpfc_iocbq *abts_wqeq;
2949         union lpfc_wqe128 *abts_wqe;
2950         struct lpfc_nodelist *ndlp;
2951         unsigned long flags;
2952         int rc;
2953
2954         tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
2955         if (!ctxp->wqeq) {
2956                 ctxp->wqeq = ctxp->ctxbuf->iocbq;
2957                 ctxp->wqeq->hba_wqidx = 0;
2958         }
2959
2960         ndlp = lpfc_findnode_did(phba->pport, sid);
2961         if (!ndlp || !NLP_CHK_NODE_ACT(ndlp) ||
2962             ((ndlp->nlp_state != NLP_STE_UNMAPPED_NODE) &&
2963             (ndlp->nlp_state != NLP_STE_MAPPED_NODE))) {
2964                 atomic_inc(&tgtp->xmt_abort_rsp_error);
2965                 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_ABTS,
2966                                 "6160 Drop ABORT - wrong NDLP state x%x.\n",
2967                                 (ndlp) ? ndlp->nlp_state : NLP_STE_MAX_STATE);
2968
2969                 /* No failure to an ABTS request. */
2970                 ctxp->flag &= ~LPFC_NVMET_ABORT_OP;
2971                 return 0;
2972         }
2973
2974         /* Issue ABTS for this WQE based on iotag */
2975         ctxp->abort_wqeq = lpfc_sli_get_iocbq(phba);
2976         if (!ctxp->abort_wqeq) {
2977                 atomic_inc(&tgtp->xmt_abort_rsp_error);
2978                 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_ABTS,
2979                                 "6161 ABORT failed: No wqeqs: "
2980                                 "xri: x%x\n", ctxp->oxid);
2981                 /* No failure to an ABTS request. */
2982                 ctxp->flag &= ~LPFC_NVMET_ABORT_OP;
2983                 return 0;
2984         }
2985         abts_wqeq = ctxp->abort_wqeq;
2986         abts_wqe = &abts_wqeq->wqe;
2987         ctxp->state = LPFC_NVMET_STE_ABORT;
2988
2989         /* Announce entry to new IO submit field. */
2990         lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
2991                         "6162 ABORT Request to rport DID x%06x "
2992                         "for xri x%x x%x\n",
2993                         ctxp->sid, ctxp->oxid, ctxp->wqeq->sli4_xritag);
2994
2995         /* If the hba is getting reset, this flag is set.  It is
2996          * cleared when the reset is complete and rings reestablished.
2997          */
2998         spin_lock_irqsave(&phba->hbalock, flags);
2999         /* driver queued commands are in process of being flushed */
3000         if (phba->hba_flag & HBA_NVME_IOQ_FLUSH) {
3001                 spin_unlock_irqrestore(&phba->hbalock, flags);
3002                 atomic_inc(&tgtp->xmt_abort_rsp_error);
3003                 lpfc_printf_log(phba, KERN_ERR, LOG_NVME,
3004                                 "6163 Driver in reset cleanup - flushing "
3005                                 "NVME Req now. hba_flag x%x oxid x%x\n",
3006                                 phba->hba_flag, ctxp->oxid);
3007                 lpfc_sli_release_iocbq(phba, abts_wqeq);
3008                 ctxp->flag &= ~LPFC_NVMET_ABORT_OP;
3009                 return 0;
3010         }
3011
3012         /* Outstanding abort is in progress */
3013         if (abts_wqeq->iocb_flag & LPFC_DRIVER_ABORTED) {
3014                 spin_unlock_irqrestore(&phba->hbalock, flags);
3015                 atomic_inc(&tgtp->xmt_abort_rsp_error);
3016                 lpfc_printf_log(phba, KERN_ERR, LOG_NVME,
3017                                 "6164 Outstanding NVME I/O Abort Request "
3018                                 "still pending on oxid x%x\n",
3019                                 ctxp->oxid);
3020                 lpfc_sli_release_iocbq(phba, abts_wqeq);
3021                 ctxp->flag &= ~LPFC_NVMET_ABORT_OP;
3022                 return 0;
3023         }
3024
3025         /* Ready - mark outstanding as aborted by driver. */
3026         abts_wqeq->iocb_flag |= LPFC_DRIVER_ABORTED;
3027
3028         /* WQEs are reused.  Clear stale data and set key fields to
3029          * zero like ia, iaab, iaar, xri_tag, and ctxt_tag.
3030          */
3031         memset(abts_wqe, 0, sizeof(union lpfc_wqe));
3032
3033         /* word 3 */
3034         bf_set(abort_cmd_criteria, &abts_wqe->abort_cmd, T_XRI_TAG);
3035
3036         /* word 7 */
3037         bf_set(wqe_ct, &abts_wqe->abort_cmd.wqe_com, 0);
3038         bf_set(wqe_cmnd, &abts_wqe->abort_cmd.wqe_com, CMD_ABORT_XRI_CX);
3039
3040         /* word 8 - tell the FW to abort the IO associated with this
3041          * outstanding exchange ID.
3042          */
3043         abts_wqe->abort_cmd.wqe_com.abort_tag = ctxp->wqeq->sli4_xritag;
3044
3045         /* word 9 - this is the iotag for the abts_wqe completion. */
3046         bf_set(wqe_reqtag, &abts_wqe->abort_cmd.wqe_com,
3047                abts_wqeq->iotag);
3048
3049         /* word 10 */
3050         bf_set(wqe_qosd, &abts_wqe->abort_cmd.wqe_com, 1);
3051         bf_set(wqe_lenloc, &abts_wqe->abort_cmd.wqe_com, LPFC_WQE_LENLOC_NONE);
3052
3053         /* word 11 */
3054         bf_set(wqe_cmd_type, &abts_wqe->abort_cmd.wqe_com, OTHER_COMMAND);
3055         bf_set(wqe_wqec, &abts_wqe->abort_cmd.wqe_com, 1);
3056         bf_set(wqe_cqid, &abts_wqe->abort_cmd.wqe_com, LPFC_WQE_CQ_ID_DEFAULT);
3057
3058         /* ABTS WQE must go to the same WQ as the WQE to be aborted */
3059         abts_wqeq->hba_wqidx = ctxp->wqeq->hba_wqidx;
3060         abts_wqeq->wqe_cmpl = lpfc_nvmet_sol_fcp_abort_cmp;
3061         abts_wqeq->iocb_cmpl = 0;
3062         abts_wqeq->iocb_flag |= LPFC_IO_NVME;
3063         abts_wqeq->context2 = ctxp;
3064         abts_wqeq->vport = phba->pport;
3065         rc = lpfc_sli4_issue_wqe(phba, LPFC_FCP_RING, abts_wqeq);
3066         spin_unlock_irqrestore(&phba->hbalock, flags);
3067         if (rc == WQE_SUCCESS) {
3068                 atomic_inc(&tgtp->xmt_abort_sol);
3069                 return 0;
3070         }
3071
3072         atomic_inc(&tgtp->xmt_abort_rsp_error);
3073         ctxp->flag &= ~LPFC_NVMET_ABORT_OP;
3074         lpfc_sli_release_iocbq(phba, abts_wqeq);
3075         lpfc_printf_log(phba, KERN_ERR, LOG_NVME_ABTS,
3076                         "6166 Failed ABORT issue_wqe with status x%x "
3077                         "for oxid x%x.\n",
3078                         rc, ctxp->oxid);
3079         return 1;
3080 }
3081
3082
3083 static int
3084 lpfc_nvmet_unsol_fcp_issue_abort(struct lpfc_hba *phba,
3085                                  struct lpfc_nvmet_rcv_ctx *ctxp,
3086                                  uint32_t sid, uint16_t xri)
3087 {
3088         struct lpfc_nvmet_tgtport *tgtp;
3089         struct lpfc_iocbq *abts_wqeq;
3090         unsigned long flags;
3091         int rc;
3092
3093         tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
3094         if (!ctxp->wqeq) {
3095                 ctxp->wqeq = ctxp->ctxbuf->iocbq;
3096                 ctxp->wqeq->hba_wqidx = 0;
3097         }
3098
3099         if (ctxp->state == LPFC_NVMET_STE_FREE) {
3100                 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
3101                                 "6417 NVMET ABORT ctx freed %d %d oxid x%x\n",
3102                                 ctxp->state, ctxp->entry_cnt, ctxp->oxid);
3103                 rc = WQE_BUSY;
3104                 goto aerr;
3105         }
3106         ctxp->state = LPFC_NVMET_STE_ABORT;
3107         ctxp->entry_cnt++;
3108         rc = lpfc_nvmet_unsol_issue_abort(phba, ctxp, sid, xri);
3109         if (rc == 0)
3110                 goto aerr;
3111
3112         spin_lock_irqsave(&phba->hbalock, flags);
3113         abts_wqeq = ctxp->wqeq;
3114         abts_wqeq->wqe_cmpl = lpfc_nvmet_unsol_fcp_abort_cmp;
3115         abts_wqeq->iocb_cmpl = NULL;
3116         abts_wqeq->iocb_flag |= LPFC_IO_NVMET;
3117         rc = lpfc_sli4_issue_wqe(phba, LPFC_FCP_RING, abts_wqeq);
3118         spin_unlock_irqrestore(&phba->hbalock, flags);
3119         if (rc == WQE_SUCCESS) {
3120                 return 0;
3121         }
3122
3123 aerr:
3124         spin_lock_irqsave(&ctxp->ctxlock, flags);
3125         if (ctxp->flag & LPFC_NVMET_CTX_RLS)
3126                 list_del(&ctxp->list);
3127         ctxp->flag &= ~(LPFC_NVMET_ABORT_OP | LPFC_NVMET_CTX_RLS);
3128         spin_unlock_irqrestore(&ctxp->ctxlock, flags);
3129
3130         atomic_inc(&tgtp->xmt_abort_rsp_error);
3131         lpfc_printf_log(phba, KERN_ERR, LOG_NVME_ABTS,
3132                         "6135 Failed to Issue ABTS for oxid x%x. Status x%x\n",
3133                         ctxp->oxid, rc);
3134         lpfc_nvmet_ctxbuf_post(phba, ctxp->ctxbuf);
3135         return 1;
3136 }
3137
3138 static int
3139 lpfc_nvmet_unsol_ls_issue_abort(struct lpfc_hba *phba,
3140                                 struct lpfc_nvmet_rcv_ctx *ctxp,
3141                                 uint32_t sid, uint16_t xri)
3142 {
3143         struct lpfc_nvmet_tgtport *tgtp;
3144         struct lpfc_iocbq *abts_wqeq;
3145         union lpfc_wqe128 *wqe_abts;
3146         unsigned long flags;
3147         int rc;
3148
3149         if ((ctxp->state == LPFC_NVMET_STE_LS_RCV && ctxp->entry_cnt == 1) ||
3150             (ctxp->state == LPFC_NVMET_STE_LS_RSP && ctxp->entry_cnt == 2)) {
3151                 ctxp->state = LPFC_NVMET_STE_LS_ABORT;
3152                 ctxp->entry_cnt++;
3153         } else {
3154                 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
3155                                 "6418 NVMET LS abort state mismatch "
3156                                 "IO x%x: %d %d\n",
3157                                 ctxp->oxid, ctxp->state, ctxp->entry_cnt);
3158                 ctxp->state = LPFC_NVMET_STE_LS_ABORT;
3159         }
3160
3161         tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
3162         if (!ctxp->wqeq) {
3163                 /* Issue ABTS for this WQE based on iotag */
3164                 ctxp->wqeq = lpfc_sli_get_iocbq(phba);
3165                 if (!ctxp->wqeq) {
3166                         lpfc_printf_log(phba, KERN_ERR, LOG_NVME_ABTS,
3167                                         "6068 Abort failed: No wqeqs: "
3168                                         "xri: x%x\n", xri);
3169                         /* No failure to an ABTS request. */
3170                         kfree(ctxp);
3171                         return 0;
3172                 }
3173         }
3174         abts_wqeq = ctxp->wqeq;
3175         wqe_abts = &abts_wqeq->wqe;
3176
3177         if (lpfc_nvmet_unsol_issue_abort(phba, ctxp, sid, xri) == 0) {
3178                 rc = WQE_BUSY;
3179                 goto out;
3180         }
3181
3182         spin_lock_irqsave(&phba->hbalock, flags);
3183         abts_wqeq->wqe_cmpl = lpfc_nvmet_xmt_ls_abort_cmp;
3184         abts_wqeq->iocb_cmpl = 0;
3185         abts_wqeq->iocb_flag |=  LPFC_IO_NVME_LS;
3186         rc = lpfc_sli4_issue_wqe(phba, LPFC_ELS_RING, abts_wqeq);
3187         spin_unlock_irqrestore(&phba->hbalock, flags);
3188         if (rc == WQE_SUCCESS) {
3189                 atomic_inc(&tgtp->xmt_abort_unsol);
3190                 return 0;
3191         }
3192 out:
3193         atomic_inc(&tgtp->xmt_abort_rsp_error);
3194         abts_wqeq->context2 = NULL;
3195         abts_wqeq->context3 = NULL;
3196         lpfc_sli_release_iocbq(phba, abts_wqeq);
3197         kfree(ctxp);
3198         lpfc_printf_log(phba, KERN_ERR, LOG_NVME_ABTS,
3199                         "6056 Failed to Issue ABTS. Status x%x\n", rc);
3200         return 0;
3201 }