drivers/scsi/mpt3sas/mpt3sas_base.c

   1 /*
   2  * This is the Fusion MPT base driver providing common API layer interface
   3  * for access to MPT (Message Passing Technology) firmware.
   4  *
   5  * This code is based on drivers/scsi/mpt3sas/mpt3sas_base.c
   6  * Copyright (C) 2012-2014  LSI Corporation
   7  * Copyright (C) 2013-2014 Avago Technologies
   8  *  (mailto: MPT-FusionLinux.pdl@avagotech.com)
   9  *
  10  * This program is free software; you can redistribute it and/or
  11  * modify it under the terms of the GNU General Public License
  12  * as published by the Free Software Foundation; either version 2
  13  * of the License, or (at your option) any later version.
  14  *
  15  * This program is distributed in the hope that it will be useful,
  16  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  18  * GNU General Public License for more details.
  19  *
  20  * NO WARRANTY
  21  * THE PROGRAM IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OR
  22  * CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED INCLUDING, WITHOUT
  23  * LIMITATION, ANY WARRANTIES OR CONDITIONS OF TITLE, NON-INFRINGEMENT,
  24  * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Each Recipient is
  25  * solely responsible for determining the appropriateness of using and
  26  * distributing the Program and assumes all risks associated with its
  27  * exercise of rights under this Agreement, including but not limited to
  28  * the risks and costs of program errors, damage to or loss of data,
  29  * programs or equipment, and unavailability or interruption of operations.
  30
  31  * DISCLAIMER OF LIABILITY
  32  * NEITHER RECIPIENT NOR ANY CONTRIBUTORS SHALL HAVE ANY LIABILITY FOR ANY
  33  * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  34  * DAMAGES (INCLUDING WITHOUT LIMITATION LOST PROFITS), HOWEVER CAUSED AND
  35  * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
  36  * TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
  37  * USE OR DISTRIBUTION OF THE PROGRAM OR THE EXERCISE OF ANY RIGHTS GRANTED
  38  * HEREUNDER, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGES
  39
  40  * You should have received a copy of the GNU General Public License
  41  * along with this program; if not, write to the Free Software
  42  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301,
  43  * USA.
  44  */
  45
  46 #include <linux/kernel.h>
  47 #include <linux/module.h>
  48 #include <linux/errno.h>
  49 #include <linux/init.h>
  50 #include <linux/slab.h>
  51 #include <linux/types.h>
  52 #include <linux/pci.h>
  53 #include <linux/kdev_t.h>
  54 #include <linux/blkdev.h>
  55 #include <linux/delay.h>
  56 #include <linux/interrupt.h>
  57 #include <linux/dma-mapping.h>
  58 #include <linux/io.h>
  59 #include <linux/time.h>
  60 #include <linux/ktime.h>
  61 #include <linux/kthread.h>
  62 #include <asm/page.h>        /* To get host page size per arch */
  63 #include <linux/aer.h>
  64
  65
  66 #include "mpt3sas_base.h"
  67
  68 static MPT_CALLBACK     mpt_callbacks[MPT_MAX_CALLBACKS];
  69
  70
  71 #define FAULT_POLLING_INTERVAL 1000 /* in milliseconds */
  72
  73  /* maximum controller queue depth */
  74 #define MAX_HBA_QUEUE_DEPTH     30000
  75 #define MAX_CHAIN_DEPTH         100000
  76 static int max_queue_depth = -1;
  77 module_param(max_queue_depth, int, 0);
  78 MODULE_PARM_DESC(max_queue_depth, " max controller queue depth ");
  79
  80 static int max_sgl_entries = -1;
  81 module_param(max_sgl_entries, int, 0);
  82 MODULE_PARM_DESC(max_sgl_entries, " max sg entries ");
  83
  84 static int msix_disable = -1;
  85 module_param(msix_disable, int, 0);
  86 MODULE_PARM_DESC(msix_disable, " disable msix routed interrupts (default=0)");
  87
  88 static int smp_affinity_enable = 1;
  89 module_param(smp_affinity_enable, int, S_IRUGO);
  90 MODULE_PARM_DESC(smp_affinity_enable, "SMP affinity feature enable/disable Default: enable(1)");
  91
  92 static int max_msix_vectors = -1;
  93 module_param(max_msix_vectors, int, 0);
  94 MODULE_PARM_DESC(max_msix_vectors,
  95         " max msix vectors");
  96
  97 static int mpt3sas_fwfault_debug;
  98 MODULE_PARM_DESC(mpt3sas_fwfault_debug,
  99         " enable detection of firmware fault and halt firmware - (default=0)");
 100
 101 static int
 102 _base_get_ioc_facts(struct MPT3SAS_ADAPTER *ioc);
 103
 104 /**
 105  * mpt3sas_base_check_cmd_timeout - Function
 106  *              to check timeout and command termination due
 107  *              to Host reset.
 108  *
 109  * @ioc:        per adapter object.
 110  * @status:     Status of issued command.
 111  * @mpi_request:mf request pointer.
 112  * @sz:         size of buffer.
 113  *
 114  * @Returns - 1/0 Reset to be done or Not
 115  */
 116 u8
 117 mpt3sas_base_check_cmd_timeout(struct MPT3SAS_ADAPTER *ioc,
 118                 u8 status, void *mpi_request, int sz)
 119 {
 120         u8 issue_reset = 0;
 121
 122         if (!(status & MPT3_CMD_RESET))
 123                 issue_reset = 1;
 124
 125         pr_err(MPT3SAS_FMT "Command %s\n", ioc->name,
 126             ((issue_reset == 0) ? "terminated due to Host Reset" : "Timeout"));
 127         _debug_dump_mf(mpi_request, sz);
 128
 129         return issue_reset;
 130 }
 131
 132 /**
 133  * _scsih_set_fwfault_debug - global setting of ioc->fwfault_debug.
 134  * @val: ?
 135  * @kp: ?
 136  *
 137  * Return: ?
 138  */
 139 static int
 140 _scsih_set_fwfault_debug(const char *val, const struct kernel_param *kp)
 141 {
 142         int ret = param_set_int(val, kp);
 143         struct MPT3SAS_ADAPTER *ioc;
 144
 145         if (ret)
 146                 return ret;
 147
 148         /* global ioc spinlock to protect controller list on list operations */
 149         pr_info("setting fwfault_debug(%d)\n", mpt3sas_fwfault_debug);
 150         spin_lock(&gioc_lock);
 151         list_for_each_entry(ioc, &mpt3sas_ioc_list, list)
 152                 ioc->fwfault_debug = mpt3sas_fwfault_debug;
 153         spin_unlock(&gioc_lock);
 154         return 0;
 155 }
 156 module_param_call(mpt3sas_fwfault_debug, _scsih_set_fwfault_debug,
 157         param_get_int, &mpt3sas_fwfault_debug, 0644);
 158
 159 /**
 160  * _base_clone_reply_to_sys_mem - copies reply to reply free iomem
 161  *                                in BAR0 space.
 162  *
 163  * @ioc: per adapter object
 164  * @reply: reply message frame(lower 32bit addr)
 165  * @index: System request message index.
 166  */
 167 static void
 168 _base_clone_reply_to_sys_mem(struct MPT3SAS_ADAPTER *ioc, u32 reply,
 169                 u32 index)
 170 {
 171         /*
 172          * 256 is offset within sys register.
 173          * 256 offset MPI frame starts. Max MPI frame supported is 32.
 174          * 32 * 128 = 4K. From here, Clone of reply free for mcpu starts
 175          */
 176         u16 cmd_credit = ioc->facts.RequestCredit + 1;
 177         void __iomem *reply_free_iomem = (void __iomem *)ioc->chip +
 178                         MPI_FRAME_START_OFFSET +
 179                         (cmd_credit * ioc->request_sz) + (index * sizeof(u32));
 180
 181         writel(reply, reply_free_iomem);
 182 }
 183
 184 /**
 185  * _base_clone_mpi_to_sys_mem - Writes/copies MPI frames
 186  *                              to system/BAR0 region.
 187  *
 188  * @dst_iomem: Pointer to the destination location in BAR0 space.
 189  * @src: Pointer to the Source data.
 190  * @size: Size of data to be copied.
 191  */
 192 static void
 193 _base_clone_mpi_to_sys_mem(void *dst_iomem, void *src, u32 size)
 194 {
 195         int i;
 196         u32 *src_virt_mem = (u32 *)src;
 197
 198         for (i = 0; i < size/4; i++)
 199                 writel((u32)src_virt_mem[i],
 200                                 (void __iomem *)dst_iomem + (i * 4));
 201 }
 202
 203 /**
 204  * _base_clone_to_sys_mem - Writes/copies data to system/BAR0 region
 205  *
 206  * @dst_iomem: Pointer to the destination location in BAR0 space.
 207  * @src: Pointer to the Source data.
 208  * @size: Size of data to be copied.
 209  */
 210 static void
 211 _base_clone_to_sys_mem(void __iomem *dst_iomem, void *src, u32 size)
 212 {
 213         int i;
 214         u32 *src_virt_mem = (u32 *)(src);
 215
 216         for (i = 0; i < size/4; i++)
 217                 writel((u32)src_virt_mem[i],
 218                         (void __iomem *)dst_iomem + (i * 4));
 219 }
 220
 221 /**
 222  * _base_get_chain - Calculates and Returns virtual chain address
 223  *                       for the provided smid in BAR0 space.
 224  *
 225  * @ioc: per adapter object
 226  * @smid: system request message index
 227  * @sge_chain_count: Scatter gather chain count.
 228  *
 229  * Return: the chain address.
 230  */
 231 static inline void __iomem*
 232 _base_get_chain(struct MPT3SAS_ADAPTER *ioc, u16 smid,
 233                 u8 sge_chain_count)
 234 {
 235         void __iomem *base_chain, *chain_virt;
 236         u16 cmd_credit = ioc->facts.RequestCredit + 1;
 237
 238         base_chain  = (void __iomem *)ioc->chip + MPI_FRAME_START_OFFSET +
 239                 (cmd_credit * ioc->request_sz) +
 240                 REPLY_FREE_POOL_SIZE;
 241         chain_virt = base_chain + (smid * ioc->facts.MaxChainDepth *
 242                         ioc->request_sz) + (sge_chain_count * ioc->request_sz);
 243         return chain_virt;
 244 }
 245
 246 /**
 247  * _base_get_chain_phys - Calculates and Returns physical address
 248  *                      in BAR0 for scatter gather chains, for
 249  *                      the provided smid.
 250  *
 251  * @ioc: per adapter object
 252  * @smid: system request message index
 253  * @sge_chain_count: Scatter gather chain count.
 254  *
 255  * Return: Physical chain address.
 256  */
 257 static inline phys_addr_t
 258 _base_get_chain_phys(struct MPT3SAS_ADAPTER *ioc, u16 smid,
 259                 u8 sge_chain_count)
 260 {
 261         phys_addr_t base_chain_phys, chain_phys;
 262         u16 cmd_credit = ioc->facts.RequestCredit + 1;
 263
 264         base_chain_phys  = ioc->chip_phys + MPI_FRAME_START_OFFSET +
 265                 (cmd_credit * ioc->request_sz) +
 266                 REPLY_FREE_POOL_SIZE;
 267         chain_phys = base_chain_phys + (smid * ioc->facts.MaxChainDepth *
 268                         ioc->request_sz) + (sge_chain_count * ioc->request_sz);
 269         return chain_phys;
 270 }
 271
 272 /**
 273  * _base_get_buffer_bar0 - Calculates and Returns BAR0 mapped Host
 274  *                      buffer address for the provided smid.
 275  *                      (Each smid can have 64K starts from 17024)
 276  *
 277  * @ioc: per adapter object
 278  * @smid: system request message index
 279  *
 280  * Return: Pointer to buffer location in BAR0.
 281  */
 282
 283 static void __iomem *
 284 _base_get_buffer_bar0(struct MPT3SAS_ADAPTER *ioc, u16 smid)
 285 {
 286         u16 cmd_credit = ioc->facts.RequestCredit + 1;
 287         // Added extra 1 to reach end of chain.
 288         void __iomem *chain_end = _base_get_chain(ioc,
 289                         cmd_credit + 1,
 290                         ioc->facts.MaxChainDepth);
 291         return chain_end + (smid * 64 * 1024);
 292 }
 293
 294 /**
 295  * _base_get_buffer_phys_bar0 - Calculates and Returns BAR0 mapped
 296  *              Host buffer Physical address for the provided smid.
 297  *              (Each smid can have 64K starts from 17024)
 298  *
 299  * @ioc: per adapter object
 300  * @smid: system request message index
 301  *
 302  * Return: Pointer to buffer location in BAR0.
 303  */
 304 static phys_addr_t
 305 _base_get_buffer_phys_bar0(struct MPT3SAS_ADAPTER *ioc, u16 smid)
 306 {
 307         u16 cmd_credit = ioc->facts.RequestCredit + 1;
 308         phys_addr_t chain_end_phys = _base_get_chain_phys(ioc,
 309                         cmd_credit + 1,
 310                         ioc->facts.MaxChainDepth);
 311         return chain_end_phys + (smid * 64 * 1024);
 312 }
 313
 314 /**
 315  * _base_get_chain_buffer_dma_to_chain_buffer - Iterates chain
 316  *                      lookup list and Provides chain_buffer
 317  *                      address for the matching dma address.
 318  *                      (Each smid can have 64K starts from 17024)
 319  *
 320  * @ioc: per adapter object
 321  * @chain_buffer_dma: Chain buffer dma address.
 322  *
 323  * Return: Pointer to chain buffer. Or Null on Failure.
 324  */
 325 static void *
 326 _base_get_chain_buffer_dma_to_chain_buffer(struct MPT3SAS_ADAPTER *ioc,
 327                 dma_addr_t chain_buffer_dma)
 328 {
 329         u16 index, j;
 330         struct chain_tracker *ct;
 331
 332         for (index = 0; index < ioc->scsiio_depth; index++) {
 333                 for (j = 0; j < ioc->chains_needed_per_io; j++) {
 334                         ct = &ioc->chain_lookup[index].chains_per_smid[j];
 335                         if (ct && ct->chain_buffer_dma == chain_buffer_dma)
 336                                 return ct->chain_buffer;
 337                 }
 338         }
 339         pr_info(MPT3SAS_FMT
 340             "Provided chain_buffer_dma address is not in the lookup list\n",
 341             ioc->name);
 342         return NULL;
 343 }
 344
 345 /**
 346  * _clone_sg_entries -  MPI EP's scsiio and config requests
 347  *                      are handled here. Base function for
 348  *                      double buffering, before submitting
 349  *                      the requests.
 350  *
 351  * @ioc: per adapter object.
 352  * @mpi_request: mf request pointer.
 353  * @smid: system request message index.
 354  */
 355 static void _clone_sg_entries(struct MPT3SAS_ADAPTER *ioc,
 356                 void *mpi_request, u16 smid)
 357 {
 358         Mpi2SGESimple32_t *sgel, *sgel_next;
 359         u32  sgl_flags, sge_chain_count = 0;
 360         bool is_write = 0;
 361         u16 i = 0;
 362         void __iomem *buffer_iomem;
 363         phys_addr_t buffer_iomem_phys;
 364         void __iomem *buff_ptr;
 365         phys_addr_t buff_ptr_phys;
 366         void __iomem *dst_chain_addr[MCPU_MAX_CHAINS_PER_IO];
 367         void *src_chain_addr[MCPU_MAX_CHAINS_PER_IO];
 368         phys_addr_t dst_addr_phys;
 369         MPI2RequestHeader_t *request_hdr;
 370         struct scsi_cmnd *scmd;
 371         struct scatterlist *sg_scmd = NULL;
 372         int is_scsiio_req = 0;
 373
 374         request_hdr = (MPI2RequestHeader_t *) mpi_request;
 375
 376         if (request_hdr->Function == MPI2_FUNCTION_SCSI_IO_REQUEST) {
 377                 Mpi25SCSIIORequest_t *scsiio_request =
 378                         (Mpi25SCSIIORequest_t *)mpi_request;
 379                 sgel = (Mpi2SGESimple32_t *) &scsiio_request->SGL;
 380                 is_scsiio_req = 1;
 381         } else if (request_hdr->Function == MPI2_FUNCTION_CONFIG) {
 382                 Mpi2ConfigRequest_t  *config_req =
 383                         (Mpi2ConfigRequest_t *)mpi_request;
 384                 sgel = (Mpi2SGESimple32_t *) &config_req->PageBufferSGE;
 385         } else
 386                 return;
 387
 388         /* From smid we can get scsi_cmd, once we have sg_scmd,
 389          * we just need to get sg_virt and sg_next to get virual
 390          * address associated with sgel->Address.
 391          */
 392
 393         if (is_scsiio_req) {
 394                 /* Get scsi_cmd using smid */
 395                 scmd = mpt3sas_scsih_scsi_lookup_get(ioc, smid);
 396                 if (scmd == NULL) {
 397                         pr_err(MPT3SAS_FMT "scmd is NULL\n", ioc->name);
 398                         return;
 399                 }
 400
 401                 /* Get sg_scmd from scmd provided */
 402                 sg_scmd = scsi_sglist(scmd);
 403         }
 404
 405         /*
 406          * 0 - 255      System register
 407          * 256 - 4352   MPI Frame. (This is based on maxCredit 32)
 408          * 4352 - 4864  Reply_free pool (512 byte is reserved
 409          *              considering maxCredit 32. Reply need extra
 410          *              room, for mCPU case kept four times of
 411          *              maxCredit).
 412          * 4864 - 17152 SGE chain element. (32cmd * 3 chain of
 413          *              128 byte size = 12288)
 414          * 17152 - x    Host buffer mapped with smid.
 415          *              (Each smid can have 64K Max IO.)
 416          * BAR0+Last 1K MSIX Addr and Data
 417          * Total size in use 2113664 bytes of 4MB BAR0
 418          */
 419
 420         buffer_iomem = _base_get_buffer_bar0(ioc, smid);
 421         buffer_iomem_phys = _base_get_buffer_phys_bar0(ioc, smid);
 422
 423         buff_ptr = buffer_iomem;
 424         buff_ptr_phys = buffer_iomem_phys;
 425         WARN_ON(buff_ptr_phys > U32_MAX);
 426
 427         if (le32_to_cpu(sgel->FlagsLength) &
 428                         (MPI2_SGE_FLAGS_HOST_TO_IOC << MPI2_SGE_FLAGS_SHIFT))
 429                 is_write = 1;
 430
 431         for (i = 0; i < MPT_MIN_PHYS_SEGMENTS + ioc->facts.MaxChainDepth; i++) {
 432
 433                 sgl_flags =
 434                     (le32_to_cpu(sgel->FlagsLength) >> MPI2_SGE_FLAGS_SHIFT);
 435
 436                 switch (sgl_flags & MPI2_SGE_FLAGS_ELEMENT_MASK) {
 437                 case MPI2_SGE_FLAGS_CHAIN_ELEMENT:
 438                         /*
 439                          * Helper function which on passing
 440                          * chain_buffer_dma returns chain_buffer. Get
 441                          * the virtual address for sgel->Address
 442                          */
 443                         sgel_next =
 444                                 _base_get_chain_buffer_dma_to_chain_buffer(ioc,
 445                                                 le32_to_cpu(sgel->Address));
 446                         if (sgel_next == NULL)
 447                                 return;
 448                         /*
 449                          * This is coping 128 byte chain
 450                          * frame (not a host buffer)
 451                          */
 452                         dst_chain_addr[sge_chain_count] =
 453                                 _base_get_chain(ioc,
 454                                         smid, sge_chain_count);
 455                         src_chain_addr[sge_chain_count] =
 456                                                 (void *) sgel_next;
 457                         dst_addr_phys = _base_get_chain_phys(ioc,
 458                                                 smid, sge_chain_count);
 459                         WARN_ON(dst_addr_phys > U32_MAX);
 460                         sgel->Address =
 461                                 cpu_to_le32(lower_32_bits(dst_addr_phys));
 462                         sgel = sgel_next;
 463                         sge_chain_count++;
 464                         break;
 465                 case MPI2_SGE_FLAGS_SIMPLE_ELEMENT:
 466                         if (is_write) {
 467                                 if (is_scsiio_req) {
 468                                         _base_clone_to_sys_mem(buff_ptr,
 469                                             sg_virt(sg_scmd),
 470                                             (le32_to_cpu(sgel->FlagsLength) &
 471                                             0x00ffffff));
 472                                         /*
 473                                          * FIXME: this relies on a a zero
 474                                          * PCI mem_offset.
 475                                          */
 476                                         sgel->Address =
 477                                             cpu_to_le32((u32)buff_ptr_phys);
 478                                 } else {
 479                                         _base_clone_to_sys_mem(buff_ptr,
 480                                             ioc->config_vaddr,
 481                                             (le32_to_cpu(sgel->FlagsLength) &
 482                                             0x00ffffff));
 483                                         sgel->Address =
 484                                             cpu_to_le32((u32)buff_ptr_phys);
 485                                 }
 486                         }
 487                         buff_ptr += (le32_to_cpu(sgel->FlagsLength) &
 488                             0x00ffffff);
 489                         buff_ptr_phys += (le32_to_cpu(sgel->FlagsLength) &
 490                             0x00ffffff);
 491                         if ((le32_to_cpu(sgel->FlagsLength) &
 492                             (MPI2_SGE_FLAGS_END_OF_BUFFER
 493                                         << MPI2_SGE_FLAGS_SHIFT)))
 494                                 goto eob_clone_chain;
 495                         else {
 496                                 /*
 497                                  * Every single element in MPT will have
 498                                  * associated sg_next. Better to sanity that
 499                                  * sg_next is not NULL, but it will be a bug
 500                                  * if it is null.
 501                                  */
 502                                 if (is_scsiio_req) {
 503                                         sg_scmd = sg_next(sg_scmd);
 504                                         if (sg_scmd)
 505                                                 sgel++;
 506                                         else
 507                                                 goto eob_clone_chain;
 508                                 }
 509                         }
 510                         break;
 511                 }
 512         }
 513
 514 eob_clone_chain:
 515         for (i = 0; i < sge_chain_count; i++) {
 516                 if (is_scsiio_req)
 517                         _base_clone_to_sys_mem(dst_chain_addr[i],
 518                                 src_chain_addr[i], ioc->request_sz);
 519         }
 520 }
 521
 522 /**
 523  *  mpt3sas_remove_dead_ioc_func - kthread context to remove dead ioc
 524  * @arg: input argument, used to derive ioc
 525  *
 526  * Return:
 527  * 0 if controller is removed from pci subsystem.
 528  * -1 for other case.
 529  */
 530 static int mpt3sas_remove_dead_ioc_func(void *arg)
 531 {
 532         struct MPT3SAS_ADAPTER *ioc = (struct MPT3SAS_ADAPTER *)arg;
 533         struct pci_dev *pdev;
 534
 535         if ((ioc == NULL))
 536                 return -1;
 537
 538         pdev = ioc->pdev;
 539         if ((pdev == NULL))
 540                 return -1;
 541         pci_stop_and_remove_bus_device_locked(pdev);
 542         return 0;
 543 }
 544
 545 /**
 546  * _base_fault_reset_work - workq handling ioc fault conditions
 547  * @work: input argument, used to derive ioc
 548  *
 549  * Context: sleep.
 550  */
 551 static void
 552 _base_fault_reset_work(struct work_struct *work)
 553 {
 554         struct MPT3SAS_ADAPTER *ioc =
 555             container_of(work, struct MPT3SAS_ADAPTER, fault_reset_work.work);
 556         unsigned long    flags;
 557         u32 doorbell;
 558         int rc;
 559         struct task_struct *p;
 560
 561
 562         spin_lock_irqsave(&ioc->ioc_reset_in_progress_lock, flags);
 563         if (ioc->shost_recovery || ioc->pci_error_recovery)
 564                 goto rearm_timer;
 565         spin_unlock_irqrestore(&ioc->ioc_reset_in_progress_lock, flags);
 566
 567         doorbell = mpt3sas_base_get_iocstate(ioc, 0);
 568         if ((doorbell & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_MASK) {
 569                 pr_err(MPT3SAS_FMT "SAS host is non-operational !!!!\n",
 570                     ioc->name);
 571
 572                 /* It may be possible that EEH recovery can resolve some of
 573                  * pci bus failure issues rather removing the dead ioc function
 574                  * by considering controller is in a non-operational state. So
 575                  * here priority is given to the EEH recovery. If it doesn't
 576                  * not resolve this issue, mpt3sas driver will consider this
 577                  * controller to non-operational state and remove the dead ioc
 578                  * function.
 579                  */
 580                 if (ioc->non_operational_loop++ < 5) {
 581                         spin_lock_irqsave(&ioc->ioc_reset_in_progress_lock,
 582                                                          flags);
 583                         goto rearm_timer;
 584                 }
 585
 586                 /*
 587                  * Call _scsih_flush_pending_cmds callback so that we flush all
 588                  * pending commands back to OS. This call is required to aovid
 589                  * deadlock at block layer. Dead IOC will fail to do diag reset,
 590                  * and this call is safe since dead ioc will never return any
 591                  * command back from HW.
 592                  */
 593                 ioc->schedule_dead_ioc_flush_running_cmds(ioc);
 594                 /*
 595                  * Set remove_host flag early since kernel thread will
 596                  * take some time to execute.
 597                  */
 598                 ioc->remove_host = 1;
 599                 /*Remove the Dead Host */
 600                 p = kthread_run(mpt3sas_remove_dead_ioc_func, ioc,
 601                     "%s_dead_ioc_%d", ioc->driver_name, ioc->id);
 602                 if (IS_ERR(p))
 603                         pr_err(MPT3SAS_FMT
 604                         "%s: Running mpt3sas_dead_ioc thread failed !!!!\n",
 605                         ioc->name, __func__);
 606                 else
 607                         pr_err(MPT3SAS_FMT
 608                         "%s: Running mpt3sas_dead_ioc thread success !!!!\n",
 609                         ioc->name, __func__);
 610                 return; /* don't rearm timer */
 611         }
 612
 613         ioc->non_operational_loop = 0;
 614
 615         if ((doorbell & MPI2_IOC_STATE_MASK) != MPI2_IOC_STATE_OPERATIONAL) {
 616                 rc = mpt3sas_base_hard_reset_handler(ioc, FORCE_BIG_HAMMER);
 617                 pr_warn(MPT3SAS_FMT "%s: hard reset: %s\n", ioc->name,
 618                     __func__, (rc == 0) ? "success" : "failed");
 619                 doorbell = mpt3sas_base_get_iocstate(ioc, 0);
 620                 if ((doorbell & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_FAULT)
 621                         mpt3sas_base_fault_info(ioc, doorbell &
 622                             MPI2_DOORBELL_DATA_MASK);
 623                 if (rc && (doorbell & MPI2_IOC_STATE_MASK) !=
 624                     MPI2_IOC_STATE_OPERATIONAL)
 625                         return; /* don't rearm timer */
 626         }
 627
 628         spin_lock_irqsave(&ioc->ioc_reset_in_progress_lock, flags);
 629  rearm_timer:
 630         if (ioc->fault_reset_work_q)
 631                 queue_delayed_work(ioc->fault_reset_work_q,
 632                     &ioc->fault_reset_work,
 633                     msecs_to_jiffies(FAULT_POLLING_INTERVAL));
 634         spin_unlock_irqrestore(&ioc->ioc_reset_in_progress_lock, flags);
 635 }
 636
 637 /**
 638  * mpt3sas_base_start_watchdog - start the fault_reset_work_q
 639  * @ioc: per adapter object
 640  *
 641  * Context: sleep.
 642  */
 643 void
 644 mpt3sas_base_start_watchdog(struct MPT3SAS_ADAPTER *ioc)
 645 {
 646         unsigned long    flags;
 647
 648         if (ioc->fault_reset_work_q)
 649                 return;
 650
 651         /* initialize fault polling */
 652
 653         INIT_DELAYED_WORK(&ioc->fault_reset_work, _base_fault_reset_work);
 654         snprintf(ioc->fault_reset_work_q_name,
 655             sizeof(ioc->fault_reset_work_q_name), "poll_%s%d_status",
 656             ioc->driver_name, ioc->id);
 657         ioc->fault_reset_work_q =
 658                 create_singlethread_workqueue(ioc->fault_reset_work_q_name);
 659         if (!ioc->fault_reset_work_q) {
 660                 pr_err(MPT3SAS_FMT "%s: failed (line=%d)\n",
 661                     ioc->name, __func__, __LINE__);
 662                 return;
 663         }
 664         spin_lock_irqsave(&ioc->ioc_reset_in_progress_lock, flags);
 665         if (ioc->fault_reset_work_q)
 666                 queue_delayed_work(ioc->fault_reset_work_q,
 667                     &ioc->fault_reset_work,
 668                     msecs_to_jiffies(FAULT_POLLING_INTERVAL));
 669         spin_unlock_irqrestore(&ioc->ioc_reset_in_progress_lock, flags);
 670 }
 671
 672 /**
 673  * mpt3sas_base_stop_watchdog - stop the fault_reset_work_q
 674  * @ioc: per adapter object
 675  *
 676  * Context: sleep.
 677  */
 678 void
 679 mpt3sas_base_stop_watchdog(struct MPT3SAS_ADAPTER *ioc)
 680 {
 681         unsigned long flags;
 682         struct workqueue_struct *wq;
 683
 684         spin_lock_irqsave(&ioc->ioc_reset_in_progress_lock, flags);
 685         wq = ioc->fault_reset_work_q;
 686         ioc->fault_reset_work_q = NULL;
 687         spin_unlock_irqrestore(&ioc->ioc_reset_in_progress_lock, flags);
 688         if (wq) {
 689                 if (!cancel_delayed_work_sync(&ioc->fault_reset_work))
 690                         flush_workqueue(wq);
 691                 destroy_workqueue(wq);
 692         }
 693 }
 694
 695 /**
 696  * mpt3sas_base_fault_info - verbose translation of firmware FAULT code
 697  * @ioc: per adapter object
 698  * @fault_code: fault code
 699  */
 700 void
 701 mpt3sas_base_fault_info(struct MPT3SAS_ADAPTER *ioc , u16 fault_code)
 702 {
 703         pr_err(MPT3SAS_FMT "fault_state(0x%04x)!\n",
 704             ioc->name, fault_code);
 705 }
 706
 707 /**
 708  * mpt3sas_halt_firmware - halt's mpt controller firmware
 709  * @ioc: per adapter object
 710  *
 711  * For debugging timeout related issues.  Writing 0xCOFFEE00
 712  * to the doorbell register will halt controller firmware. With
 713  * the purpose to stop both driver and firmware, the enduser can
 714  * obtain a ring buffer from controller UART.
 715  */
 716 void
 717 mpt3sas_halt_firmware(struct MPT3SAS_ADAPTER *ioc)
 718 {
 719         u32 doorbell;
 720
 721         if (!ioc->fwfault_debug)
 722                 return;
 723
 724         dump_stack();
 725
 726         doorbell = readl(&ioc->chip->Doorbell);
 727         if ((doorbell & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_FAULT)
 728                 mpt3sas_base_fault_info(ioc , doorbell);
 729         else {
 730                 writel(0xC0FFEE00, &ioc->chip->Doorbell);
 731                 pr_err(MPT3SAS_FMT "Firmware is halted due to command timeout\n",
 732                         ioc->name);
 733         }
 734
 735         if (ioc->fwfault_debug == 2)
 736                 for (;;)
 737                         ;
 738         else
 739                 panic("panic in %s\n", __func__);
 740 }
 741
 742 /**
 743  * _base_sas_ioc_info - verbose translation of the ioc status
 744  * @ioc: per adapter object
 745  * @mpi_reply: reply mf payload returned from firmware
 746  * @request_hdr: request mf
 747  */
 748 static void
 749 _base_sas_ioc_info(struct MPT3SAS_ADAPTER *ioc, MPI2DefaultReply_t *mpi_reply,
 750         MPI2RequestHeader_t *request_hdr)
 751 {
 752         u16 ioc_status = le16_to_cpu(mpi_reply->IOCStatus) &
 753             MPI2_IOCSTATUS_MASK;
 754         char *desc = NULL;
 755         u16 frame_sz;
 756         char *func_str = NULL;
 757
 758         /* SCSI_IO, RAID_PASS are handled from _scsih_scsi_ioc_info */
 759         if (request_hdr->Function == MPI2_FUNCTION_SCSI_IO_REQUEST ||
 760             request_hdr->Function == MPI2_FUNCTION_RAID_SCSI_IO_PASSTHROUGH ||
 761             request_hdr->Function == MPI2_FUNCTION_EVENT_NOTIFICATION)
 762                 return;
 763
 764         if (ioc_status == MPI2_IOCSTATUS_CONFIG_INVALID_PAGE)
 765                 return;
 766
 767         switch (ioc_status) {
 768
 769 /****************************************************************************
 770 *  Common IOCStatus values for all replies
 771 ****************************************************************************/
 772
 773         case MPI2_IOCSTATUS_INVALID_FUNCTION:
 774                 desc = "invalid function";
 775                 break;
 776         case MPI2_IOCSTATUS_BUSY:
 777                 desc = "busy";
 778                 break;
 779         case MPI2_IOCSTATUS_INVALID_SGL:
 780                 desc = "invalid sgl";
 781                 break;
 782         case MPI2_IOCSTATUS_INTERNAL_ERROR:
 783                 desc = "internal error";
 784                 break;
 785         case MPI2_IOCSTATUS_INVALID_VPID:
 786                 desc = "invalid vpid";
 787                 break;
 788         case MPI2_IOCSTATUS_INSUFFICIENT_RESOURCES:
 789                 desc = "insufficient resources";
 790                 break;
 791         case MPI2_IOCSTATUS_INSUFFICIENT_POWER:
 792                 desc = "insufficient power";
 793                 break;
 794         case MPI2_IOCSTATUS_INVALID_FIELD:
 795                 desc = "invalid field";
 796                 break;
 797         case MPI2_IOCSTATUS_INVALID_STATE:
 798                 desc = "invalid state";
 799                 break;
 800         case MPI2_IOCSTATUS_OP_STATE_NOT_SUPPORTED:
 801                 desc = "op state not supported";
 802                 break;
 803
 804 /****************************************************************************
 805 *  Config IOCStatus values
 806 ****************************************************************************/
 807
 808         case MPI2_IOCSTATUS_CONFIG_INVALID_ACTION:
 809                 desc = "config invalid action";
 810                 break;
 811         case MPI2_IOCSTATUS_CONFIG_INVALID_TYPE:
 812                 desc = "config invalid type";
 813                 break;
 814         case MPI2_IOCSTATUS_CONFIG_INVALID_PAGE:
 815                 desc = "config invalid page";
 816                 break;
 817         case MPI2_IOCSTATUS_CONFIG_INVALID_DATA:
 818                 desc = "config invalid data";
 819                 break;
 820         case MPI2_IOCSTATUS_CONFIG_NO_DEFAULTS:
 821                 desc = "config no defaults";
 822                 break;
 823         case MPI2_IOCSTATUS_CONFIG_CANT_COMMIT:
 824                 desc = "config cant commit";
 825                 break;
 826
 827 /****************************************************************************
 828 *  SCSI IO Reply
 829 ****************************************************************************/
 830
 831         case MPI2_IOCSTATUS_SCSI_RECOVERED_ERROR:
 832         case MPI2_IOCSTATUS_SCSI_INVALID_DEVHANDLE:
 833         case MPI2_IOCSTATUS_SCSI_DEVICE_NOT_THERE:
 834         case MPI2_IOCSTATUS_SCSI_DATA_OVERRUN:
 835         case MPI2_IOCSTATUS_SCSI_DATA_UNDERRUN:
 836         case MPI2_IOCSTATUS_SCSI_IO_DATA_ERROR:
 837         case MPI2_IOCSTATUS_SCSI_PROTOCOL_ERROR:
 838         case MPI2_IOCSTATUS_SCSI_TASK_TERMINATED:
 839         case MPI2_IOCSTATUS_SCSI_RESIDUAL_MISMATCH:
 840         case MPI2_IOCSTATUS_SCSI_TASK_MGMT_FAILED:
 841         case MPI2_IOCSTATUS_SCSI_IOC_TERMINATED:
 842         case MPI2_IOCSTATUS_SCSI_EXT_TERMINATED:
 843                 break;
 844
 845 /****************************************************************************
 846 *  For use by SCSI Initiator and SCSI Target end-to-end data protection
 847 ****************************************************************************/
 848
 849         case MPI2_IOCSTATUS_EEDP_GUARD_ERROR:
 850                 desc = "eedp guard error";
 851                 break;
 852         case MPI2_IOCSTATUS_EEDP_REF_TAG_ERROR:
 853                 desc = "eedp ref tag error";
 854                 break;
 855         case MPI2_IOCSTATUS_EEDP_APP_TAG_ERROR:
 856                 desc = "eedp app tag error";
 857                 break;
 858
 859 /****************************************************************************
 860 *  SCSI Target values
 861 ****************************************************************************/
 862
 863         case MPI2_IOCSTATUS_TARGET_INVALID_IO_INDEX:
 864                 desc = "target invalid io index";
 865                 break;
 866         case MPI2_IOCSTATUS_TARGET_ABORTED:
 867                 desc = "target aborted";
 868                 break;
 869         case MPI2_IOCSTATUS_TARGET_NO_CONN_RETRYABLE:
 870                 desc = "target no conn retryable";
 871                 break;
 872         case MPI2_IOCSTATUS_TARGET_NO_CONNECTION:
 873                 desc = "target no connection";
 874                 break;
 875         case MPI2_IOCSTATUS_TARGET_XFER_COUNT_MISMATCH:
 876                 desc = "target xfer count mismatch";
 877                 break;
 878         case MPI2_IOCSTATUS_TARGET_DATA_OFFSET_ERROR:
 879                 desc = "target data offset error";
 880                 break;
 881         case MPI2_IOCSTATUS_TARGET_TOO_MUCH_WRITE_DATA:
 882                 desc = "target too much write data";
 883                 break;
 884         case MPI2_IOCSTATUS_TARGET_IU_TOO_SHORT:
 885                 desc = "target iu too short";
 886                 break;
 887         case MPI2_IOCSTATUS_TARGET_ACK_NAK_TIMEOUT:
 888                 desc = "target ack nak timeout";
 889                 break;
 890         case MPI2_IOCSTATUS_TARGET_NAK_RECEIVED:
 891                 desc = "target nak received";
 892                 break;
 893
 894 /****************************************************************************
 895 *  Serial Attached SCSI values
 896 ****************************************************************************/
 897
 898         case MPI2_IOCSTATUS_SAS_SMP_REQUEST_FAILED:
 899                 desc = "smp request failed";
 900                 break;
 901         case MPI2_IOCSTATUS_SAS_SMP_DATA_OVERRUN:
 902                 desc = "smp data overrun";
 903                 break;
 904
 905 /****************************************************************************
 906 *  Diagnostic Buffer Post / Diagnostic Release values
 907 ****************************************************************************/
 908
 909         case MPI2_IOCSTATUS_DIAGNOSTIC_RELEASED:
 910                 desc = "diagnostic released";
 911                 break;
 912         default:
 913                 break;
 914         }
 915
 916         if (!desc)
 917                 return;
 918
 919         switch (request_hdr->Function) {
 920         case MPI2_FUNCTION_CONFIG:
 921                 frame_sz = sizeof(Mpi2ConfigRequest_t) + ioc->sge_size;
 922                 func_str = "config_page";
 923                 break;
 924         case MPI2_FUNCTION_SCSI_TASK_MGMT:
 925                 frame_sz = sizeof(Mpi2SCSITaskManagementRequest_t);
 926                 func_str = "task_mgmt";
 927                 break;
 928         case MPI2_FUNCTION_SAS_IO_UNIT_CONTROL:
 929                 frame_sz = sizeof(Mpi2SasIoUnitControlRequest_t);
 930                 func_str = "sas_iounit_ctl";
 931                 break;
 932         case MPI2_FUNCTION_SCSI_ENCLOSURE_PROCESSOR:
 933                 frame_sz = sizeof(Mpi2SepRequest_t);
 934                 func_str = "enclosure";
 935                 break;
 936         case MPI2_FUNCTION_IOC_INIT:
 937                 frame_sz = sizeof(Mpi2IOCInitRequest_t);
 938                 func_str = "ioc_init";
 939                 break;
 940         case MPI2_FUNCTION_PORT_ENABLE:
 941                 frame_sz = sizeof(Mpi2PortEnableRequest_t);
 942                 func_str = "port_enable";
 943                 break;
 944         case MPI2_FUNCTION_SMP_PASSTHROUGH:
 945                 frame_sz = sizeof(Mpi2SmpPassthroughRequest_t) + ioc->sge_size;
 946                 func_str = "smp_passthru";
 947                 break;
 948         case MPI2_FUNCTION_NVME_ENCAPSULATED:
 949                 frame_sz = sizeof(Mpi26NVMeEncapsulatedRequest_t) +
 950                     ioc->sge_size;
 951                 func_str = "nvme_encapsulated";
 952                 break;
 953         default:
 954                 frame_sz = 32;
 955                 func_str = "unknown";
 956                 break;
 957         }
 958
 959         pr_warn(MPT3SAS_FMT "ioc_status: %s(0x%04x), request(0x%p),(%s)\n",
 960                 ioc->name, desc, ioc_status, request_hdr, func_str);
 961
 962         _debug_dump_mf(request_hdr, frame_sz/4);
 963 }
 964
 965 /**
 966  * _base_display_event_data - verbose translation of firmware asyn events
 967  * @ioc: per adapter object
 968  * @mpi_reply: reply mf payload returned from firmware
 969  */
 970 static void
 971 _base_display_event_data(struct MPT3SAS_ADAPTER *ioc,
 972         Mpi2EventNotificationReply_t *mpi_reply)
 973 {
 974         char *desc = NULL;
 975         u16 event;
 976
 977         if (!(ioc->logging_level & MPT_DEBUG_EVENTS))
 978                 return;
 979
 980         event = le16_to_cpu(mpi_reply->Event);
 981
 982         switch (event) {
 983         case MPI2_EVENT_LOG_DATA:
 984                 desc = "Log Data";
 985                 break;
 986         case MPI2_EVENT_STATE_CHANGE:
 987                 desc = "Status Change";
 988                 break;
 989         case MPI2_EVENT_HARD_RESET_RECEIVED:
 990                 desc = "Hard Reset Received";
 991                 break;
 992         case MPI2_EVENT_EVENT_CHANGE:
 993                 desc = "Event Change";
 994                 break;
 995         case MPI2_EVENT_SAS_DEVICE_STATUS_CHANGE:
 996                 desc = "Device Status Change";
 997                 break;
 998         case MPI2_EVENT_IR_OPERATION_STATUS:
 999                 if (!ioc->hide_ir_msg)
1000                         desc = "IR Operation Status";
1001                 break;
1002         case MPI2_EVENT_SAS_DISCOVERY:
1003         {
1004                 Mpi2EventDataSasDiscovery_t *event_data =
1005                     (Mpi2EventDataSasDiscovery_t *)mpi_reply->EventData;
1006                 pr_info(MPT3SAS_FMT "Discovery: (%s)", ioc->name,
1007                     (event_data->ReasonCode == MPI2_EVENT_SAS_DISC_RC_STARTED) ?
1008                     "start" : "stop");
1009                 if (event_data->DiscoveryStatus)
1010                         pr_cont(" discovery_status(0x%08x)",
1011                             le32_to_cpu(event_data->DiscoveryStatus));
1012                 pr_cont("\n");
1013                 return;
1014         }
1015         case MPI2_EVENT_SAS_BROADCAST_PRIMITIVE:
1016                 desc = "SAS Broadcast Primitive";
1017                 break;
1018         case MPI2_EVENT_SAS_INIT_DEVICE_STATUS_CHANGE:
1019                 desc = "SAS Init Device Status Change";
1020                 break;
1021         case MPI2_EVENT_SAS_INIT_TABLE_OVERFLOW:
1022                 desc = "SAS Init Table Overflow";
1023                 break;
1024         case MPI2_EVENT_SAS_TOPOLOGY_CHANGE_LIST:
1025                 desc = "SAS Topology Change List";
1026                 break;
1027         case MPI2_EVENT_SAS_ENCL_DEVICE_STATUS_CHANGE:
1028                 desc = "SAS Enclosure Device Status Change";
1029                 break;
1030         case MPI2_EVENT_IR_VOLUME:
1031                 if (!ioc->hide_ir_msg)
1032                         desc = "IR Volume";
1033                 break;
1034         case MPI2_EVENT_IR_PHYSICAL_DISK:
1035                 if (!ioc->hide_ir_msg)
1036                         desc = "IR Physical Disk";
1037                 break;
1038         case MPI2_EVENT_IR_CONFIGURATION_CHANGE_LIST:
1039                 if (!ioc->hide_ir_msg)
1040                         desc = "IR Configuration Change List";
1041                 break;
1042         case MPI2_EVENT_LOG_ENTRY_ADDED:
1043                 if (!ioc->hide_ir_msg)
1044                         desc = "Log Entry Added";
1045                 break;
1046         case MPI2_EVENT_TEMP_THRESHOLD:
1047                 desc = "Temperature Threshold";
1048                 break;
1049         case MPI2_EVENT_ACTIVE_CABLE_EXCEPTION:
1050                 desc = "Cable Event";
1051                 break;
1052         case MPI2_EVENT_SAS_DEVICE_DISCOVERY_ERROR:
1053                 desc = "SAS Device Discovery Error";
1054                 break;
1055         case MPI2_EVENT_PCIE_DEVICE_STATUS_CHANGE:
1056                 desc = "PCIE Device Status Change";
1057                 break;
1058         case MPI2_EVENT_PCIE_ENUMERATION:
1059         {
1060                 Mpi26EventDataPCIeEnumeration_t *event_data =
1061                         (Mpi26EventDataPCIeEnumeration_t *)mpi_reply->EventData;
1062                 pr_info(MPT3SAS_FMT "PCIE Enumeration: (%s)", ioc->name,
1063                            (event_data->ReasonCode ==
1064                                 MPI26_EVENT_PCIE_ENUM_RC_STARTED) ?
1065                                 "start" : "stop");
1066                 if (event_data->EnumerationStatus)
1067                         pr_info("enumeration_status(0x%08x)",
1068                                    le32_to_cpu(event_data->EnumerationStatus));
1069                 pr_info("\n");
1070                 return;
1071         }
1072         case MPI2_EVENT_PCIE_TOPOLOGY_CHANGE_LIST:
1073                 desc = "PCIE Topology Change List";
1074                 break;
1075         }
1076
1077         if (!desc)
1078                 return;
1079
1080         pr_info(MPT3SAS_FMT "%s\n", ioc->name, desc);
1081 }
1082
1083 /**
1084  * _base_sas_log_info - verbose translation of firmware log info
1085  * @ioc: per adapter object
1086  * @log_info: log info
1087  */
1088 static void
1089 _base_sas_log_info(struct MPT3SAS_ADAPTER *ioc , u32 log_info)
1090 {
1091         union loginfo_type {
1092                 u32     loginfo;
1093                 struct {
1094                         u32     subcode:16;
1095                         u32     code:8;
1096                         u32     originator:4;
1097                         u32     bus_type:4;
1098                 } dw;
1099         };
1100         union loginfo_type sas_loginfo;
1101         char *originator_str = NULL;
1102
1103         sas_loginfo.loginfo = log_info;
1104         if (sas_loginfo.dw.bus_type != 3 /*SAS*/)
1105                 return;
1106
1107         /* each nexus loss loginfo */
1108         if (log_info == 0x31170000)
1109                 return;
1110
1111         /* eat the loginfos associated with task aborts */
1112         if (ioc->ignore_loginfos && (log_info == 0x30050000 || log_info ==
1113             0x31140000 || log_info == 0x31130000))
1114                 return;
1115
1116         switch (sas_loginfo.dw.originator) {
1117         case 0:
1118                 originator_str = "IOP";
1119                 break;
1120         case 1:
1121                 originator_str = "PL";
1122                 break;
1123         case 2:
1124                 if (!ioc->hide_ir_msg)
1125                         originator_str = "IR";
1126                 else
1127                         originator_str = "WarpDrive";
1128                 break;
1129         }
1130
1131         pr_warn(MPT3SAS_FMT
1132                 "log_info(0x%08x): originator(%s), code(0x%02x), sub_code(0x%04x)\n",
1133                 ioc->name, log_info,
1134              originator_str, sas_loginfo.dw.code,
1135              sas_loginfo.dw.subcode);
1136 }
1137
1138 /**
1139  * _base_display_reply_info -
1140  * @ioc: per adapter object
1141  * @smid: system request message index
1142  * @msix_index: MSIX table index supplied by the OS
1143  * @reply: reply message frame(lower 32bit addr)
1144  */
1145 static void
1146 _base_display_reply_info(struct MPT3SAS_ADAPTER *ioc, u16 smid, u8 msix_index,
1147         u32 reply)
1148 {
1149         MPI2DefaultReply_t *mpi_reply;
1150         u16 ioc_status;
1151         u32 loginfo = 0;
1152
1153         mpi_reply = mpt3sas_base_get_reply_virt_addr(ioc, reply);
1154         if (unlikely(!mpi_reply)) {
1155                 pr_err(MPT3SAS_FMT "mpi_reply not valid at %s:%d/%s()!\n",
1156                     ioc->name, __FILE__, __LINE__, __func__);
1157                 return;
1158         }
1159         ioc_status = le16_to_cpu(mpi_reply->IOCStatus);
1160
1161         if ((ioc_status & MPI2_IOCSTATUS_MASK) &&
1162             (ioc->logging_level & MPT_DEBUG_REPLY)) {
1163                 _base_sas_ioc_info(ioc , mpi_reply,
1164                    mpt3sas_base_get_msg_frame(ioc, smid));
1165         }
1166
1167         if (ioc_status & MPI2_IOCSTATUS_FLAG_LOG_INFO_AVAILABLE) {
1168                 loginfo = le32_to_cpu(mpi_reply->IOCLogInfo);
1169                 _base_sas_log_info(ioc, loginfo);
1170         }
1171
1172         if (ioc_status || loginfo) {
1173                 ioc_status &= MPI2_IOCSTATUS_MASK;
1174                 mpt3sas_trigger_mpi(ioc, ioc_status, loginfo);
1175         }
1176 }
1177
1178 /**
1179  * mpt3sas_base_done - base internal command completion routine
1180  * @ioc: per adapter object
1181  * @smid: system request message index
1182  * @msix_index: MSIX table index supplied by the OS
1183  * @reply: reply message frame(lower 32bit addr)
1184  *
1185  * Return:
1186  * 1 meaning mf should be freed from _base_interrupt
1187  * 0 means the mf is freed from this function.
1188  */
1189 u8
1190 mpt3sas_base_done(struct MPT3SAS_ADAPTER *ioc, u16 smid, u8 msix_index,
1191         u32 reply)
1192 {
1193         MPI2DefaultReply_t *mpi_reply;
1194
1195         mpi_reply = mpt3sas_base_get_reply_virt_addr(ioc, reply);
1196         if (mpi_reply && mpi_reply->Function == MPI2_FUNCTION_EVENT_ACK)
1197                 return mpt3sas_check_for_pending_internal_cmds(ioc, smid);
1198
1199         if (ioc->base_cmds.status == MPT3_CMD_NOT_USED)
1200                 return 1;
1201
1202         ioc->base_cmds.status |= MPT3_CMD_COMPLETE;
1203         if (mpi_reply) {
1204                 ioc->base_cmds.status |= MPT3_CMD_REPLY_VALID;
1205                 memcpy(ioc->base_cmds.reply, mpi_reply, mpi_reply->MsgLength*4);
1206         }
1207         ioc->base_cmds.status &= ~MPT3_CMD_PENDING;
1208
1209         complete(&ioc->base_cmds.done);
1210         return 1;
1211 }
1212
1213 /**
1214  * _base_async_event - main callback handler for firmware asyn events
1215  * @ioc: per adapter object
1216  * @msix_index: MSIX table index supplied by the OS
1217  * @reply: reply message frame(lower 32bit addr)
1218  *
1219  * Return:
1220  * 1 meaning mf should be freed from _base_interrupt
1221  * 0 means the mf is freed from this function.
1222  */
1223 static u8
1224 _base_async_event(struct MPT3SAS_ADAPTER *ioc, u8 msix_index, u32 reply)
1225 {
1226         Mpi2EventNotificationReply_t *mpi_reply;
1227         Mpi2EventAckRequest_t *ack_request;
1228         u16 smid;
1229         struct _event_ack_list *delayed_event_ack;
1230
1231         mpi_reply = mpt3sas_base_get_reply_virt_addr(ioc, reply);
1232         if (!mpi_reply)
1233                 return 1;
1234         if (mpi_reply->Function != MPI2_FUNCTION_EVENT_NOTIFICATION)
1235                 return 1;
1236
1237         _base_display_event_data(ioc, mpi_reply);
1238
1239         if (!(mpi_reply->AckRequired & MPI2_EVENT_NOTIFICATION_ACK_REQUIRED))
1240                 goto out;
1241         smid = mpt3sas_base_get_smid(ioc, ioc->base_cb_idx);
1242         if (!smid) {
1243                 delayed_event_ack = kzalloc(sizeof(*delayed_event_ack),
1244                                         GFP_ATOMIC);
1245                 if (!delayed_event_ack)
1246                         goto out;
1247                 INIT_LIST_HEAD(&delayed_event_ack->list);
1248                 delayed_event_ack->Event = mpi_reply->Event;
1249                 delayed_event_ack->EventContext = mpi_reply->EventContext;
1250                 list_add_tail(&delayed_event_ack->list,
1251                                 &ioc->delayed_event_ack_list);
1252                 dewtprintk(ioc, pr_info(MPT3SAS_FMT
1253                                 "DELAYED: EVENT ACK: event (0x%04x)\n",
1254                                 ioc->name, le16_to_cpu(mpi_reply->Event)));
1255                 goto out;
1256         }
1257
1258         ack_request = mpt3sas_base_get_msg_frame(ioc, smid);
1259         memset(ack_request, 0, sizeof(Mpi2EventAckRequest_t));
1260         ack_request->Function = MPI2_FUNCTION_EVENT_ACK;
1261         ack_request->Event = mpi_reply->Event;
1262         ack_request->EventContext = mpi_reply->EventContext;
1263         ack_request->VF_ID = 0;  /* TODO */
1264         ack_request->VP_ID = 0;
1265         mpt3sas_base_put_smid_default(ioc, smid);
1266
1267  out:
1268
1269         /* scsih callback handler */
1270         mpt3sas_scsih_event_callback(ioc, msix_index, reply);
1271
1272         /* ctl callback handler */
1273         mpt3sas_ctl_event_callback(ioc, msix_index, reply);
1274
1275         return 1;
1276 }
1277
1278 static struct scsiio_tracker *
1279 _get_st_from_smid(struct MPT3SAS_ADAPTER *ioc, u16 smid)
1280 {
1281         struct scsi_cmnd *cmd;
1282
1283         if (WARN_ON(!smid) ||
1284             WARN_ON(smid >= ioc->hi_priority_smid))
1285                 return NULL;
1286
1287         cmd = mpt3sas_scsih_scsi_lookup_get(ioc, smid);
1288         if (cmd)
1289                 return scsi_cmd_priv(cmd);
1290
1291         return NULL;
1292 }
1293
1294 /**
1295  * _base_get_cb_idx - obtain the callback index
1296  * @ioc: per adapter object
1297  * @smid: system request message index
1298  *
1299  * Return: callback index.
1300  */
1301 static u8
1302 _base_get_cb_idx(struct MPT3SAS_ADAPTER *ioc, u16 smid)
1303 {
1304         int i;
1305         u16 ctl_smid = ioc->scsiio_depth - INTERNAL_SCSIIO_CMDS_COUNT + 1;
1306         u8 cb_idx = 0xFF;
1307
1308         if (smid < ioc->hi_priority_smid) {
1309                 struct scsiio_tracker *st;
1310
1311                 if (smid < ctl_smid) {
1312                         st = _get_st_from_smid(ioc, smid);
1313                         if (st)
1314                                 cb_idx = st->cb_idx;
1315                 } else if (smid == ctl_smid)
1316                         cb_idx = ioc->ctl_cb_idx;
1317         } else if (smid < ioc->internal_smid) {
1318                 i = smid - ioc->hi_priority_smid;
1319                 cb_idx = ioc->hpr_lookup[i].cb_idx;
1320         } else if (smid <= ioc->hba_queue_depth) {
1321                 i = smid - ioc->internal_smid;
1322                 cb_idx = ioc->internal_lookup[i].cb_idx;
1323         }
1324         return cb_idx;
1325 }
1326
1327 /**
1328  * _base_mask_interrupts - disable interrupts
1329  * @ioc: per adapter object
1330  *
1331  * Disabling ResetIRQ, Reply and Doorbell Interrupts
1332  */
1333 static void
1334 _base_mask_interrupts(struct MPT3SAS_ADAPTER *ioc)
1335 {
1336         u32 him_register;
1337
1338         ioc->mask_interrupts = 1;
1339         him_register = readl(&ioc->chip->HostInterruptMask);
1340         him_register |= MPI2_HIM_DIM + MPI2_HIM_RIM + MPI2_HIM_RESET_IRQ_MASK;
1341         writel(him_register, &ioc->chip->HostInterruptMask);
1342         readl(&ioc->chip->HostInterruptMask);
1343 }
1344
1345 /**
1346  * _base_unmask_interrupts - enable interrupts
1347  * @ioc: per adapter object
1348  *
1349  * Enabling only Reply Interrupts
1350  */
1351 static void
1352 _base_unmask_interrupts(struct MPT3SAS_ADAPTER *ioc)
1353 {
1354         u32 him_register;
1355
1356         him_register = readl(&ioc->chip->HostInterruptMask);
1357         him_register &= ~MPI2_HIM_RIM;
1358         writel(him_register, &ioc->chip->HostInterruptMask);
1359         ioc->mask_interrupts = 0;
1360 }
1361
1362 union reply_descriptor {
1363         u64 word;
1364         struct {
1365                 u32 low;
1366                 u32 high;
1367         } u;
1368 };
1369
1370 /**
1371  * _base_interrupt - MPT adapter (IOC) specific interrupt handler.
1372  * @irq: irq number (not used)
1373  * @bus_id: bus identifier cookie == pointer to MPT_ADAPTER structure
1374  *
1375  * Return: IRQ_HANDLED if processed, else IRQ_NONE.
1376  */
1377 static irqreturn_t
1378 _base_interrupt(int irq, void *bus_id)
1379 {
1380         struct adapter_reply_queue *reply_q = bus_id;
1381         union reply_descriptor rd;
1382         u32 completed_cmds;
1383         u8 request_desript_type;
1384         u16 smid;
1385         u8 cb_idx;
1386         u32 reply;
1387         u8 msix_index = reply_q->msix_index;
1388         struct MPT3SAS_ADAPTER *ioc = reply_q->ioc;
1389         Mpi2ReplyDescriptorsUnion_t *rpf;
1390         u8 rc;
1391
1392         if (ioc->mask_interrupts)
1393                 return IRQ_NONE;
1394
1395         if (!atomic_add_unless(&reply_q->busy, 1, 1))
1396                 return IRQ_NONE;
1397
1398         rpf = &reply_q->reply_post_free[reply_q->reply_post_host_index];
1399         request_desript_type = rpf->Default.ReplyFlags
1400              & MPI2_RPY_DESCRIPT_FLAGS_TYPE_MASK;
1401         if (request_desript_type == MPI2_RPY_DESCRIPT_FLAGS_UNUSED) {
1402                 atomic_dec(&reply_q->busy);
1403                 return IRQ_NONE;
1404         }
1405
1406         completed_cmds = 0;
1407         cb_idx = 0xFF;
1408         do {
1409                 rd.word = le64_to_cpu(rpf->Words);
1410                 if (rd.u.low == UINT_MAX || rd.u.high == UINT_MAX)
1411                         goto out;
1412                 reply = 0;
1413                 smid = le16_to_cpu(rpf->Default.DescriptorTypeDependent1);
1414                 if (request_desript_type ==
1415                     MPI25_RPY_DESCRIPT_FLAGS_FAST_PATH_SCSI_IO_SUCCESS ||
1416                     request_desript_type ==
1417                     MPI2_RPY_DESCRIPT_FLAGS_SCSI_IO_SUCCESS ||
1418                     request_desript_type ==
1419                     MPI26_RPY_DESCRIPT_FLAGS_PCIE_ENCAPSULATED_SUCCESS) {
1420                         cb_idx = _base_get_cb_idx(ioc, smid);
1421                         if ((likely(cb_idx < MPT_MAX_CALLBACKS)) &&
1422                             (likely(mpt_callbacks[cb_idx] != NULL))) {
1423                                 rc = mpt_callbacks[cb_idx](ioc, smid,
1424                                     msix_index, 0);
1425                                 if (rc)
1426                                         mpt3sas_base_free_smid(ioc, smid);
1427                         }
1428                 } else if (request_desript_type ==
1429                     MPI2_RPY_DESCRIPT_FLAGS_ADDRESS_REPLY) {
1430                         reply = le32_to_cpu(
1431                             rpf->AddressReply.ReplyFrameAddress);
1432                         if (reply > ioc->reply_dma_max_address ||
1433                             reply < ioc->reply_dma_min_address)
1434                                 reply = 0;
1435                         if (smid) {
1436                                 cb_idx = _base_get_cb_idx(ioc, smid);
1437                                 if ((likely(cb_idx < MPT_MAX_CALLBACKS)) &&
1438                                     (likely(mpt_callbacks[cb_idx] != NULL))) {
1439                                         rc = mpt_callbacks[cb_idx](ioc, smid,
1440                                             msix_index, reply);
1441                                         if (reply)
1442                                                 _base_display_reply_info(ioc,
1443                                                     smid, msix_index, reply);
1444                                         if (rc)
1445                                                 mpt3sas_base_free_smid(ioc,
1446                                                     smid);
1447                                 }
1448                         } else {
1449                                 _base_async_event(ioc, msix_index, reply);
1450                         }
1451
1452                         /* reply free queue handling */
1453                         if (reply) {
1454                                 ioc->reply_free_host_index =
1455                                     (ioc->reply_free_host_index ==
1456                                     (ioc->reply_free_queue_depth - 1)) ?
1457                                     0 : ioc->reply_free_host_index + 1;
1458                                 ioc->reply_free[ioc->reply_free_host_index] =
1459                                     cpu_to_le32(reply);
1460                                 if (ioc->is_mcpu_endpoint)
1461                                         _base_clone_reply_to_sys_mem(ioc,
1462                                                 reply,
1463                                                 ioc->reply_free_host_index);
1464                                 writel(ioc->reply_free_host_index,
1465                                     &ioc->chip->ReplyFreeHostIndex);
1466                         }
1467                 }
1468
1469                 rpf->Words = cpu_to_le64(ULLONG_MAX);
1470                 reply_q->reply_post_host_index =
1471                     (reply_q->reply_post_host_index ==
1472                     (ioc->reply_post_queue_depth - 1)) ? 0 :
1473                     reply_q->reply_post_host_index + 1;
1474                 request_desript_type =
1475                     reply_q->reply_post_free[reply_q->reply_post_host_index].
1476                     Default.ReplyFlags & MPI2_RPY_DESCRIPT_FLAGS_TYPE_MASK;
1477                 completed_cmds++;
1478                 /* Update the reply post host index after continuously
1479                  * processing the threshold number of Reply Descriptors.
1480                  * So that FW can find enough entries to post the Reply
1481                  * Descriptors in the reply descriptor post queue.
1482                  */
1483                 if (completed_cmds > ioc->hba_queue_depth/3) {
1484                         if (ioc->combined_reply_queue) {
1485                                 writel(reply_q->reply_post_host_index |
1486                                                 ((msix_index  & 7) <<
1487                                                  MPI2_RPHI_MSIX_INDEX_SHIFT),
1488                                     ioc->replyPostRegisterIndex[msix_index/8]);
1489                         } else {
1490                                 writel(reply_q->reply_post_host_index |
1491                                                 (msix_index <<
1492                                                  MPI2_RPHI_MSIX_INDEX_SHIFT),
1493                                                 &ioc->chip->ReplyPostHostIndex);
1494                         }
1495                         completed_cmds = 1;
1496                 }
1497                 if (request_desript_type == MPI2_RPY_DESCRIPT_FLAGS_UNUSED)
1498                         goto out;
1499                 if (!reply_q->reply_post_host_index)
1500                         rpf = reply_q->reply_post_free;
1501                 else
1502                         rpf++;
1503         } while (1);
1504
1505  out:
1506
1507         if (!completed_cmds) {
1508                 atomic_dec(&reply_q->busy);
1509                 return IRQ_NONE;
1510         }
1511
1512         if (ioc->is_warpdrive) {
1513                 writel(reply_q->reply_post_host_index,
1514                 ioc->reply_post_host_index[msix_index]);
1515                 atomic_dec(&reply_q->busy);
1516                 return IRQ_HANDLED;
1517         }
1518
1519         /* Update Reply Post Host Index.
1520          * For those HBA's which support combined reply queue feature
1521          * 1. Get the correct Supplemental Reply Post Host Index Register.
1522          *    i.e. (msix_index / 8)th entry from Supplemental Reply Post Host
1523          *    Index Register address bank i.e replyPostRegisterIndex[],
1524          * 2. Then update this register with new reply host index value
1525          *    in ReplyPostIndex field and the MSIxIndex field with
1526          *    msix_index value reduced to a value between 0 and 7,
1527          *    using a modulo 8 operation. Since each Supplemental Reply Post
1528          *    Host Index Register supports 8 MSI-X vectors.
1529          *
1530          * For other HBA's just update the Reply Post Host Index register with
1531          * new reply host index value in ReplyPostIndex Field and msix_index
1532          * value in MSIxIndex field.
1533          */
1534         if (ioc->combined_reply_queue)
1535                 writel(reply_q->reply_post_host_index | ((msix_index  & 7) <<
1536                         MPI2_RPHI_MSIX_INDEX_SHIFT),
1537                         ioc->replyPostRegisterIndex[msix_index/8]);
1538         else
1539                 writel(reply_q->reply_post_host_index | (msix_index <<
1540                         MPI2_RPHI_MSIX_INDEX_SHIFT),
1541                         &ioc->chip->ReplyPostHostIndex);
1542         atomic_dec(&reply_q->busy);
1543         return IRQ_HANDLED;
1544 }
1545
1546 /**
1547  * _base_is_controller_msix_enabled - is controller support muli-reply queues
1548  * @ioc: per adapter object
1549  *
1550  * Return: Whether or not MSI/X is enabled.
1551  */
1552 static inline int
1553 _base_is_controller_msix_enabled(struct MPT3SAS_ADAPTER *ioc)
1554 {
1555         return (ioc->facts.IOCCapabilities &
1556             MPI2_IOCFACTS_CAPABILITY_MSI_X_INDEX) && ioc->msix_enable;
1557 }
1558
1559 /**
1560  * mpt3sas_base_sync_reply_irqs - flush pending MSIX interrupts
1561  * @ioc: per adapter object
1562  * Context: non ISR conext
1563  *
1564  * Called when a Task Management request has completed.
1565  */
1566 void
1567 mpt3sas_base_sync_reply_irqs(struct MPT3SAS_ADAPTER *ioc)
1568 {
1569         struct adapter_reply_queue *reply_q;
1570
1571         /* If MSIX capability is turned off
1572          * then multi-queues are not enabled
1573          */
1574         if (!_base_is_controller_msix_enabled(ioc))
1575                 return;
1576
1577         list_for_each_entry(reply_q, &ioc->reply_queue_list, list) {
1578                 if (ioc->shost_recovery || ioc->remove_host ||
1579                                 ioc->pci_error_recovery)
1580                         return;
1581                 /* TMs are on msix_index == 0 */
1582                 if (reply_q->msix_index == 0)
1583                         continue;
1584                 synchronize_irq(pci_irq_vector(ioc->pdev, reply_q->msix_index));
1585         }
1586 }
1587
1588 /**
1589  * mpt3sas_base_release_callback_handler - clear interrupt callback handler
1590  * @cb_idx: callback index
1591  */
1592 void
1593 mpt3sas_base_release_callback_handler(u8 cb_idx)
1594 {
1595         mpt_callbacks[cb_idx] = NULL;
1596 }
1597
1598 /**
1599  * mpt3sas_base_register_callback_handler - obtain index for the interrupt callback handler
1600  * @cb_func: callback function
1601  *
1602  * Return: Index of @cb_func.
1603  */
1604 u8
1605 mpt3sas_base_register_callback_handler(MPT_CALLBACK cb_func)
1606 {
1607         u8 cb_idx;
1608
1609         for (cb_idx = MPT_MAX_CALLBACKS-1; cb_idx; cb_idx--)
1610                 if (mpt_callbacks[cb_idx] == NULL)
1611                         break;
1612
1613         mpt_callbacks[cb_idx] = cb_func;
1614         return cb_idx;
1615 }
1616
1617 /**
1618  * mpt3sas_base_initialize_callback_handler - initialize the interrupt callback handler
1619  */
1620 void
1621 mpt3sas_base_initialize_callback_handler(void)
1622 {
1623         u8 cb_idx;
1624
1625         for (cb_idx = 0; cb_idx < MPT_MAX_CALLBACKS; cb_idx++)
1626                 mpt3sas_base_release_callback_handler(cb_idx);
1627 }
1628
1629
1630 /**
1631  * _base_build_zero_len_sge - build zero length sg entry
1632  * @ioc: per adapter object
1633  * @paddr: virtual address for SGE
1634  *
1635  * Create a zero length scatter gather entry to insure the IOCs hardware has
1636  * something to use if the target device goes brain dead and tries
1637  * to send data even when none is asked for.
1638  */
1639 static void
1640 _base_build_zero_len_sge(struct MPT3SAS_ADAPTER *ioc, void *paddr)
1641 {
1642         u32 flags_length = (u32)((MPI2_SGE_FLAGS_LAST_ELEMENT |
1643             MPI2_SGE_FLAGS_END_OF_BUFFER | MPI2_SGE_FLAGS_END_OF_LIST |
1644             MPI2_SGE_FLAGS_SIMPLE_ELEMENT) <<
1645             MPI2_SGE_FLAGS_SHIFT);
1646         ioc->base_add_sg_single(paddr, flags_length, -1);
1647 }
1648
1649 /**
1650  * _base_add_sg_single_32 - Place a simple 32 bit SGE at address pAddr.
1651  * @paddr: virtual address for SGE
1652  * @flags_length: SGE flags and data transfer length
1653  * @dma_addr: Physical address
1654  */
1655 static void
1656 _base_add_sg_single_32(void *paddr, u32 flags_length, dma_addr_t dma_addr)
1657 {
1658         Mpi2SGESimple32_t *sgel = paddr;
1659
1660         flags_length |= (MPI2_SGE_FLAGS_32_BIT_ADDRESSING |
1661             MPI2_SGE_FLAGS_SYSTEM_ADDRESS) << MPI2_SGE_FLAGS_SHIFT;
1662         sgel->FlagsLength = cpu_to_le32(flags_length);
1663         sgel->Address = cpu_to_le32(dma_addr);
1664 }
1665
1666
1667 /**
1668  * _base_add_sg_single_64 - Place a simple 64 bit SGE at address pAddr.
1669  * @paddr: virtual address for SGE
1670  * @flags_length: SGE flags and data transfer length
1671  * @dma_addr: Physical address
1672  */
1673 static void
1674 _base_add_sg_single_64(void *paddr, u32 flags_length, dma_addr_t dma_addr)
1675 {
1676         Mpi2SGESimple64_t *sgel = paddr;
1677
1678         flags_length |= (MPI2_SGE_FLAGS_64_BIT_ADDRESSING |
1679             MPI2_SGE_FLAGS_SYSTEM_ADDRESS) << MPI2_SGE_FLAGS_SHIFT;
1680         sgel->FlagsLength = cpu_to_le32(flags_length);
1681         sgel->Address = cpu_to_le64(dma_addr);
1682 }
1683
1684 /**
1685  * _base_get_chain_buffer_tracker - obtain chain tracker
1686  * @ioc: per adapter object
1687  * @scmd: SCSI commands of the IO request
1688  *
1689  * Return: chain tracker from chain_lookup table using key as
1690  * smid and smid's chain_offset.
1691  */
1692 static struct chain_tracker *
1693 _base_get_chain_buffer_tracker(struct MPT3SAS_ADAPTER *ioc,
1694                                struct scsi_cmnd *scmd)
1695 {
1696         struct chain_tracker *chain_req;
1697         struct scsiio_tracker *st = scsi_cmd_priv(scmd);
1698         u16 smid = st->smid;
1699         u8 chain_offset =
1700            atomic_read(&ioc->chain_lookup[smid - 1].chain_offset);
1701
1702         if (chain_offset == ioc->chains_needed_per_io)
1703                 return NULL;
1704
1705         chain_req = &ioc->chain_lookup[smid - 1].chains_per_smid[chain_offset];
1706         atomic_inc(&ioc->chain_lookup[smid - 1].chain_offset);
1707         return chain_req;
1708 }
1709
1710
1711 /**
1712  * _base_build_sg - build generic sg
1713  * @ioc: per adapter object
1714  * @psge: virtual address for SGE
1715  * @data_out_dma: physical address for WRITES
1716  * @data_out_sz: data xfer size for WRITES
1717  * @data_in_dma: physical address for READS
1718  * @data_in_sz: data xfer size for READS
1719  */
1720 static void
1721 _base_build_sg(struct MPT3SAS_ADAPTER *ioc, void *psge,
1722         dma_addr_t data_out_dma, size_t data_out_sz, dma_addr_t data_in_dma,
1723         size_t data_in_sz)
1724 {
1725         u32 sgl_flags;
1726
1727         if (!data_out_sz && !data_in_sz) {
1728                 _base_build_zero_len_sge(ioc, psge);
1729                 return;
1730         }
1731
1732         if (data_out_sz && data_in_sz) {
1733                 /* WRITE sgel first */
1734                 sgl_flags = (MPI2_SGE_FLAGS_SIMPLE_ELEMENT |
1735                     MPI2_SGE_FLAGS_END_OF_BUFFER | MPI2_SGE_FLAGS_HOST_TO_IOC);
1736                 sgl_flags = sgl_flags << MPI2_SGE_FLAGS_SHIFT;
1737                 ioc->base_add_sg_single(psge, sgl_flags |
1738                     data_out_sz, data_out_dma);
1739
1740                 /* incr sgel */
1741                 psge += ioc->sge_size;
1742
1743                 /* READ sgel last */
1744                 sgl_flags = (MPI2_SGE_FLAGS_SIMPLE_ELEMENT |
1745                     MPI2_SGE_FLAGS_LAST_ELEMENT | MPI2_SGE_FLAGS_END_OF_BUFFER |
1746                     MPI2_SGE_FLAGS_END_OF_LIST);
1747                 sgl_flags = sgl_flags << MPI2_SGE_FLAGS_SHIFT;
1748                 ioc->base_add_sg_single(psge, sgl_flags |
1749                     data_in_sz, data_in_dma);
1750         } else if (data_out_sz) /* WRITE */ {
1751                 sgl_flags = (MPI2_SGE_FLAGS_SIMPLE_ELEMENT |
1752                     MPI2_SGE_FLAGS_LAST_ELEMENT | MPI2_SGE_FLAGS_END_OF_BUFFER |
1753                     MPI2_SGE_FLAGS_END_OF_LIST | MPI2_SGE_FLAGS_HOST_TO_IOC);
1754                 sgl_flags = sgl_flags << MPI2_SGE_FLAGS_SHIFT;
1755                 ioc->base_add_sg_single(psge, sgl_flags |
1756                     data_out_sz, data_out_dma);
1757         } else if (data_in_sz) /* READ */ {
1758                 sgl_flags = (MPI2_SGE_FLAGS_SIMPLE_ELEMENT |
1759                     MPI2_SGE_FLAGS_LAST_ELEMENT | MPI2_SGE_FLAGS_END_OF_BUFFER |
1760                     MPI2_SGE_FLAGS_END_OF_LIST);
1761                 sgl_flags = sgl_flags << MPI2_SGE_FLAGS_SHIFT;
1762                 ioc->base_add_sg_single(psge, sgl_flags |
1763                     data_in_sz, data_in_dma);
1764         }
1765 }
1766
1767 /* IEEE format sgls */
1768
1769 /**
1770  * _base_build_nvme_prp - This function is called for NVMe end devices to build
1771  * a native SGL (NVMe PRP). The native SGL is built starting in the first PRP
1772  * entry of the NVMe message (PRP1).  If the data buffer is small enough to be
1773  * described entirely using PRP1, then PRP2 is not used.  If needed, PRP2 is
1774  * used to describe a larger data buffer.  If the data buffer is too large to
1775  * describe using the two PRP entriess inside the NVMe message, then PRP1
1776  * describes the first data memory segment, and PRP2 contains a pointer to a PRP
1777  * list located elsewhere in memory to describe the remaining data memory
1778  * segments.  The PRP list will be contiguous.
1779  *
1780  * The native SGL for NVMe devices is a Physical Region Page (PRP).  A PRP
1781  * consists of a list of PRP entries to describe a number of noncontigous
1782  * physical memory segments as a single memory buffer, just as a SGL does.  Note
1783  * however, that this function is only used by the IOCTL call, so the memory
1784  * given will be guaranteed to be contiguous.  There is no need to translate
1785  * non-contiguous SGL into a PRP in this case.  All PRPs will describe
1786  * contiguous space that is one page size each.
1787  *
1788  * Each NVMe message contains two PRP entries.  The first (PRP1) either contains
1789  * a PRP list pointer or a PRP element, depending upon the command.  PRP2
1790  * contains the second PRP element if the memory being described fits within 2
1791  * PRP entries, or a PRP list pointer if the PRP spans more than two entries.
1792  *
1793  * A PRP list pointer contains the address of a PRP list, structured as a linear
1794  * array of PRP entries.  Each PRP entry in this list describes a segment of
1795  * physical memory.
1796  *
1797  * Each 64-bit PRP entry comprises an address and an offset field.  The address
1798  * always points at the beginning of a 4KB physical memory page, and the offset
1799  * describes where within that 4KB page the memory segment begins.  Only the
1800  * first element in a PRP list may contain a non-zero offest, implying that all
1801  * memory segments following the first begin at the start of a 4KB page.
1802  *
1803  * Each PRP element normally describes 4KB of physical memory, with exceptions
1804  * for the first and last elements in the list.  If the memory being described
1805  * by the list begins at a non-zero offset within the first 4KB page, then the
1806  * first PRP element will contain a non-zero offset indicating where the region
1807  * begins within the 4KB page.  The last memory segment may end before the end
1808  * of the 4KB segment, depending upon the overall size of the memory being
1809  * described by the PRP list.
1810  *
1811  * Since PRP entries lack any indication of size, the overall data buffer length
1812  * is used to determine where the end of the data memory buffer is located, and
1813  * how many PRP entries are required to describe it.
1814  *
1815  * @ioc: per adapter object
1816  * @smid: system request message index for getting asscociated SGL
1817  * @nvme_encap_request: the NVMe request msg frame pointer
1818  * @data_out_dma: physical address for WRITES
1819  * @data_out_sz: data xfer size for WRITES
1820  * @data_in_dma: physical address for READS
1821  * @data_in_sz: data xfer size for READS
1822  */
1823 static void
1824 _base_build_nvme_prp(struct MPT3SAS_ADAPTER *ioc, u16 smid,
1825         Mpi26NVMeEncapsulatedRequest_t *nvme_encap_request,
1826         dma_addr_t data_out_dma, size_t data_out_sz, dma_addr_t data_in_dma,
1827         size_t data_in_sz)
1828 {
1829         int             prp_size = NVME_PRP_SIZE;
1830         __le64          *prp_entry, *prp1_entry, *prp2_entry;
1831         __le64          *prp_page;
1832         dma_addr_t      prp_entry_dma, prp_page_dma, dma_addr;
1833         u32             offset, entry_len;
1834         u32             page_mask_result, page_mask;
1835         size_t          length;
1836         struct mpt3sas_nvme_cmd *nvme_cmd =
1837                 (void *)nvme_encap_request->NVMe_Command;
1838
1839         /*
1840          * Not all commands require a data transfer. If no data, just return
1841          * without constructing any PRP.
1842          */
1843         if (!data_in_sz && !data_out_sz)
1844                 return;
1845         prp1_entry = &nvme_cmd->prp1;
1846         prp2_entry = &nvme_cmd->prp2;
1847         prp_entry = prp1_entry;
1848         /*
1849          * For the PRP entries, use the specially allocated buffer of
1850          * contiguous memory.
1851          */
1852         prp_page = (__le64 *)mpt3sas_base_get_pcie_sgl(ioc, smid);
1853         prp_page_dma = mpt3sas_base_get_pcie_sgl_dma(ioc, smid);
1854
1855         /*
1856          * Check if we are within 1 entry of a page boundary we don't
1857          * want our first entry to be a PRP List entry.
1858          */
1859         page_mask = ioc->page_size - 1;
1860         page_mask_result = (uintptr_t)((u8 *)prp_page + prp_size) & page_mask;
1861         if (!page_mask_result) {
1862                 /* Bump up to next page boundary. */
1863                 prp_page = (__le64 *)((u8 *)prp_page + prp_size);
1864                 prp_page_dma = prp_page_dma + prp_size;
1865         }
1866
1867         /*
1868          * Set PRP physical pointer, which initially points to the current PRP
1869          * DMA memory page.
1870          */
1871         prp_entry_dma = prp_page_dma;
1872
1873         /* Get physical address and length of the data buffer. */
1874         if (data_in_sz) {
1875                 dma_addr = data_in_dma;
1876                 length = data_in_sz;
1877         } else {
1878                 dma_addr = data_out_dma;
1879                 length = data_out_sz;
1880         }
1881
1882         /* Loop while the length is not zero. */
1883         while (length) {
1884                 /*
1885                  * Check if we need to put a list pointer here if we are at
1886                  * page boundary - prp_size (8 bytes).
1887                  */
1888                 page_mask_result = (prp_entry_dma + prp_size) & page_mask;
1889                 if (!page_mask_result) {
1890                         /*
1891                          * This is the last entry in a PRP List, so we need to
1892                          * put a PRP list pointer here.  What this does is:
1893                          *   - bump the current memory pointer to the next
1894                          *     address, which will be the next full page.
1895                          *   - set the PRP Entry to point to that page.  This
1896                          *     is now the PRP List pointer.
1897                          *   - bump the PRP Entry pointer the start of the
1898                          *     next page.  Since all of this PRP memory is
1899                          *     contiguous, no need to get a new page - it's
1900                          *     just the next address.
1901                          */
1902                         prp_entry_dma++;
1903                         *prp_entry = cpu_to_le64(prp_entry_dma);
1904                         prp_entry++;
1905                 }
1906
1907                 /* Need to handle if entry will be part of a page. */
1908                 offset = dma_addr & page_mask;
1909                 entry_len = ioc->page_size - offset;
1910
1911                 if (prp_entry == prp1_entry) {
1912                         /*
1913                          * Must fill in the first PRP pointer (PRP1) before
1914                          * moving on.
1915                          */
1916                         *prp1_entry = cpu_to_le64(dma_addr);
1917
1918                         /*
1919                          * Now point to the second PRP entry within the
1920                          * command (PRP2).
1921                          */
1922                         prp_entry = prp2_entry;
1923                 } else if (prp_entry == prp2_entry) {
1924                         /*
1925                          * Should the PRP2 entry be a PRP List pointer or just
1926                          * a regular PRP pointer?  If there is more than one
1927                          * more page of data, must use a PRP List pointer.
1928                          */
1929                         if (length > ioc->page_size) {
1930                                 /*
1931                                  * PRP2 will contain a PRP List pointer because
1932                                  * more PRP's are needed with this command. The
1933                                  * list will start at the beginning of the
1934                                  * contiguous buffer.
1935                                  */
1936                                 *prp2_entry = cpu_to_le64(prp_entry_dma);
1937
1938                                 /*
1939                                  * The next PRP Entry will be the start of the
1940                                  * first PRP List.
1941                                  */
1942                                 prp_entry = prp_page;
1943                         } else {
1944                                 /*
1945                                  * After this, the PRP Entries are complete.
1946                                  * This command uses 2 PRP's and no PRP list.
1947                                  */
1948                                 *prp2_entry = cpu_to_le64(dma_addr);
1949                         }
1950                 } else {
1951                         /*
1952                          * Put entry in list and bump the addresses.
1953                          *
1954                          * After PRP1 and PRP2 are filled in, this will fill in
1955                          * all remaining PRP entries in a PRP List, one per
1956                          * each time through the loop.
1957                          */
1958                         *prp_entry = cpu_to_le64(dma_addr);
1959                         prp_entry++;
1960                         prp_entry_dma++;
1961                 }
1962
1963                 /*
1964                  * Bump the phys address of the command's data buffer by the
1965                  * entry_len.
1966                  */
1967                 dma_addr += entry_len;
1968
1969                 /* Decrement length accounting for last partial page. */
1970                 if (entry_len > length)
1971                         length = 0;
1972                 else
1973                         length -= entry_len;
1974         }
1975 }
1976
1977 /**
1978  * base_make_prp_nvme -
1979  * Prepare PRPs(Physical Region Page)- SGLs specific to NVMe drives only
1980  *
1981  * @ioc:                per adapter object
1982  * @scmd:               SCSI command from the mid-layer
1983  * @mpi_request:        mpi request
1984  * @smid:               msg Index
1985  * @sge_count:          scatter gather element count.
1986  *
1987  * Return:              true: PRPs are built
1988  *                      false: IEEE SGLs needs to be built
1989  */
1990 static void
1991 base_make_prp_nvme(struct MPT3SAS_ADAPTER *ioc,
1992                 struct scsi_cmnd *scmd,
1993                 Mpi25SCSIIORequest_t *mpi_request,
1994                 u16 smid, int sge_count)
1995 {
1996         int sge_len, num_prp_in_chain = 0;
1997         Mpi25IeeeSgeChain64_t *main_chain_element, *ptr_first_sgl;
1998         __le64 *curr_buff;
1999         dma_addr_t msg_dma, sge_addr, offset;
2000         u32 page_mask, page_mask_result;
2001         struct scatterlist *sg_scmd;
2002         u32 first_prp_len;
2003         int data_len = scsi_bufflen(scmd);
2004         u32 nvme_pg_size;
2005
2006         nvme_pg_size = max_t(u32, ioc->page_size, NVME_PRP_PAGE_SIZE);
2007         /*
2008          * Nvme has a very convoluted prp format.  One prp is required
2009          * for each page or partial page. Driver need to split up OS sg_list
2010          * entries if it is longer than one page or cross a page
2011          * boundary.  Driver also have to insert a PRP list pointer entry as
2012          * the last entry in each physical page of the PRP list.
2013          *
2014          * NOTE: The first PRP "entry" is actually placed in the first
2015          * SGL entry in the main message as IEEE 64 format.  The 2nd
2016          * entry in the main message is the chain element, and the rest
2017          * of the PRP entries are built in the contiguous pcie buffer.
2018          */
2019         page_mask = nvme_pg_size - 1;
2020
2021         /*
2022          * Native SGL is needed.
2023          * Put a chain element in main message frame that points to the first
2024          * chain buffer.
2025          *
2026          * NOTE:  The ChainOffset field must be 0 when using a chain pointer to
2027          *        a native SGL.
2028          */
2029
2030         /* Set main message chain element pointer */
2031         main_chain_element = (pMpi25IeeeSgeChain64_t)&mpi_request->SGL;
2032         /*
2033          * For NVMe the chain element needs to be the 2nd SG entry in the main
2034          * message.
2035          */
2036         main_chain_element = (Mpi25IeeeSgeChain64_t *)
2037                 ((u8 *)main_chain_element + sizeof(MPI25_IEEE_SGE_CHAIN64));
2038
2039         /*
2040          * For the PRP entries, use the specially allocated buffer of
2041          * contiguous memory.  Normal chain buffers can't be used
2042          * because each chain buffer would need to be the size of an OS
2043          * page (4k).
2044          */
2045         curr_buff = mpt3sas_base_get_pcie_sgl(ioc, smid);
2046         msg_dma = mpt3sas_base_get_pcie_sgl_dma(ioc, smid);
2047
2048         main_chain_element->Address = cpu_to_le64(msg_dma);
2049         main_chain_element->NextChainOffset = 0;
2050         main_chain_element->Flags = MPI2_IEEE_SGE_FLAGS_CHAIN_ELEMENT |
2051                         MPI2_IEEE_SGE_FLAGS_SYSTEM_ADDR |
2052                         MPI26_IEEE_SGE_FLAGS_NSF_NVME_PRP;
2053
2054         /* Build first prp, sge need not to be page aligned*/
2055         ptr_first_sgl = (pMpi25IeeeSgeChain64_t)&mpi_request->SGL;
2056         sg_scmd = scsi_sglist(scmd);
2057         sge_addr = sg_dma_address(sg_scmd);
2058         sge_len = sg_dma_len(sg_scmd);
2059
2060         offset = sge_addr & page_mask;
2061         first_prp_len = nvme_pg_size - offset;
2062
2063         ptr_first_sgl->Address = cpu_to_le64(sge_addr);
2064         ptr_first_sgl->Length = cpu_to_le32(first_prp_len);
2065
2066         data_len -= first_prp_len;
2067
2068         if (sge_len > first_prp_len) {
2069                 sge_addr += first_prp_len;
2070                 sge_len -= first_prp_len;
2071         } else if (data_len && (sge_len == first_prp_len)) {
2072                 sg_scmd = sg_next(sg_scmd);
2073                 sge_addr = sg_dma_address(sg_scmd);
2074                 sge_len = sg_dma_len(sg_scmd);
2075         }
2076
2077         for (;;) {
2078                 offset = sge_addr & page_mask;
2079
2080                 /* Put PRP pointer due to page boundary*/
2081                 page_mask_result = (uintptr_t)(curr_buff + 1) & page_mask;
2082                 if (unlikely(!page_mask_result)) {
2083                         scmd_printk(KERN_NOTICE,
2084                                 scmd, "page boundary curr_buff: 0x%p\n",
2085                                 curr_buff);
2086                         msg_dma += 8;
2087                         *curr_buff = cpu_to_le64(msg_dma);
2088                         curr_buff++;
2089                         num_prp_in_chain++;
2090                 }
2091
2092                 *curr_buff = cpu_to_le64(sge_addr);
2093                 curr_buff++;
2094                 msg_dma += 8;
2095                 num_prp_in_chain++;
2096
2097                 sge_addr += nvme_pg_size;
2098                 sge_len -= nvme_pg_size;
2099                 data_len -= nvme_pg_size;
2100
2101                 if (data_len <= 0)
2102                         break;
2103
2104                 if (sge_len > 0)
2105                         continue;
2106
2107                 sg_scmd = sg_next(sg_scmd);
2108                 sge_addr = sg_dma_address(sg_scmd);
2109                 sge_len = sg_dma_len(sg_scmd);
2110         }
2111
2112         main_chain_element->Length =
2113                 cpu_to_le32(num_prp_in_chain * sizeof(u64));
2114         return;
2115 }
2116
2117 static bool
2118 base_is_prp_possible(struct MPT3SAS_ADAPTER *ioc,
2119         struct _pcie_device *pcie_device, struct scsi_cmnd *scmd, int sge_count)
2120 {
2121         u32 data_length = 0;
2122         bool build_prp = true;
2123
2124         data_length = scsi_bufflen(scmd);
2125
2126         /* If Datalenth is <= 16K and number of SGE’s entries are <= 2
2127          * we built IEEE SGL
2128          */
2129         if ((data_length <= NVME_PRP_PAGE_SIZE*4) && (sge_count <= 2))
2130                 build_prp = false;
2131
2132         return build_prp;
2133 }
2134
2135 /**
2136  * _base_check_pcie_native_sgl - This function is called for PCIe end devices to
2137  * determine if the driver needs to build a native SGL.  If so, that native
2138  * SGL is built in the special contiguous buffers allocated especially for
2139  * PCIe SGL creation.  If the driver will not build a native SGL, return
2140  * TRUE and a normal IEEE SGL will be built.  Currently this routine
2141  * supports NVMe.
2142  * @ioc: per adapter object
2143  * @mpi_request: mf request pointer
2144  * @smid: system request message index
2145  * @scmd: scsi command
2146  * @pcie_device: points to the PCIe device's info
2147  *
2148  * Return: 0 if native SGL was built, 1 if no SGL was built
2149  */
2150 static int
2151 _base_check_pcie_native_sgl(struct MPT3SAS_ADAPTER *ioc,
2152         Mpi25SCSIIORequest_t *mpi_request, u16 smid, struct scsi_cmnd *scmd,
2153         struct _pcie_device *pcie_device)
2154 {
2155         int sges_left;
2156
2157         /* Get the SG list pointer and info. */
2158         sges_left = scsi_dma_map(scmd);
2159         if (sges_left < 0) {
2160                 sdev_printk(KERN_ERR, scmd->device,
2161                         "scsi_dma_map failed: request for %d bytes!\n",
2162                         scsi_bufflen(scmd));
2163                 return 1;
2164         }
2165
2166         /* Check if we need to build a native SG list. */
2167         if (base_is_prp_possible(ioc, pcie_device,
2168                                 scmd, sges_left) == 0) {
2169                 /* We built a native SG list, just return. */
2170                 goto out;
2171         }
2172
2173         /*
2174          * Build native NVMe PRP.
2175          */
2176         base_make_prp_nvme(ioc, scmd, mpi_request,
2177                         smid, sges_left);
2178
2179         return 0;
2180 out:
2181         scsi_dma_unmap(scmd);
2182         return 1;
2183 }
2184
2185 /**
2186  * _base_add_sg_single_ieee - add sg element for IEEE format
2187  * @paddr: virtual address for SGE
2188  * @flags: SGE flags
2189  * @chain_offset: number of 128 byte elements from start of segment
2190  * @length: data transfer length
2191  * @dma_addr: Physical address
2192  */
2193 static void
2194 _base_add_sg_single_ieee(void *paddr, u8 flags, u8 chain_offset, u32 length,
2195         dma_addr_t dma_addr)
2196 {
2197         Mpi25IeeeSgeChain64_t *sgel = paddr;
2198
2199         sgel->Flags = flags;
2200         sgel->NextChainOffset = chain_offset;
2201         sgel->Length = cpu_to_le32(length);
2202         sgel->Address = cpu_to_le64(dma_addr);
2203 }
2204
2205 /**
2206  * _base_build_zero_len_sge_ieee - build zero length sg entry for IEEE format
2207  * @ioc: per adapter object
2208  * @paddr: virtual address for SGE
2209  *
2210  * Create a zero length scatter gather entry to insure the IOCs hardware has
2211  * something to use if the target device goes brain dead and tries
2212  * to send data even when none is asked for.
2213  */
2214 static void
2215 _base_build_zero_len_sge_ieee(struct MPT3SAS_ADAPTER *ioc, void *paddr)
2216 {
2217         u8 sgl_flags = (MPI2_IEEE_SGE_FLAGS_SIMPLE_ELEMENT |
2218                 MPI2_IEEE_SGE_FLAGS_SYSTEM_ADDR |
2219                 MPI25_IEEE_SGE_FLAGS_END_OF_LIST);
2220
2221         _base_add_sg_single_ieee(paddr, sgl_flags, 0, 0, -1);
2222 }
2223
2224 /**
2225  * _base_build_sg_scmd - main sg creation routine
2226  *              pcie_device is unused here!
2227  * @ioc: per adapter object
2228  * @scmd: scsi command
2229  * @smid: system request message index
2230  * @unused: unused pcie_device pointer
2231  * Context: none.
2232  *
2233  * The main routine that builds scatter gather table from a given
2234  * scsi request sent via the .queuecommand main handler.
2235  *
2236  * Return: 0 success, anything else error
2237  */
2238 static int
2239 _base_build_sg_scmd(struct MPT3SAS_ADAPTER *ioc,
2240         struct scsi_cmnd *scmd, u16 smid, struct _pcie_device *unused)
2241 {
2242         Mpi2SCSIIORequest_t *mpi_request;
2243         dma_addr_t chain_dma;
2244         struct scatterlist *sg_scmd;
2245         void *sg_local, *chain;
2246         u32 chain_offset;
2247         u32 chain_length;
2248         u32 chain_flags;
2249         int sges_left;
2250         u32 sges_in_segment;
2251         u32 sgl_flags;
2252         u32 sgl_flags_last_element;
2253         u32 sgl_flags_end_buffer;
2254         struct chain_tracker *chain_req;
2255
2256         mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);
2257
2258         /* init scatter gather flags */
2259         sgl_flags = MPI2_SGE_FLAGS_SIMPLE_ELEMENT;
2260         if (scmd->sc_data_direction == DMA_TO_DEVICE)
2261                 sgl_flags |= MPI2_SGE_FLAGS_HOST_TO_IOC;
2262         sgl_flags_last_element = (sgl_flags | MPI2_SGE_FLAGS_LAST_ELEMENT)
2263             << MPI2_SGE_FLAGS_SHIFT;
2264         sgl_flags_end_buffer = (sgl_flags | MPI2_SGE_FLAGS_LAST_ELEMENT |
2265             MPI2_SGE_FLAGS_END_OF_BUFFER | MPI2_SGE_FLAGS_END_OF_LIST)
2266             << MPI2_SGE_FLAGS_SHIFT;
2267         sgl_flags = sgl_flags << MPI2_SGE_FLAGS_SHIFT;
2268
2269         sg_scmd = scsi_sglist(scmd);
2270         sges_left = scsi_dma_map(scmd);
2271         if (sges_left < 0) {
2272                 sdev_printk(KERN_ERR, scmd->device,
2273                  "pci_map_sg failed: request for %d bytes!\n",
2274                  scsi_bufflen(scmd));
2275                 return -ENOMEM;
2276         }
2277
2278         sg_local = &mpi_request->SGL;
2279         sges_in_segment = ioc->max_sges_in_main_message;
2280         if (sges_left <= sges_in_segment)
2281                 goto fill_in_last_segment;
2282
2283         mpi_request->ChainOffset = (offsetof(Mpi2SCSIIORequest_t, SGL) +
2284             (sges_in_segment * ioc->sge_size))/4;
2285
2286         /* fill in main message segment when there is a chain following */
2287         while (sges_in_segment) {
2288                 if (sges_in_segment == 1)
2289                         ioc->base_add_sg_single(sg_local,
2290                             sgl_flags_last_element | sg_dma_len(sg_scmd),
2291                             sg_dma_address(sg_scmd));
2292                 else
2293                         ioc->base_add_sg_single(sg_local, sgl_flags |
2294                             sg_dma_len(sg_scmd), sg_dma_address(sg_scmd));
2295                 sg_scmd = sg_next(sg_scmd);
2296                 sg_local += ioc->sge_size;
2297                 sges_left--;
2298                 sges_in_segment--;
2299         }
2300
2301         /* initializing the chain flags and pointers */
2302         chain_flags = MPI2_SGE_FLAGS_CHAIN_ELEMENT << MPI2_SGE_FLAGS_SHIFT;
2303         chain_req = _base_get_chain_buffer_tracker(ioc, scmd);
2304         if (!chain_req)
2305                 return -1;
2306         chain = chain_req->chain_buffer;
2307         chain_dma = chain_req->chain_buffer_dma;
2308         do {
2309                 sges_in_segment = (sges_left <=
2310                     ioc->max_sges_in_chain_message) ? sges_left :
2311                     ioc->max_sges_in_chain_message;
2312                 chain_offset = (sges_left == sges_in_segment) ?
2313                     0 : (sges_in_segment * ioc->sge_size)/4;
2314                 chain_length = sges_in_segment * ioc->sge_size;
2315                 if (chain_offset) {
2316                         chain_offset = chain_offset <<
2317                             MPI2_SGE_CHAIN_OFFSET_SHIFT;
2318                         chain_length += ioc->sge_size;
2319                 }
2320                 ioc->base_add_sg_single(sg_local, chain_flags | chain_offset |
2321                     chain_length, chain_dma);
2322                 sg_local = chain;
2323                 if (!chain_offset)
2324                         goto fill_in_last_segment;
2325
2326                 /* fill in chain segments */
2327                 while (sges_in_segment) {
2328                         if (sges_in_segment == 1)
2329                                 ioc->base_add_sg_single(sg_local,
2330                                     sgl_flags_last_element |
2331                                     sg_dma_len(sg_scmd),
2332                                     sg_dma_address(sg_scmd));
2333                         else
2334                                 ioc->base_add_sg_single(sg_local, sgl_flags |
2335                                     sg_dma_len(sg_scmd),
2336                                     sg_dma_address(sg_scmd));
2337                         sg_scmd = sg_next(sg_scmd);
2338                         sg_local += ioc->sge_size;
2339                         sges_left--;
2340                         sges_in_segment--;
2341                 }
2342
2343                 chain_req = _base_get_chain_buffer_tracker(ioc, scmd);
2344                 if (!chain_req)
2345                         return -1;
2346                 chain = chain_req->chain_buffer;
2347                 chain_dma = chain_req->chain_buffer_dma;
2348         } while (1);
2349
2350
2351  fill_in_last_segment:
2352
2353         /* fill the last segment */
2354         while (sges_left) {
2355                 if (sges_left == 1)
2356                         ioc->base_add_sg_single(sg_local, sgl_flags_end_buffer |
2357                             sg_dma_len(sg_scmd), sg_dma_address(sg_scmd));
2358                 else
2359                         ioc->base_add_sg_single(sg_local, sgl_flags |
2360                             sg_dma_len(sg_scmd), sg_dma_address(sg_scmd));
2361                 sg_scmd = sg_next(sg_scmd);
2362                 sg_local += ioc->sge_size;
2363                 sges_left--;
2364         }
2365
2366         return 0;
2367 }
2368
2369 /**
2370  * _base_build_sg_scmd_ieee - main sg creation routine for IEEE format
2371  * @ioc: per adapter object
2372  * @scmd: scsi command
2373  * @smid: system request message index
2374  * @pcie_device: Pointer to pcie_device. If set, the pcie native sgl will be
2375  * constructed on need.
2376  * Context: none.
2377  *
2378  * The main routine that builds scatter gather table from a given
2379  * scsi request sent via the .queuecommand main handler.
2380  *
2381  * Return: 0 success, anything else error
2382  */
2383 static int
2384 _base_build_sg_scmd_ieee(struct MPT3SAS_ADAPTER *ioc,
2385         struct scsi_cmnd *scmd, u16 smid, struct _pcie_device *pcie_device)
2386 {
2387         Mpi25SCSIIORequest_t *mpi_request;
2388         dma_addr_t chain_dma;
2389         struct scatterlist *sg_scmd;
2390         void *sg_local, *chain;
2391         u32 chain_offset;
2392         u32 chain_length;
2393         int sges_left;
2394         u32 sges_in_segment;
2395         u8 simple_sgl_flags;
2396         u8 simple_sgl_flags_last;
2397         u8 chain_sgl_flags;
2398         struct chain_tracker *chain_req;
2399
2400         mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);
2401
2402         /* init scatter gather flags */
2403         simple_sgl_flags = MPI2_IEEE_SGE_FLAGS_SIMPLE_ELEMENT |
2404             MPI2_IEEE_SGE_FLAGS_SYSTEM_ADDR;
2405         simple_sgl_flags_last = simple_sgl_flags |
2406             MPI25_IEEE_SGE_FLAGS_END_OF_LIST;
2407         chain_sgl_flags = MPI2_IEEE_SGE_FLAGS_CHAIN_ELEMENT |
2408             MPI2_IEEE_SGE_FLAGS_SYSTEM_ADDR;
2409
2410         /* Check if we need to build a native SG list. */
2411         if ((pcie_device) && (_base_check_pcie_native_sgl(ioc, mpi_request,
2412                         smid, scmd, pcie_device) == 0)) {
2413                 /* We built a native SG list, just return. */
2414                 return 0;
2415         }
2416
2417         sg_scmd = scsi_sglist(scmd);
2418         sges_left = scsi_dma_map(scmd);
2419         if (sges_left < 0) {
2420                 sdev_printk(KERN_ERR, scmd->device,
2421                         "pci_map_sg failed: request for %d bytes!\n",
2422                         scsi_bufflen(scmd));
2423                 return -ENOMEM;
2424         }
2425
2426         sg_local = &mpi_request->SGL;
2427         sges_in_segment = (ioc->request_sz -
2428                    offsetof(Mpi25SCSIIORequest_t, SGL))/ioc->sge_size_ieee;
2429         if (sges_left <= sges_in_segment)
2430                 goto fill_in_last_segment;
2431
2432         mpi_request->ChainOffset = (sges_in_segment - 1 /* chain element */) +
2433             (offsetof(Mpi25SCSIIORequest_t, SGL)/ioc->sge_size_ieee);
2434
2435         /* fill in main message segment when there is a chain following */
2436         while (sges_in_segment > 1) {
2437                 _base_add_sg_single_ieee(sg_local, simple_sgl_flags, 0,
2438                     sg_dma_len(sg_scmd), sg_dma_address(sg_scmd));
2439                 sg_scmd = sg_next(sg_scmd);
2440                 sg_local += ioc->sge_size_ieee;
2441                 sges_left--;
2442                 sges_in_segment--;
2443         }
2444
2445         /* initializing the pointers */
2446         chain_req = _base_get_chain_buffer_tracker(ioc, scmd);
2447         if (!chain_req)
2448                 return -1;
2449         chain = chain_req->chain_buffer;
2450         chain_dma = chain_req->chain_buffer_dma;
2451         do {
2452                 sges_in_segment = (sges_left <=
2453                     ioc->max_sges_in_chain_message) ? sges_left :
2454                     ioc->max_sges_in_chain_message;
2455                 chain_offset = (sges_left == sges_in_segment) ?
2456                     0 : sges_in_segment;
2457                 chain_length = sges_in_segment * ioc->sge_size_ieee;
2458                 if (chain_offset)
2459                         chain_length += ioc->sge_size_ieee;
2460                 _base_add_sg_single_ieee(sg_local, chain_sgl_flags,
2461                     chain_offset, chain_length, chain_dma);
2462
2463                 sg_local = chain;
2464                 if (!chain_offset)
2465                         goto fill_in_last_segment;
2466
2467                 /* fill in chain segments */
2468                 while (sges_in_segment) {
2469                         _base_add_sg_single_ieee(sg_local, simple_sgl_flags, 0,
2470                             sg_dma_len(sg_scmd), sg_dma_address(sg_scmd));
2471                         sg_scmd = sg_next(sg_scmd);
2472                         sg_local += ioc->sge_size_ieee;
2473                         sges_left--;
2474                         sges_in_segment--;
2475                 }
2476
2477                 chain_req = _base_get_chain_buffer_tracker(ioc, scmd);
2478                 if (!chain_req)
2479                         return -1;
2480                 chain = chain_req->chain_buffer;
2481                 chain_dma = chain_req->chain_buffer_dma;
2482         } while (1);
2483
2484
2485  fill_in_last_segment:
2486
2487         /* fill the last segment */
2488         while (sges_left > 0) {
2489                 if (sges_left == 1)
2490                         _base_add_sg_single_ieee(sg_local,
2491                             simple_sgl_flags_last, 0, sg_dma_len(sg_scmd),
2492                             sg_dma_address(sg_scmd));
2493                 else
2494                         _base_add_sg_single_ieee(sg_local, simple_sgl_flags, 0,
2495                             sg_dma_len(sg_scmd), sg_dma_address(sg_scmd));
2496                 sg_scmd = sg_next(sg_scmd);
2497                 sg_local += ioc->sge_size_ieee;
2498                 sges_left--;
2499         }
2500
2501         return 0;
2502 }
2503
2504 /**
2505  * _base_build_sg_ieee - build generic sg for IEEE format
2506  * @ioc: per adapter object
2507  * @psge: virtual address for SGE
2508  * @data_out_dma: physical address for WRITES
2509  * @data_out_sz: data xfer size for WRITES
2510  * @data_in_dma: physical address for READS
2511  * @data_in_sz: data xfer size for READS
2512  */
2513 static void
2514 _base_build_sg_ieee(struct MPT3SAS_ADAPTER *ioc, void *psge,
2515         dma_addr_t data_out_dma, size_t data_out_sz, dma_addr_t data_in_dma,
2516         size_t data_in_sz)
2517 {
2518         u8 sgl_flags;
2519
2520         if (!data_out_sz && !data_in_sz) {
2521                 _base_build_zero_len_sge_ieee(ioc, psge);
2522                 return;
2523         }
2524
2525         if (data_out_sz && data_in_sz) {
2526                 /* WRITE sgel first */
2527                 sgl_flags = MPI2_IEEE_SGE_FLAGS_SIMPLE_ELEMENT |
2528                     MPI2_IEEE_SGE_FLAGS_SYSTEM_ADDR;
2529                 _base_add_sg_single_ieee(psge, sgl_flags, 0, data_out_sz,
2530                     data_out_dma);
2531
2532                 /* incr sgel */
2533                 psge += ioc->sge_size_ieee;
2534
2535                 /* READ sgel last */
2536                 sgl_flags |= MPI25_IEEE_SGE_FLAGS_END_OF_LIST;
2537                 _base_add_sg_single_ieee(psge, sgl_flags, 0, data_in_sz,
2538                     data_in_dma);
2539         } else if (data_out_sz) /* WRITE */ {
2540                 sgl_flags = MPI2_IEEE_SGE_FLAGS_SIMPLE_ELEMENT |
2541                     MPI25_IEEE_SGE_FLAGS_END_OF_LIST |
2542                     MPI2_IEEE_SGE_FLAGS_SYSTEM_ADDR;
2543                 _base_add_sg_single_ieee(psge, sgl_flags, 0, data_out_sz,
2544                     data_out_dma);
2545         } else if (data_in_sz) /* READ */ {
2546                 sgl_flags = MPI2_IEEE_SGE_FLAGS_SIMPLE_ELEMENT |
2547                     MPI25_IEEE_SGE_FLAGS_END_OF_LIST |
2548                     MPI2_IEEE_SGE_FLAGS_SYSTEM_ADDR;
2549                 _base_add_sg_single_ieee(psge, sgl_flags, 0, data_in_sz,
2550                     data_in_dma);
2551         }
2552 }
2553
2554 #define convert_to_kb(x) ((x) << (PAGE_SHIFT - 10))
2555
2556 /**
2557  * _base_config_dma_addressing - set dma addressing
2558  * @ioc: per adapter object
2559  * @pdev: PCI device struct
2560  *
2561  * Return: 0 for success, non-zero for failure.
2562  */
2563 static int
2564 _base_config_dma_addressing(struct MPT3SAS_ADAPTER *ioc, struct pci_dev *pdev)
2565 {
2566         struct sysinfo s;
2567         u64 consistent_dma_mask;
2568         /* Set 63 bit DMA mask for all SAS3 and SAS35 controllers */
2569         int dma_mask = (ioc->hba_mpi_version_belonged > MPI2_VERSION) ? 63 : 64;
2570
2571         if (ioc->is_mcpu_endpoint)
2572                 goto try_32bit;
2573
2574         if (ioc->dma_mask)
2575                 consistent_dma_mask = DMA_BIT_MASK(dma_mask);
2576         else
2577                 consistent_dma_mask = DMA_BIT_MASK(32);
2578
2579         if (sizeof(dma_addr_t) > 4) {
2580                 const uint64_t required_mask =
2581                     dma_get_required_mask(&pdev->dev);
2582                 if ((required_mask > DMA_BIT_MASK(32)) &&
2583                     !pci_set_dma_mask(pdev, DMA_BIT_MASK(dma_mask)) &&
2584                     !pci_set_consistent_dma_mask(pdev, consistent_dma_mask)) {
2585                         ioc->base_add_sg_single = &_base_add_sg_single_64;
2586                         ioc->sge_size = sizeof(Mpi2SGESimple64_t);
2587                         ioc->dma_mask = dma_mask;
2588                         goto out;
2589                 }
2590         }
2591
2592  try_32bit:
2593         if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(32))
2594             && !pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32))) {
2595                 ioc->base_add_sg_single = &_base_add_sg_single_32;
2596                 ioc->sge_size = sizeof(Mpi2SGESimple32_t);
2597                 ioc->dma_mask = 32;
2598         } else
2599                 return -ENODEV;
2600
2601  out:
2602         si_meminfo(&s);
2603         pr_info(MPT3SAS_FMT
2604                 "%d BIT PCI BUS DMA ADDRESSING SUPPORTED, total mem (%ld kB)\n",
2605                 ioc->name, ioc->dma_mask, convert_to_kb(s.totalram));
2606
2607         return 0;
2608 }
2609
2610 static int
2611 _base_change_consistent_dma_mask(struct MPT3SAS_ADAPTER *ioc,
2612                                       struct pci_dev *pdev)
2613 {
2614         if (pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(ioc->dma_mask))) {
2615                 if (pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)))
2616                         return -ENODEV;
2617         }
2618         return 0;
2619 }
2620
2621 /**
2622  * _base_check_enable_msix - checks MSIX capabable.
2623  * @ioc: per adapter object
2624  *
2625  * Check to see if card is capable of MSIX, and set number
2626  * of available msix vectors
2627  */
2628 static int
2629 _base_check_enable_msix(struct MPT3SAS_ADAPTER *ioc)
2630 {
2631         int base;
2632         u16 message_control;
2633
2634         /* Check whether controller SAS2008 B0 controller,
2635          * if it is SAS2008 B0 controller use IO-APIC instead of MSIX
2636          */
2637         if (ioc->pdev->device == MPI2_MFGPAGE_DEVID_SAS2008 &&
2638             ioc->pdev->revision == SAS2_PCI_DEVICE_B0_REVISION) {
2639                 return -EINVAL;
2640         }
2641
2642         base = pci_find_capability(ioc->pdev, PCI_CAP_ID_MSIX);
2643         if (!base) {
2644                 dfailprintk(ioc, pr_info(MPT3SAS_FMT "msix not supported\n",
2645                         ioc->name));
2646                 return -EINVAL;
2647         }
2648
2649         /* get msix vector count */
2650         /* NUMA_IO not supported for older controllers */
2651         if (ioc->pdev->device == MPI2_MFGPAGE_DEVID_SAS2004 ||
2652             ioc->pdev->device == MPI2_MFGPAGE_DEVID_SAS2008 ||
2653             ioc->pdev->device == MPI2_MFGPAGE_DEVID_SAS2108_1 ||
2654             ioc->pdev->device == MPI2_MFGPAGE_DEVID_SAS2108_2 ||
2655             ioc->pdev->device == MPI2_MFGPAGE_DEVID_SAS2108_3 ||
2656             ioc->pdev->device == MPI2_MFGPAGE_DEVID_SAS2116_1 ||
2657             ioc->pdev->device == MPI2_MFGPAGE_DEVID_SAS2116_2)
2658                 ioc->msix_vector_count = 1;
2659         else {
2660                 pci_read_config_word(ioc->pdev, base + 2, &message_control);
2661                 ioc->msix_vector_count = (message_control & 0x3FF) + 1;
2662         }
2663         dinitprintk(ioc, pr_info(MPT3SAS_FMT
2664                 "msix is supported, vector_count(%d)\n",
2665                 ioc->name, ioc->msix_vector_count));
2666         return 0;
2667 }
2668
2669 /**
2670  * _base_free_irq - free irq
2671  * @ioc: per adapter object
2672  *
2673  * Freeing respective reply_queue from the list.
2674  */
2675 static void
2676 _base_free_irq(struct MPT3SAS_ADAPTER *ioc)
2677 {
2678         struct adapter_reply_queue *reply_q, *next;
2679
2680         if (list_empty(&ioc->reply_queue_list))
2681                 return;
2682
2683         list_for_each_entry_safe(reply_q, next, &ioc->reply_queue_list, list) {
2684                 list_del(&reply_q->list);
2685                 free_irq(pci_irq_vector(ioc->pdev, reply_q->msix_index),
2686                          reply_q);
2687                 kfree(reply_q);
2688         }
2689 }
2690
2691 /**
2692  * _base_request_irq - request irq
2693  * @ioc: per adapter object
2694  * @index: msix index into vector table
2695  *
2696  * Inserting respective reply_queue into the list.
2697  */
2698 static int
2699 _base_request_irq(struct MPT3SAS_ADAPTER *ioc, u8 index)
2700 {
2701         struct pci_dev *pdev = ioc->pdev;
2702         struct adapter_reply_queue *reply_q;
2703         int r;
2704
2705         reply_q =  kzalloc(sizeof(struct adapter_reply_queue), GFP_KERNEL);
2706         if (!reply_q) {
2707                 pr_err(MPT3SAS_FMT "unable to allocate memory %d!\n",
2708                     ioc->name, (int)sizeof(struct adapter_reply_queue));
2709                 return -ENOMEM;
2710         }
2711         reply_q->ioc = ioc;
2712         reply_q->msix_index = index;
2713
2714         atomic_set(&reply_q->busy, 0);
2715         if (ioc->msix_enable)
2716                 snprintf(reply_q->name, MPT_NAME_LENGTH, "%s%d-msix%d",
2717                     ioc->driver_name, ioc->id, index);
2718         else
2719                 snprintf(reply_q->name, MPT_NAME_LENGTH, "%s%d",
2720                     ioc->driver_name, ioc->id);
2721         r = request_irq(pci_irq_vector(pdev, index), _base_interrupt,
2722                         IRQF_SHARED, reply_q->name, reply_q);
2723         if (r) {
2724                 pr_err(MPT3SAS_FMT "unable to allocate interrupt %d!\n",
2725                        reply_q->name, pci_irq_vector(pdev, index));
2726                 kfree(reply_q);
2727                 return -EBUSY;
2728         }
2729
2730         INIT_LIST_HEAD(&reply_q->list);
2731         list_add_tail(&reply_q->list, &ioc->reply_queue_list);
2732         return 0;
2733 }
2734
2735 /**
2736  * _base_assign_reply_queues - assigning msix index for each cpu
2737  * @ioc: per adapter object
2738  *
2739  * The enduser would need to set the affinity via /proc/irq/#/smp_affinity
2740  *
2741  * It would nice if we could call irq_set_affinity, however it is not
2742  * an exported symbol
2743  */
2744 static void
2745 _base_assign_reply_queues(struct MPT3SAS_ADAPTER *ioc)
2746 {
2747         unsigned int cpu, nr_cpus, nr_msix, index = 0;
2748         struct adapter_reply_queue *reply_q;
2749
2750         if (!_base_is_controller_msix_enabled(ioc))
2751                 return;
2752
2753         memset(ioc->cpu_msix_table, 0, ioc->cpu_msix_table_sz);
2754
2755         nr_cpus = num_online_cpus();
2756         nr_msix = ioc->reply_queue_count = min(ioc->reply_queue_count,
2757                                                ioc->facts.MaxMSIxVectors);
2758         if (!nr_msix)
2759                 return;
2760
2761         if (smp_affinity_enable) {
2762                 list_for_each_entry(reply_q, &ioc->reply_queue_list, list) {
2763                         const cpumask_t *mask = pci_irq_get_affinity(ioc->pdev,
2764                                                         reply_q->msix_index);
2765                         if (!mask) {
2766                                 pr_warn(MPT3SAS_FMT "no affinity for msi %x\n",
2767                                         ioc->name, reply_q->msix_index);
2768                                 continue;
2769                         }
2770
2771                         for_each_cpu_and(cpu, mask, cpu_online_mask) {
2772                                 if (cpu >= ioc->cpu_msix_table_sz)
2773                                         break;
2774                                 ioc->cpu_msix_table[cpu] = reply_q->msix_index;
2775                         }
2776                 }
2777                 return;
2778         }
2779         cpu = cpumask_first(cpu_online_mask);
2780
2781         list_for_each_entry(reply_q, &ioc->reply_queue_list, list) {
2782
2783                 unsigned int i, group = nr_cpus / nr_msix;
2784
2785                 if (cpu >= nr_cpus)
2786                         break;
2787
2788                 if (index < nr_cpus % nr_msix)
2789                         group++;
2790
2791                 for (i = 0 ; i < group ; i++) {
2792                         ioc->cpu_msix_table[cpu] = reply_q->msix_index;
2793                         cpu = cpumask_next(cpu, cpu_online_mask);
2794                 }
2795                 index++;
2796         }
2797 }
2798
2799 /**
2800  * _base_disable_msix - disables msix
2801  * @ioc: per adapter object
2802  *
2803  */
2804 static void
2805 _base_disable_msix(struct MPT3SAS_ADAPTER *ioc)
2806 {
2807         if (!ioc->msix_enable)
2808                 return;
2809         pci_disable_msix(ioc->pdev);
2810         ioc->msix_enable = 0;
2811 }
2812
2813 /**
2814  * _base_enable_msix - enables msix, failback to io_apic
2815  * @ioc: per adapter object
2816  *
2817  */
2818 static int
2819 _base_enable_msix(struct MPT3SAS_ADAPTER *ioc)
2820 {
2821         int r;
2822         int i, local_max_msix_vectors;
2823         u8 try_msix = 0;
2824         unsigned int irq_flags = PCI_IRQ_MSIX;
2825
2826         if (msix_disable == -1 || msix_disable == 0)
2827                 try_msix = 1;
2828
2829         if (!try_msix)
2830                 goto try_ioapic;
2831
2832         if (_base_check_enable_msix(ioc) != 0)
2833                 goto try_ioapic;
2834
2835         ioc->reply_queue_count = min_t(int, ioc->cpu_count,
2836                 ioc->msix_vector_count);
2837
2838         printk(MPT3SAS_FMT "MSI-X vectors supported: %d, no of cores"
2839           ": %d, max_msix_vectors: %d\n", ioc->name, ioc->msix_vector_count,
2840           ioc->cpu_count, max_msix_vectors);
2841
2842         if (!ioc->rdpq_array_enable && max_msix_vectors == -1)
2843                 local_max_msix_vectors = (reset_devices) ? 1 : 8;
2844         else
2845                 local_max_msix_vectors = max_msix_vectors;
2846
2847         if (local_max_msix_vectors > 0)
2848                 ioc->reply_queue_count = min_t(int, local_max_msix_vectors,
2849                         ioc->reply_queue_count);
2850         else if (local_max_msix_vectors == 0)
2851                 goto try_ioapic;
2852
2853         if (ioc->msix_vector_count < ioc->cpu_count)
2854                 smp_affinity_enable = 0;
2855
2856         if (smp_affinity_enable)
2857                 irq_flags |= PCI_IRQ_AFFINITY;
2858
2859         r = pci_alloc_irq_vectors(ioc->pdev, 1, ioc->reply_queue_count,
2860                                   irq_flags);
2861         if (r < 0) {
2862                 dfailprintk(ioc, pr_info(MPT3SAS_FMT
2863                         "pci_alloc_irq_vectors failed (r=%d) !!!\n",
2864                         ioc->name, r));
2865                 goto try_ioapic;
2866         }
2867
2868         ioc->msix_enable = 1;
2869         ioc->reply_queue_count = r;
2870         for (i = 0; i < ioc->reply_queue_count; i++) {
2871                 r = _base_request_irq(ioc, i);
2872                 if (r) {
2873                         _base_free_irq(ioc);
2874                         _base_disable_msix(ioc);
2875                         goto try_ioapic;
2876                 }
2877         }
2878
2879         return 0;
2880
2881 /* failback to io_apic interrupt routing */
2882  try_ioapic:
2883
2884         ioc->reply_queue_count = 1;
2885         r = pci_alloc_irq_vectors(ioc->pdev, 1, 1, PCI_IRQ_LEGACY);
2886         if (r < 0) {
2887                 dfailprintk(ioc, pr_info(MPT3SAS_FMT
2888                         "pci_alloc_irq_vector(legacy) failed (r=%d) !!!\n",
2889                         ioc->name, r));
2890         } else
2891                 r = _base_request_irq(ioc, 0);
2892
2893         return r;
2894 }
2895
2896 /**
2897  * mpt3sas_base_unmap_resources - free controller resources
2898  * @ioc: per adapter object
2899  */
2900 static void
2901 mpt3sas_base_unmap_resources(struct MPT3SAS_ADAPTER *ioc)
2902 {
2903         struct pci_dev *pdev = ioc->pdev;
2904
2905         dexitprintk(ioc, printk(MPT3SAS_FMT "%s\n",
2906                 ioc->name, __func__));
2907
2908         _base_free_irq(ioc);
2909         _base_disable_msix(ioc);
2910
2911         kfree(ioc->replyPostRegisterIndex);
2912         ioc->replyPostRegisterIndex = NULL;
2913
2914
2915         if (ioc->chip_phys) {
2916                 iounmap(ioc->chip);
2917                 ioc->chip_phys = 0;
2918         }
2919
2920         if (pci_is_enabled(pdev)) {
2921                 pci_release_selected_regions(ioc->pdev, ioc->bars);
2922                 pci_disable_pcie_error_reporting(pdev);
2923                 pci_disable_device(pdev);
2924         }
2925 }
2926
2927 /**
2928  * mpt3sas_base_map_resources - map in controller resources (io/irq/memap)
2929  * @ioc: per adapter object
2930  *
2931  * Return: 0 for success, non-zero for failure.
2932  */
2933 int
2934 mpt3sas_base_map_resources(struct MPT3SAS_ADAPTER *ioc)
2935 {
2936         struct pci_dev *pdev = ioc->pdev;
2937         u32 memap_sz;
2938         u32 pio_sz;
2939         int i, r = 0;
2940         u64 pio_chip = 0;
2941         phys_addr_t chip_phys = 0;
2942         struct adapter_reply_queue *reply_q;
2943
2944         dinitprintk(ioc, pr_info(MPT3SAS_FMT "%s\n",
2945             ioc->name, __func__));
2946
2947         ioc->bars = pci_select_bars(pdev, IORESOURCE_MEM);
2948         if (pci_enable_device_mem(pdev)) {
2949                 pr_warn(MPT3SAS_FMT "pci_enable_device_mem: failed\n",
2950                         ioc->name);
2951                 ioc->bars = 0;
2952                 return -ENODEV;
2953         }
2954
2955
2956         if (pci_request_selected_regions(pdev, ioc->bars,
2957             ioc->driver_name)) {
2958                 pr_warn(MPT3SAS_FMT "pci_request_selected_regions: failed\n",
2959                         ioc->name);
2960                 ioc->bars = 0;
2961                 r = -ENODEV;
2962                 goto out_fail;
2963         }
2964
2965 /* AER (Advanced Error Reporting) hooks */
2966         pci_enable_pcie_error_reporting(pdev);
2967
2968         pci_set_master(pdev);
2969
2970
2971         if (_base_config_dma_addressing(ioc, pdev) != 0) {
2972                 pr_warn(MPT3SAS_FMT "no suitable DMA mask for %s\n",
2973                     ioc->name, pci_name(pdev));
2974                 r = -ENODEV;
2975                 goto out_fail;
2976         }
2977
2978         for (i = 0, memap_sz = 0, pio_sz = 0; (i < DEVICE_COUNT_RESOURCE) &&
2979              (!memap_sz || !pio_sz); i++) {
2980                 if (pci_resource_flags(pdev, i) & IORESOURCE_IO) {
2981                         if (pio_sz)
2982                                 continue;
2983                         pio_chip = (u64)pci_resource_start(pdev, i);
2984                         pio_sz = pci_resource_len(pdev, i);
2985                 } else if (pci_resource_flags(pdev, i) & IORESOURCE_MEM) {
2986                         if (memap_sz)
2987                                 continue;
2988                         ioc->chip_phys = pci_resource_start(pdev, i);
2989                         chip_phys = ioc->chip_phys;
2990                         memap_sz = pci_resource_len(pdev, i);
2991                         ioc->chip = ioremap(ioc->chip_phys, memap_sz);
2992                 }
2993         }
2994
2995         if (ioc->chip == NULL) {
2996                 pr_err(MPT3SAS_FMT "unable to map adapter memory! "
2997                         " or resource not found\n", ioc->name);
2998                 r = -EINVAL;
2999                 goto out_fail;
3000         }
3001
3002         _base_mask_interrupts(ioc);
3003
3004         r = _base_get_ioc_facts(ioc);
3005         if (r)
3006                 goto out_fail;
3007
3008         if (!ioc->rdpq_array_enable_assigned) {
3009                 ioc->rdpq_array_enable = ioc->rdpq_array_capable;
3010                 ioc->rdpq_array_enable_assigned = 1;
3011         }
3012
3013         r = _base_enable_msix(ioc);
3014         if (r)
3015                 goto out_fail;
3016
3017         /* Use the Combined reply queue feature only for SAS3 C0 & higher
3018          * revision HBAs and also only when reply queue count is greater than 8
3019          */
3020         if (ioc->combined_reply_queue) {
3021                 /* Determine the Supplemental Reply Post Host Index Registers
3022                  * Addresse. Supplemental Reply Post Host Index Registers
3023                  * starts at offset MPI25_SUP_REPLY_POST_HOST_INDEX_OFFSET and
3024                  * each register is at offset bytes of
3025                  * MPT3_SUP_REPLY_POST_HOST_INDEX_REG_OFFSET from previous one.
3026                  */
3027                 ioc->replyPostRegisterIndex = kcalloc(
3028                      ioc->combined_reply_index_count,
3029                      sizeof(resource_size_t *), GFP_KERNEL);
3030                 if (!ioc->replyPostRegisterIndex) {
3031                         dfailprintk(ioc, printk(MPT3SAS_FMT
3032                         "allocation for reply Post Register Index failed!!!\n",
3033                                                                    ioc->name));
3034                         r = -ENOMEM;
3035                         goto out_fail;
3036                 }
3037
3038                 for (i = 0; i < ioc->combined_reply_index_count; i++) {
3039                         ioc->replyPostRegisterIndex[i] = (resource_size_t *)
3040                              ((u8 __force *)&ioc->chip->Doorbell +
3041                              MPI25_SUP_REPLY_POST_HOST_INDEX_OFFSET +
3042                              (i * MPT3_SUP_REPLY_POST_HOST_INDEX_REG_OFFSET));
3043                 }
3044         }
3045
3046         if (ioc->is_warpdrive) {
3047                 ioc->reply_post_host_index[0] = (resource_size_t __iomem *)
3048                     &ioc->chip->ReplyPostHostIndex;
3049
3050                 for (i = 1; i < ioc->cpu_msix_table_sz; i++)
3051                         ioc->reply_post_host_index[i] =
3052                         (resource_size_t __iomem *)
3053                         ((u8 __iomem *)&ioc->chip->Doorbell + (0x4000 + ((i - 1)
3054                         * 4)));
3055         }
3056
3057         list_for_each_entry(reply_q, &ioc->reply_queue_list, list)
3058                 pr_info(MPT3SAS_FMT "%s: IRQ %d\n",
3059                     reply_q->name,  ((ioc->msix_enable) ? "PCI-MSI-X enabled" :
3060                     "IO-APIC enabled"),
3061                     pci_irq_vector(ioc->pdev, reply_q->msix_index));
3062
3063         pr_info(MPT3SAS_FMT "iomem(%pap), mapped(0x%p), size(%d)\n",
3064             ioc->name, &chip_phys, ioc->chip, memap_sz);
3065         pr_info(MPT3SAS_FMT "ioport(0x%016llx), size(%d)\n",
3066             ioc->name, (unsigned long long)pio_chip, pio_sz);
3067
3068         /* Save PCI configuration state for recovery from PCI AER/EEH errors */
3069         pci_save_state(pdev);
3070         return 0;
3071
3072  out_fail:
3073         mpt3sas_base_unmap_resources(ioc);
3074         return r;
3075 }
3076
3077 /**
3078  * mpt3sas_base_get_msg_frame - obtain request mf pointer
3079  * @ioc: per adapter object
3080  * @smid: system request message index(smid zero is invalid)
3081  *
3082  * Return: virt pointer to message frame.
3083  */
3084 void *
3085 mpt3sas_base_get_msg_frame(struct MPT3SAS_ADAPTER *ioc, u16 smid)
3086 {
3087         return (void *)(ioc->request + (smid * ioc->request_sz));
3088 }
3089
3090 /**
3091  * mpt3sas_base_get_sense_buffer - obtain a sense buffer virt addr
3092  * @ioc: per adapter object
3093  * @smid: system request message index
3094  *
3095  * Return: virt pointer to sense buffer.
3096  */
3097 void *
3098 mpt3sas_base_get_sense_buffer(struct MPT3SAS_ADAPTER *ioc, u16 smid)
3099 {
3100         return (void *)(ioc->sense + ((smid - 1) * SCSI_SENSE_BUFFERSIZE));
3101 }
3102
3103 /**
3104  * mpt3sas_base_get_sense_buffer_dma - obtain a sense buffer dma addr
3105  * @ioc: per adapter object
3106  * @smid: system request message index
3107  *
3108  * Return: phys pointer to the low 32bit address of the sense buffer.
3109  */
3110 __le32
3111 mpt3sas_base_get_sense_buffer_dma(struct MPT3SAS_ADAPTER *ioc, u16 smid)
3112 {
3113         return cpu_to_le32(ioc->sense_dma + ((smid - 1) *
3114             SCSI_SENSE_BUFFERSIZE));
3115 }
3116
3117 /**
3118  * mpt3sas_base_get_pcie_sgl - obtain a PCIe SGL virt addr
3119  * @ioc: per adapter object
3120  * @smid: system request message index
3121  *
3122  * Return: virt pointer to a PCIe SGL.
3123  */
3124 void *
3125 mpt3sas_base_get_pcie_sgl(struct MPT3SAS_ADAPTER *ioc, u16 smid)
3126 {
3127         return (void *)(ioc->pcie_sg_lookup[smid - 1].pcie_sgl);
3128 }
3129
3130 /**
3131  * mpt3sas_base_get_pcie_sgl_dma - obtain a PCIe SGL dma addr
3132  * @ioc: per adapter object
3133  * @smid: system request message index
3134  *
3135  * Return: phys pointer to the address of the PCIe buffer.
3136  */
3137 dma_addr_t
3138 mpt3sas_base_get_pcie_sgl_dma(struct MPT3SAS_ADAPTER *ioc, u16 smid)
3139 {
3140         return ioc->pcie_sg_lookup[smid - 1].pcie_sgl_dma;
3141 }
3142
3143 /**
3144  * mpt3sas_base_get_reply_virt_addr - obtain reply frames virt address
3145  * @ioc: per adapter object
3146  * @phys_addr: lower 32 physical addr of the reply
3147  *
3148  * Converts 32bit lower physical addr into a virt address.
3149  */
3150 void *
3151 mpt3sas_base_get_reply_virt_addr(struct MPT3SAS_ADAPTER *ioc, u32 phys_addr)
3152 {
3153         if (!phys_addr)
3154                 return NULL;
3155         return ioc->reply + (phys_addr - (u32)ioc->reply_dma);
3156 }
3157
3158 static inline u8
3159 _base_get_msix_index(struct MPT3SAS_ADAPTER *ioc)
3160 {
3161         return ioc->cpu_msix_table[raw_smp_processor_id()];
3162 }
3163
3164 /**
3165  * mpt3sas_base_get_smid - obtain a free smid from internal queue
3166  * @ioc: per adapter object
3167  * @cb_idx: callback index
3168  *
3169  * Return: smid (zero is invalid)
3170  */
3171 u16
3172 mpt3sas_base_get_smid(struct MPT3SAS_ADAPTER *ioc, u8 cb_idx)
3173 {
3174         unsigned long flags;
3175         struct request_tracker *request;
3176         u16 smid;
3177
3178         spin_lock_irqsave(&ioc->scsi_lookup_lock, flags);
3179         if (list_empty(&ioc->internal_free_list)) {
3180                 spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags);
3181                 pr_err(MPT3SAS_FMT "%s: smid not available\n",
3182                     ioc->name, __func__);
3183                 return 0;
3184         }
3185
3186         request = list_entry(ioc->internal_free_list.next,
3187             struct request_tracker, tracker_list);
3188         request->cb_idx = cb_idx;
3189         smid = request->smid;
3190         list_del(&request->tracker_list);
3191         spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags);
3192         return smid;
3193 }
3194
3195 /**
3196  * mpt3sas_base_get_smid_scsiio - obtain a free smid from scsiio queue
3197  * @ioc: per adapter object
3198  * @cb_idx: callback index
3199  * @scmd: pointer to scsi command object
3200  *
3201  * Return: smid (zero is invalid)
3202  */
3203 u16
3204 mpt3sas_base_get_smid_scsiio(struct MPT3SAS_ADAPTER *ioc, u8 cb_idx,
3205         struct scsi_cmnd *scmd)
3206 {
3207         struct scsiio_tracker *request = scsi_cmd_priv(scmd);
3208         unsigned int tag = scmd->request->tag;
3209         u16 smid;
3210
3211         smid = tag + 1;
3212         request->cb_idx = cb_idx;
3213         request->msix_io = _base_get_msix_index(ioc);
3214         request->smid = smid;
3215         INIT_LIST_HEAD(&request->chain_list);
3216         return smid;
3217 }
3218
3219 /**
3220  * mpt3sas_base_get_smid_hpr - obtain a free smid from hi-priority queue
3221  * @ioc: per adapter object
3222  * @cb_idx: callback index
3223  *
3224  * Return: smid (zero is invalid)
3225  */
3226 u16
3227 mpt3sas_base_get_smid_hpr(struct MPT3SAS_ADAPTER *ioc, u8 cb_idx)
3228 {
3229         unsigned long flags;
3230         struct request_tracker *request;
3231         u16 smid;
3232
3233         spin_lock_irqsave(&ioc->scsi_lookup_lock, flags);
3234         if (list_empty(&ioc->hpr_free_list)) {
3235                 spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags);
3236                 return 0;
3237         }
3238
3239         request = list_entry(ioc->hpr_free_list.next,
3240             struct request_tracker, tracker_list);
3241         request->cb_idx = cb_idx;
3242         smid = request->smid;
3243         list_del(&request->tracker_list);
3244         spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags);
3245         return smid;
3246 }
3247
3248 static void
3249 _base_recovery_check(struct MPT3SAS_ADAPTER *ioc)
3250 {
3251         /*
3252          * See _wait_for_commands_to_complete() call with regards to this code.
3253          */
3254         if (ioc->shost_recovery && ioc->pending_io_count) {
3255                 ioc->pending_io_count = scsi_host_busy(ioc->shost);
3256                 if (ioc->pending_io_count == 0)
3257                         wake_up(&ioc->reset_wq);
3258         }
3259 }
3260
3261 void mpt3sas_base_clear_st(struct MPT3SAS_ADAPTER *ioc,
3262                            struct scsiio_tracker *st)
3263 {
3264         if (WARN_ON(st->smid == 0))
3265                 return;
3266         st->cb_idx = 0xFF;
3267         st->direct_io = 0;
3268         atomic_set(&ioc->chain_lookup[st->smid - 1].chain_offset, 0);
3269         st->smid = 0;
3270 }
3271
3272 /**
3273  * mpt3sas_base_free_smid - put smid back on free_list
3274  * @ioc: per adapter object
3275  * @smid: system request message index
3276  */
3277 void
3278 mpt3sas_base_free_smid(struct MPT3SAS_ADAPTER *ioc, u16 smid)
3279 {
3280         unsigned long flags;
3281         int i;
3282
3283         if (smid < ioc->hi_priority_smid) {
3284                 struct scsiio_tracker *st;
3285                 void *request;
3286
3287                 st = _get_st_from_smid(ioc, smid);
3288                 if (!st) {
3289                         _base_recovery_check(ioc);
3290                         return;
3291                 }
3292
3293                 /* Clear MPI request frame */
3294                 request = mpt3sas_base_get_msg_frame(ioc, smid);
3295                 memset(request, 0, ioc->request_sz);
3296
3297                 mpt3sas_base_clear_st(ioc, st);
3298                 _base_recovery_check(ioc);
3299                 return;
3300         }
3301
3302         spin_lock_irqsave(&ioc->scsi_lookup_lock, flags);
3303         if (smid < ioc->internal_smid) {
3304                 /* hi-priority */
3305                 i = smid - ioc->hi_priority_smid;
3306                 ioc->hpr_lookup[i].cb_idx = 0xFF;
3307                 list_add(&ioc->hpr_lookup[i].tracker_list, &ioc->hpr_free_list);
3308         } else if (smid <= ioc->hba_queue_depth) {
3309                 /* internal queue */
3310                 i = smid - ioc->internal_smid;
3311                 ioc->internal_lookup[i].cb_idx = 0xFF;
3312                 list_add(&ioc->internal_lookup[i].tracker_list,
3313                     &ioc->internal_free_list);
3314         }
3315         spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags);
3316 }
3317
3318 /**
3319  * _base_mpi_ep_writeq - 32 bit write to MMIO
3320  * @b: data payload
3321  * @addr: address in MMIO space
3322  * @writeq_lock: spin lock
3323  *
3324  * This special handling for MPI EP to take care of 32 bit
3325  * environment where its not quarenteed to send the entire word
3326  * in one transfer.
3327  */
3328 static inline void
3329 _base_mpi_ep_writeq(__u64 b, volatile void __iomem *addr,
3330                                         spinlock_t *writeq_lock)
3331 {
3332         unsigned long flags;
3333
3334         spin_lock_irqsave(writeq_lock, flags);
3335         __raw_writel((u32)(b), addr);
3336         __raw_writel((u32)(b >> 32), (addr + 4));
3337         mmiowb();
3338         spin_unlock_irqrestore(writeq_lock, flags);
3339 }
3340
3341 /**
3342  * _base_writeq - 64 bit write to MMIO
3343  * @b: data payload
3344  * @addr: address in MMIO space
3345  * @writeq_lock: spin lock
3346  *
3347  * Glue for handling an atomic 64 bit word to MMIO. This special handling takes
3348  * care of 32 bit environment where its not quarenteed to send the entire word
3349  * in one transfer.
3350  */
3351 #if defined(writeq) && defined(CONFIG_64BIT)
3352 static inline void
3353 _base_writeq(__u64 b, volatile void __iomem *addr, spinlock_t *writeq_lock)
3354 {
3355         wmb();
3356         __raw_writeq(b, addr);
3357         barrier();
3358 }
3359 #else
3360 static inline void
3361 _base_writeq(__u64 b, volatile void __iomem *addr, spinlock_t *writeq_lock)
3362 {
3363         _base_mpi_ep_writeq(b, addr, writeq_lock);
3364 }
3365 #endif
3366
3367 /**
3368  * _base_put_smid_mpi_ep_scsi_io - send SCSI_IO request to firmware
3369  * @ioc: per adapter object
3370  * @smid: system request message index
3371  * @handle: device handle
3372  */
3373 static void
3374 _base_put_smid_mpi_ep_scsi_io(struct MPT3SAS_ADAPTER *ioc, u16 smid, u16 handle)
3375 {
3376         Mpi2RequestDescriptorUnion_t descriptor;
3377         u64 *request = (u64 *)&descriptor;
3378         void *mpi_req_iomem;
3379         __le32 *mfp = (__le32 *)mpt3sas_base_get_msg_frame(ioc, smid);
3380
3381         _clone_sg_entries(ioc, (void *) mfp, smid);
3382         mpi_req_iomem = (void __force *)ioc->chip +
3383                         MPI_FRAME_START_OFFSET + (smid * ioc->request_sz);
3384         _base_clone_mpi_to_sys_mem(mpi_req_iomem, (void *)mfp,
3385                                         ioc->request_sz);
3386         descriptor.SCSIIO.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_SCSI_IO;
3387         descriptor.SCSIIO.MSIxIndex =  _base_get_msix_index(ioc);
3388         descriptor.SCSIIO.SMID = cpu_to_le16(smid);
3389         descriptor.SCSIIO.DevHandle = cpu_to_le16(handle);
3390         descriptor.SCSIIO.LMID = 0;
3391         _base_mpi_ep_writeq(*request, &ioc->chip->RequestDescriptorPostLow,
3392             &ioc->scsi_lookup_lock);
3393 }
3394
3395 /**
3396  * _base_put_smid_scsi_io - send SCSI_IO request to firmware
3397  * @ioc: per adapter object
3398  * @smid: system request message index
3399  * @handle: device handle
3400  */
3401 static void
3402 _base_put_smid_scsi_io(struct MPT3SAS_ADAPTER *ioc, u16 smid, u16 handle)
3403 {
3404         Mpi2RequestDescriptorUnion_t descriptor;
3405         u64 *request = (u64 *)&descriptor;
3406
3407
3408         descriptor.SCSIIO.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_SCSI_IO;
3409         descriptor.SCSIIO.MSIxIndex =  _base_get_msix_index(ioc);
3410         descriptor.SCSIIO.SMID = cpu_to_le16(smid);
3411         descriptor.SCSIIO.DevHandle = cpu_to_le16(handle);
3412         descriptor.SCSIIO.LMID = 0;
3413         _base_writeq(*request, &ioc->chip->RequestDescriptorPostLow,
3414             &ioc->scsi_lookup_lock);
3415 }
3416
3417 /**
3418  * mpt3sas_base_put_smid_fast_path - send fast path request to firmware
3419  * @ioc: per adapter object
3420  * @smid: system request message index
3421  * @handle: device handle
3422  */
3423 void
3424 mpt3sas_base_put_smid_fast_path(struct MPT3SAS_ADAPTER *ioc, u16 smid,
3425         u16 handle)
3426 {
3427         Mpi2RequestDescriptorUnion_t descriptor;
3428         u64 *request = (u64 *)&descriptor;
3429
3430         descriptor.SCSIIO.RequestFlags =
3431             MPI25_REQ_DESCRIPT_FLAGS_FAST_PATH_SCSI_IO;
3432         descriptor.SCSIIO.MSIxIndex = _base_get_msix_index(ioc);
3433         descriptor.SCSIIO.SMID = cpu_to_le16(smid);
3434         descriptor.SCSIIO.DevHandle = cpu_to_le16(handle);
3435         descriptor.SCSIIO.LMID = 0;
3436         _base_writeq(*request, &ioc->chip->RequestDescriptorPostLow,
3437             &ioc->scsi_lookup_lock);
3438 }
3439
3440 /**
3441  * mpt3sas_base_put_smid_hi_priority - send Task Management request to firmware
3442  * @ioc: per adapter object
3443  * @smid: system request message index
3444  * @msix_task: msix_task will be same as msix of IO incase of task abort else 0.
3445  */
3446 void
3447 mpt3sas_base_put_smid_hi_priority(struct MPT3SAS_ADAPTER *ioc, u16 smid,
3448         u16 msix_task)
3449 {
3450         Mpi2RequestDescriptorUnion_t descriptor;
3451         void *mpi_req_iomem;
3452         u64 *request;
3453
3454         if (ioc->is_mcpu_endpoint) {
3455                 __le32 *mfp = (__le32 *)mpt3sas_base_get_msg_frame(ioc, smid);
3456
3457                 /* TBD 256 is offset within sys register. */
3458                 mpi_req_iomem = (void __force *)ioc->chip
3459                                         + MPI_FRAME_START_OFFSET
3460                                         + (smid * ioc->request_sz);
3461                 _base_clone_mpi_to_sys_mem(mpi_req_iomem, (void *)mfp,
3462                                                         ioc->request_sz);
3463         }
3464
3465         request = (u64 *)&descriptor;
3466
3467         descriptor.HighPriority.RequestFlags =
3468             MPI2_REQ_DESCRIPT_FLAGS_HIGH_PRIORITY;
3469         descriptor.HighPriority.MSIxIndex =  msix_task;
3470         descriptor.HighPriority.SMID = cpu_to_le16(smid);
3471         descriptor.HighPriority.LMID = 0;
3472         descriptor.HighPriority.Reserved1 = 0;
3473         if (ioc->is_mcpu_endpoint)
3474                 _base_mpi_ep_writeq(*request,
3475                                 &ioc->chip->RequestDescriptorPostLow,
3476                                 &ioc->scsi_lookup_lock);
3477         else
3478                 _base_writeq(*request, &ioc->chip->RequestDescriptorPostLow,
3479                     &ioc->scsi_lookup_lock);
3480 }
3481
3482 /**
3483  * mpt3sas_base_put_smid_nvme_encap - send NVMe encapsulated request to
3484  *  firmware
3485  * @ioc: per adapter object
3486  * @smid: system request message index
3487  */
3488 void
3489 mpt3sas_base_put_smid_nvme_encap(struct MPT3SAS_ADAPTER *ioc, u16 smid)
3490 {
3491         Mpi2RequestDescriptorUnion_t descriptor;
3492         u64 *request = (u64 *)&descriptor;
3493
3494         descriptor.Default.RequestFlags =
3495                 MPI26_REQ_DESCRIPT_FLAGS_PCIE_ENCAPSULATED;
3496         descriptor.Default.MSIxIndex =  _base_get_msix_index(ioc);
3497         descriptor.Default.SMID = cpu_to_le16(smid);
3498         descriptor.Default.LMID = 0;
3499         descriptor.Default.DescriptorTypeDependent = 0;
3500         _base_writeq(*request, &ioc->chip->RequestDescriptorPostLow,
3501             &ioc->scsi_lookup_lock);
3502 }
3503
3504 /**
3505  * mpt3sas_base_put_smid_default - Default, primarily used for config pages
3506  * @ioc: per adapter object
3507  * @smid: system request message index
3508  */
3509 void
3510 mpt3sas_base_put_smid_default(struct MPT3SAS_ADAPTER *ioc, u16 smid)
3511 {
3512         Mpi2RequestDescriptorUnion_t descriptor;
3513         void *mpi_req_iomem;
3514         u64 *request;
3515
3516         if (ioc->is_mcpu_endpoint) {
3517                 __le32 *mfp = (__le32 *)mpt3sas_base_get_msg_frame(ioc, smid);
3518
3519                 _clone_sg_entries(ioc, (void *) mfp, smid);
3520                 /* TBD 256 is offset within sys register */
3521                 mpi_req_iomem = (void __force *)ioc->chip +
3522                         MPI_FRAME_START_OFFSET + (smid * ioc->request_sz);
3523                 _base_clone_mpi_to_sys_mem(mpi_req_iomem, (void *)mfp,
3524                                                         ioc->request_sz);
3525         }
3526         request = (u64 *)&descriptor;
3527         descriptor.Default.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE;
3528         descriptor.Default.MSIxIndex =  _base_get_msix_index(ioc);
3529         descriptor.Default.SMID = cpu_to_le16(smid);
3530         descriptor.Default.LMID = 0;
3531         descriptor.Default.DescriptorTypeDependent = 0;
3532         if (ioc->is_mcpu_endpoint)
3533                 _base_mpi_ep_writeq(*request,
3534                                 &ioc->chip->RequestDescriptorPostLow,
3535                                 &ioc->scsi_lookup_lock);
3536         else
3537                 _base_writeq(*request, &ioc->chip->RequestDescriptorPostLow,
3538                                 &ioc->scsi_lookup_lock);
3539 }
3540
3541 /**
3542  * _base_display_OEMs_branding - Display branding string
3543  * @ioc: per adapter object
3544  */
3545 static void
3546 _base_display_OEMs_branding(struct MPT3SAS_ADAPTER *ioc)
3547 {
3548         if (ioc->pdev->subsystem_vendor != PCI_VENDOR_ID_INTEL)
3549                 return;
3550
3551         switch (ioc->pdev->subsystem_vendor) {
3552         case PCI_VENDOR_ID_INTEL:
3553                 switch (ioc->pdev->device) {
3554                 case MPI2_MFGPAGE_DEVID_SAS2008:
3555                         switch (ioc->pdev->subsystem_device) {
3556                         case MPT2SAS_INTEL_RMS2LL080_SSDID:
3557                                 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3558                                     MPT2SAS_INTEL_RMS2LL080_BRANDING);
3559                                 break;
3560                         case MPT2SAS_INTEL_RMS2LL040_SSDID:
3561                                 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3562                                     MPT2SAS_INTEL_RMS2LL040_BRANDING);
3563                                 break;
3564                         case MPT2SAS_INTEL_SSD910_SSDID:
3565                                 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3566                                     MPT2SAS_INTEL_SSD910_BRANDING);
3567                                 break;
3568                         default:
3569                                 pr_info(MPT3SAS_FMT
3570                                  "Intel(R) Controller: Subsystem ID: 0x%X\n",
3571                                  ioc->name, ioc->pdev->subsystem_device);
3572                                 break;
3573                         }
3574                 case MPI2_MFGPAGE_DEVID_SAS2308_2:
3575                         switch (ioc->pdev->subsystem_device) {
3576                         case MPT2SAS_INTEL_RS25GB008_SSDID:
3577                                 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3578                                     MPT2SAS_INTEL_RS25GB008_BRANDING);
3579                                 break;
3580                         case MPT2SAS_INTEL_RMS25JB080_SSDID:
3581                                 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3582                                     MPT2SAS_INTEL_RMS25JB080_BRANDING);
3583                                 break;
3584                         case MPT2SAS_INTEL_RMS25JB040_SSDID:
3585                                 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3586                                     MPT2SAS_INTEL_RMS25JB040_BRANDING);
3587                                 break;
3588                         case MPT2SAS_INTEL_RMS25KB080_SSDID:
3589                                 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3590                                     MPT2SAS_INTEL_RMS25KB080_BRANDING);
3591                                 break;
3592                         case MPT2SAS_INTEL_RMS25KB040_SSDID:
3593                                 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3594                                     MPT2SAS_INTEL_RMS25KB040_BRANDING);
3595                                 break;
3596                         case MPT2SAS_INTEL_RMS25LB040_SSDID:
3597                                 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3598                                     MPT2SAS_INTEL_RMS25LB040_BRANDING);
3599                                 break;
3600                         case MPT2SAS_INTEL_RMS25LB080_SSDID:
3601                                 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3602                                     MPT2SAS_INTEL_RMS25LB080_BRANDING);
3603                                 break;
3604                         default:
3605                                 pr_info(MPT3SAS_FMT
3606                                  "Intel(R) Controller: Subsystem ID: 0x%X\n",
3607                                  ioc->name, ioc->pdev->subsystem_device);
3608                                 break;
3609                         }
3610                 case MPI25_MFGPAGE_DEVID_SAS3008:
3611                         switch (ioc->pdev->subsystem_device) {
3612                         case MPT3SAS_INTEL_RMS3JC080_SSDID:
3613                                 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3614                                         MPT3SAS_INTEL_RMS3JC080_BRANDING);
3615                                 break;
3616
3617                         case MPT3SAS_INTEL_RS3GC008_SSDID:
3618                                 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3619                                         MPT3SAS_INTEL_RS3GC008_BRANDING);
3620                                 break;
3621                         case MPT3SAS_INTEL_RS3FC044_SSDID:
3622                                 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3623                                         MPT3SAS_INTEL_RS3FC044_BRANDING);
3624                                 break;
3625                         case MPT3SAS_INTEL_RS3UC080_SSDID:
3626                                 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3627                                         MPT3SAS_INTEL_RS3UC080_BRANDING);
3628                                 break;
3629                         default:
3630                                 pr_info(MPT3SAS_FMT
3631                                  "Intel(R) Controller: Subsystem ID: 0x%X\n",
3632                                  ioc->name, ioc->pdev->subsystem_device);
3633                                 break;
3634                         }
3635                         break;
3636                 default:
3637                         pr_info(MPT3SAS_FMT
3638                          "Intel(R) Controller: Subsystem ID: 0x%X\n",
3639                          ioc->name, ioc->pdev->subsystem_device);
3640                         break;
3641                 }
3642                 break;
3643         case PCI_VENDOR_ID_DELL:
3644                 switch (ioc->pdev->device) {
3645                 case MPI2_MFGPAGE_DEVID_SAS2008:
3646                         switch (ioc->pdev->subsystem_device) {
3647                         case MPT2SAS_DELL_6GBPS_SAS_HBA_SSDID:
3648                                 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3649                                  MPT2SAS_DELL_6GBPS_SAS_HBA_BRANDING);
3650                                 break;
3651                         case MPT2SAS_DELL_PERC_H200_ADAPTER_SSDID:
3652                                 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3653                                  MPT2SAS_DELL_PERC_H200_ADAPTER_BRANDING);
3654                                 break;
3655                         case MPT2SAS_DELL_PERC_H200_INTEGRATED_SSDID:
3656                                 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3657                                  MPT2SAS_DELL_PERC_H200_INTEGRATED_BRANDING);
3658                                 break;
3659                         case MPT2SAS_DELL_PERC_H200_MODULAR_SSDID:
3660                                 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3661                                  MPT2SAS_DELL_PERC_H200_MODULAR_BRANDING);
3662                                 break;
3663                         case MPT2SAS_DELL_PERC_H200_EMBEDDED_SSDID:
3664                                 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3665                                  MPT2SAS_DELL_PERC_H200_EMBEDDED_BRANDING);
3666                                 break;
3667                         case MPT2SAS_DELL_PERC_H200_SSDID:
3668                                 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3669                                  MPT2SAS_DELL_PERC_H200_BRANDING);
3670                                 break;
3671                         case MPT2SAS_DELL_6GBPS_SAS_SSDID:
3672                                 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3673                                  MPT2SAS_DELL_6GBPS_SAS_BRANDING);
3674                                 break;
3675                         default:
3676                                 pr_info(MPT3SAS_FMT
3677                                    "Dell 6Gbps HBA: Subsystem ID: 0x%X\n",
3678                                    ioc->name, ioc->pdev->subsystem_device);
3679                                 break;
3680                         }
3681                         break;
3682                 case MPI25_MFGPAGE_DEVID_SAS3008:
3683                         switch (ioc->pdev->subsystem_device) {
3684                         case MPT3SAS_DELL_12G_HBA_SSDID:
3685                                 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3686                                         MPT3SAS_DELL_12G_HBA_BRANDING);
3687                                 break;
3688                         default:
3689                                 pr_info(MPT3SAS_FMT
3690                                    "Dell 12Gbps HBA: Subsystem ID: 0x%X\n",
3691                                    ioc->name, ioc->pdev->subsystem_device);
3692                                 break;
3693                         }
3694                         break;
3695                 default:
3696                         pr_info(MPT3SAS_FMT
3697                            "Dell HBA: Subsystem ID: 0x%X\n", ioc->name,
3698                            ioc->pdev->subsystem_device);
3699                         break;
3700                 }
3701                 break;
3702         case PCI_VENDOR_ID_CISCO:
3703                 switch (ioc->pdev->device) {
3704                 case MPI25_MFGPAGE_DEVID_SAS3008:
3705                         switch (ioc->pdev->subsystem_device) {
3706                         case MPT3SAS_CISCO_12G_8E_HBA_SSDID:
3707                                 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3708                                         MPT3SAS_CISCO_12G_8E_HBA_BRANDING);
3709                                 break;
3710                         case MPT3SAS_CISCO_12G_8I_HBA_SSDID:
3711                                 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3712                                         MPT3SAS_CISCO_12G_8I_HBA_BRANDING);
3713                                 break;
3714                         case MPT3SAS_CISCO_12G_AVILA_HBA_SSDID:
3715                                 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3716                                         MPT3SAS_CISCO_12G_AVILA_HBA_BRANDING);
3717                                 break;
3718                         default:
3719                                 pr_info(MPT3SAS_FMT
3720                                   "Cisco 12Gbps SAS HBA: Subsystem ID: 0x%X\n",
3721                                   ioc->name, ioc->pdev->subsystem_device);
3722                                 break;
3723                         }
3724                         break;
3725                 case MPI25_MFGPAGE_DEVID_SAS3108_1:
3726                         switch (ioc->pdev->subsystem_device) {
3727                         case MPT3SAS_CISCO_12G_AVILA_HBA_SSDID:
3728                                 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3729                                 MPT3SAS_CISCO_12G_AVILA_HBA_BRANDING);
3730                                 break;
3731                         case MPT3SAS_CISCO_12G_COLUSA_MEZZANINE_HBA_SSDID:
3732                                 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3733                                 MPT3SAS_CISCO_12G_COLUSA_MEZZANINE_HBA_BRANDING
3734                                 );
3735                                 break;
3736                         default:
3737                                 pr_info(MPT3SAS_FMT
3738                                  "Cisco 12Gbps SAS HBA: Subsystem ID: 0x%X\n",
3739                                  ioc->name, ioc->pdev->subsystem_device);
3740                                 break;
3741                         }
3742                         break;
3743                 default:
3744                         pr_info(MPT3SAS_FMT
3745                            "Cisco SAS HBA: Subsystem ID: 0x%X\n",
3746                            ioc->name, ioc->pdev->subsystem_device);
3747                         break;
3748                 }
3749                 break;
3750         case MPT2SAS_HP_3PAR_SSVID:
3751                 switch (ioc->pdev->device) {
3752                 case MPI2_MFGPAGE_DEVID_SAS2004:
3753                         switch (ioc->pdev->subsystem_device) {
3754                         case MPT2SAS_HP_DAUGHTER_2_4_INTERNAL_SSDID:
3755                                 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3756                                     MPT2SAS_HP_DAUGHTER_2_4_INTERNAL_BRANDING);
3757                                 break;
3758                         default:
3759                                 pr_info(MPT3SAS_FMT
3760                                    "HP 6Gbps SAS HBA: Subsystem ID: 0x%X\n",
3761                                    ioc->name, ioc->pdev->subsystem_device);
3762                                 break;
3763                         }
3764                 case MPI2_MFGPAGE_DEVID_SAS2308_2:
3765                         switch (ioc->pdev->subsystem_device) {
3766                         case MPT2SAS_HP_2_4_INTERNAL_SSDID:
3767                                 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3768                                     MPT2SAS_HP_2_4_INTERNAL_BRANDING);
3769                                 break;
3770                         case MPT2SAS_HP_2_4_EXTERNAL_SSDID:
3771                                 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3772                                     MPT2SAS_HP_2_4_EXTERNAL_BRANDING);
3773                                 break;
3774                         case MPT2SAS_HP_1_4_INTERNAL_1_4_EXTERNAL_SSDID:
3775                                 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3776                                  MPT2SAS_HP_1_4_INTERNAL_1_4_EXTERNAL_BRANDING);
3777                                 break;
3778                         case MPT2SAS_HP_EMBEDDED_2_4_INTERNAL_SSDID:
3779                                 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3780                                     MPT2SAS_HP_EMBEDDED_2_4_INTERNAL_BRANDING);
3781                                 break;
3782                         default:
3783                                 pr_info(MPT3SAS_FMT
3784                                    "HP 6Gbps SAS HBA: Subsystem ID: 0x%X\n",
3785                                    ioc->name, ioc->pdev->subsystem_device);
3786                                 break;
3787                         }
3788                 default:
3789                         pr_info(MPT3SAS_FMT
3790                            "HP SAS HBA: Subsystem ID: 0x%X\n",
3791                            ioc->name, ioc->pdev->subsystem_device);
3792                         break;
3793                 }
3794         default:
3795                 break;
3796         }
3797 }
3798
3799 /**
3800  * _base_display_fwpkg_version - sends FWUpload request to pull FWPkg
3801  *                              version from FW Image Header.
3802  * @ioc: per adapter object
3803  *
3804  * Return: 0 for success, non-zero for failure.
3805  */
3806         static int
3807 _base_display_fwpkg_version(struct MPT3SAS_ADAPTER *ioc)
3808 {
3809         Mpi2FWImageHeader_t *FWImgHdr;
3810         Mpi25FWUploadRequest_t *mpi_request;
3811         Mpi2FWUploadReply_t mpi_reply;
3812         int r = 0;
3813         void *fwpkg_data = NULL;
3814         dma_addr_t fwpkg_data_dma;
3815         u16 smid, ioc_status;
3816         size_t data_length;
3817
3818         dinitprintk(ioc, pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3819                                 __func__));
3820
3821         if (ioc->base_cmds.status & MPT3_CMD_PENDING) {
3822                 pr_err(MPT3SAS_FMT "%s: internal command already in use\n",
3823                                 ioc->name, __func__);
3824                 return -EAGAIN;
3825         }
3826
3827         data_length = sizeof(Mpi2FWImageHeader_t);
3828         fwpkg_data = pci_alloc_consistent(ioc->pdev, data_length,
3829                         &fwpkg_data_dma);
3830         if (!fwpkg_data) {
3831                 pr_err(MPT3SAS_FMT "failure at %s:%d/%s()!\n",
3832                                 ioc->name, __FILE__, __LINE__, __func__);
3833                 return -ENOMEM;
3834         }
3835
3836         smid = mpt3sas_base_get_smid(ioc, ioc->base_cb_idx);
3837         if (!smid) {
3838                 pr_err(MPT3SAS_FMT "%s: failed obtaining a smid\n",
3839                                 ioc->name, __func__);
3840                 r = -EAGAIN;
3841                 goto out;
3842         }
3843
3844         ioc->base_cmds.status = MPT3_CMD_PENDING;
3845         mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);
3846         ioc->base_cmds.smid = smid;
3847         memset(mpi_request, 0, sizeof(Mpi25FWUploadRequest_t));
3848         mpi_request->Function = MPI2_FUNCTION_FW_UPLOAD;
3849         mpi_request->ImageType = MPI2_FW_UPLOAD_ITYPE_FW_FLASH;
3850         mpi_request->ImageSize = cpu_to_le32(data_length);
3851         ioc->build_sg(ioc, &mpi_request->SGL, 0, 0, fwpkg_data_dma,
3852                         data_length);
3853         init_completion(&ioc->base_cmds.done);
3854         mpt3sas_base_put_smid_default(ioc, smid);
3855         /* Wait for 15 seconds */
3856         wait_for_completion_timeout(&ioc->base_cmds.done,
3857                         FW_IMG_HDR_READ_TIMEOUT*HZ);
3858         pr_info(MPT3SAS_FMT "%s: complete\n",
3859                         ioc->name, __func__);
3860         if (!(ioc->base_cmds.status & MPT3_CMD_COMPLETE)) {
3861                 pr_err(MPT3SAS_FMT "%s: timeout\n",
3862                                 ioc->name, __func__);
3863                 _debug_dump_mf(mpi_request,
3864                                 sizeof(Mpi25FWUploadRequest_t)/4);
3865                 r = -ETIME;
3866         } else {
3867                 memset(&mpi_reply, 0, sizeof(Mpi2FWUploadReply_t));
3868                 if (ioc->base_cmds.status & MPT3_CMD_REPLY_VALID) {
3869                         memcpy(&mpi_reply, ioc->base_cmds.reply,
3870                                         sizeof(Mpi2FWUploadReply_t));
3871                         ioc_status = le16_to_cpu(mpi_reply.IOCStatus) &
3872                                                 MPI2_IOCSTATUS_MASK;
3873                         if (ioc_status == MPI2_IOCSTATUS_SUCCESS) {
3874                                 FWImgHdr = (Mpi2FWImageHeader_t *)fwpkg_data;
3875                                 if (FWImgHdr->PackageVersion.Word) {
3876                                         pr_info(MPT3SAS_FMT "FW Package Version"
3877                                         "(%02d.%02d.%02d.%02d)\n",
3878                                         ioc->name,
3879                                         FWImgHdr->PackageVersion.Struct.Major,
3880                                         FWImgHdr->PackageVersion.Struct.Minor,
3881                                         FWImgHdr->PackageVersion.Struct.Unit,
3882                                         FWImgHdr->PackageVersion.Struct.Dev);
3883                                 }
3884                         } else {
3885                                 _debug_dump_mf(&mpi_reply,
3886                                                 sizeof(Mpi2FWUploadReply_t)/4);
3887                         }
3888                 }
3889         }
3890         ioc->base_cmds.status = MPT3_CMD_NOT_USED;
3891 out:
3892         if (fwpkg_data)
3893                 pci_free_consistent(ioc->pdev, data_length, fwpkg_data,
3894                                 fwpkg_data_dma);
3895         return r;
3896 }
3897
3898 /**
3899  * _base_display_ioc_capabilities - Disply IOC's capabilities.
3900  * @ioc: per adapter object
3901  */
3902 static void
3903 _base_display_ioc_capabilities(struct MPT3SAS_ADAPTER *ioc)
3904 {
3905         int i = 0;
3906         char desc[16];
3907         u32 iounit_pg1_flags;
3908         u32 bios_version;
3909
3910         bios_version = le32_to_cpu(ioc->bios_pg3.BiosVersion);
3911         strncpy(desc, ioc->manu_pg0.ChipName, 16);
3912         pr_info(MPT3SAS_FMT "%s: FWVersion(%02d.%02d.%02d.%02d), "\
3913            "ChipRevision(0x%02x), BiosVersion(%02d.%02d.%02d.%02d)\n",
3914             ioc->name, desc,
3915            (ioc->facts.FWVersion.Word & 0xFF000000) >> 24,
3916            (ioc->facts.FWVersion.Word & 0x00FF0000) >> 16,
3917            (ioc->facts.FWVersion.Word & 0x0000FF00) >> 8,
3918            ioc->facts.FWVersion.Word & 0x000000FF,
3919            ioc->pdev->revision,
3920            (bios_version & 0xFF000000) >> 24,
3921            (bios_version & 0x00FF0000) >> 16,
3922            (bios_version & 0x0000FF00) >> 8,
3923             bios_version & 0x000000FF);
3924
3925         _base_display_OEMs_branding(ioc);
3926
3927         if (ioc->facts.ProtocolFlags & MPI2_IOCFACTS_PROTOCOL_NVME_DEVICES) {
3928                 pr_info("%sNVMe", i ? "," : "");
3929                 i++;
3930         }
3931
3932         pr_info(MPT3SAS_FMT "Protocol=(", ioc->name);
3933
3934         if (ioc->facts.ProtocolFlags & MPI2_IOCFACTS_PROTOCOL_SCSI_INITIATOR) {
3935                 pr_info("Initiator");
3936                 i++;
3937         }
3938
3939         if (ioc->facts.ProtocolFlags & MPI2_IOCFACTS_PROTOCOL_SCSI_TARGET) {
3940                 pr_info("%sTarget", i ? "," : "");
3941                 i++;
3942         }
3943
3944         i = 0;
3945         pr_info("), ");
3946         pr_info("Capabilities=(");
3947
3948         if (!ioc->hide_ir_msg) {
3949                 if (ioc->facts.IOCCapabilities &
3950                     MPI2_IOCFACTS_CAPABILITY_INTEGRATED_RAID) {
3951                         pr_info("Raid");
3952                         i++;
3953                 }
3954         }
3955
3956         if (ioc->facts.IOCCapabilities & MPI2_IOCFACTS_CAPABILITY_TLR) {
3957                 pr_info("%sTLR", i ? "," : "");
3958                 i++;
3959         }
3960
3961         if (ioc->facts.IOCCapabilities & MPI2_IOCFACTS_CAPABILITY_MULTICAST) {
3962                 pr_info("%sMulticast", i ? "," : "");
3963                 i++;
3964         }
3965
3966         if (ioc->facts.IOCCapabilities &
3967             MPI2_IOCFACTS_CAPABILITY_BIDIRECTIONAL_TARGET) {
3968                 pr_info("%sBIDI Target", i ? "," : "");
3969                 i++;
3970         }
3971
3972         if (ioc->facts.IOCCapabilities & MPI2_IOCFACTS_CAPABILITY_EEDP) {
3973                 pr_info("%sEEDP", i ? "," : "");
3974                 i++;
3975         }
3976
3977         if (ioc->facts.IOCCapabilities &
3978             MPI2_IOCFACTS_CAPABILITY_SNAPSHOT_BUFFER) {
3979                 pr_info("%sSnapshot Buffer", i ? "," : "");
3980                 i++;
3981         }
3982
3983         if (ioc->facts.IOCCapabilities &
3984             MPI2_IOCFACTS_CAPABILITY_DIAG_TRACE_BUFFER) {
3985                 pr_info("%sDiag Trace Buffer", i ? "," : "");
3986                 i++;
3987         }
3988
3989         if (ioc->facts.IOCCapabilities &
3990             MPI2_IOCFACTS_CAPABILITY_EXTENDED_BUFFER) {
3991                 pr_info("%sDiag Extended Buffer", i ? "," : "");
3992                 i++;
3993         }
3994
3995         if (ioc->facts.IOCCapabilities &
3996             MPI2_IOCFACTS_CAPABILITY_TASK_SET_FULL_HANDLING) {
3997                 pr_info("%sTask Set Full", i ? "," : "");
3998                 i++;
3999         }
4000
4001         iounit_pg1_flags = le32_to_cpu(ioc->iounit_pg1.Flags);
4002         if (!(iounit_pg1_flags & MPI2_IOUNITPAGE1_NATIVE_COMMAND_Q_DISABLE)) {
4003                 pr_info("%sNCQ", i ? "," : "");
4004                 i++;
4005         }
4006
4007         pr_info(")\n");
4008 }
4009
4010 /**
4011  * mpt3sas_base_update_missing_delay - change the missing delay timers
4012  * @ioc: per adapter object
4013  * @device_missing_delay: amount of time till device is reported missing
4014  * @io_missing_delay: interval IO is returned when there is a missing device
4015  *
4016  * Passed on the command line, this function will modify the device missing
4017  * delay, as well as the io missing delay. This should be called at driver
4018  * load time.
4019  */
4020 void
4021 mpt3sas_base_update_missing_delay(struct MPT3SAS_ADAPTER *ioc,
4022         u16 device_missing_delay, u8 io_missing_delay)
4023 {
4024         u16 dmd, dmd_new, dmd_orignal;
4025         u8 io_missing_delay_original;
4026         u16 sz;
4027         Mpi2SasIOUnitPage1_t *sas_iounit_pg1 = NULL;
4028         Mpi2ConfigReply_t mpi_reply;
4029         u8 num_phys = 0;
4030         u16 ioc_status;
4031
4032         mpt3sas_config_get_number_hba_phys(ioc, &num_phys);
4033         if (!num_phys)
4034                 return;
4035
4036         sz = offsetof(Mpi2SasIOUnitPage1_t, PhyData) + (num_phys *
4037             sizeof(Mpi2SasIOUnit1PhyData_t));
4038         sas_iounit_pg1 = kzalloc(sz, GFP_KERNEL);
4039         if (!sas_iounit_pg1) {
4040                 pr_err(MPT3SAS_FMT "failure at %s:%d/%s()!\n",
4041                     ioc->name, __FILE__, __LINE__, __func__);
4042                 goto out;
4043         }
4044         if ((mpt3sas_config_get_sas_iounit_pg1(ioc, &mpi_reply,
4045             sas_iounit_pg1, sz))) {
4046                 pr_err(MPT3SAS_FMT "failure at %s:%d/%s()!\n",
4047                     ioc->name, __FILE__, __LINE__, __func__);
4048                 goto out;
4049         }
4050         ioc_status = le16_to_cpu(mpi_reply.IOCStatus) &
4051             MPI2_IOCSTATUS_MASK;
4052         if (ioc_status != MPI2_IOCSTATUS_SUCCESS) {
4053                 pr_err(MPT3SAS_FMT "failure at %s:%d/%s()!\n",
4054                     ioc->name, __FILE__, __LINE__, __func__);
4055                 goto out;
4056         }
4057
4058         /* device missing delay */
4059         dmd = sas_iounit_pg1->ReportDeviceMissingDelay;
4060         if (dmd & MPI2_SASIOUNIT1_REPORT_MISSING_UNIT_16)
4061                 dmd = (dmd & MPI2_SASIOUNIT1_REPORT_MISSING_TIMEOUT_MASK) * 16;
4062         else
4063                 dmd = dmd & MPI2_SASIOUNIT1_REPORT_MISSING_TIMEOUT_MASK;
4064         dmd_orignal = dmd;
4065         if (device_missing_delay > 0x7F) {
4066                 dmd = (device_missing_delay > 0x7F0) ? 0x7F0 :
4067                     device_missing_delay;
4068                 dmd = dmd / 16;
4069                 dmd |= MPI2_SASIOUNIT1_REPORT_MISSING_UNIT_16;
4070         } else
4071                 dmd = device_missing_delay;
4072         sas_iounit_pg1->ReportDeviceMissingDelay = dmd;
4073
4074         /* io missing delay */
4075         io_missing_delay_original = sas_iounit_pg1->IODeviceMissingDelay;
4076         sas_iounit_pg1->IODeviceMissingDelay = io_missing_delay;
4077
4078         if (!mpt3sas_config_set_sas_iounit_pg1(ioc, &mpi_reply, sas_iounit_pg1,
4079             sz)) {
4080                 if (dmd & MPI2_SASIOUNIT1_REPORT_MISSING_UNIT_16)
4081                         dmd_new = (dmd &
4082                             MPI2_SASIOUNIT1_REPORT_MISSING_TIMEOUT_MASK) * 16;
4083                 else
4084                         dmd_new =
4085                     dmd & MPI2_SASIOUNIT1_REPORT_MISSING_TIMEOUT_MASK;
4086                 pr_info(MPT3SAS_FMT "device_missing_delay: old(%d), new(%d)\n",
4087                         ioc->name, dmd_orignal, dmd_new);
4088                 pr_info(MPT3SAS_FMT "ioc_missing_delay: old(%d), new(%d)\n",
4089                         ioc->name, io_missing_delay_original,
4090                     io_missing_delay);
4091                 ioc->device_missing_delay = dmd_new;
4092                 ioc->io_missing_delay = io_missing_delay;
4093         }
4094
4095 out:
4096         kfree(sas_iounit_pg1);
4097 }
4098
4099 /**
4100  * _base_static_config_pages - static start of day config pages
4101  * @ioc: per adapter object
4102  */
4103 static void
4104 _base_static_config_pages(struct MPT3SAS_ADAPTER *ioc)
4105 {
4106         Mpi2ConfigReply_t mpi_reply;
4107         u32 iounit_pg1_flags;
4108
4109         ioc->nvme_abort_timeout = 30;
4110         mpt3sas_config_get_manufacturing_pg0(ioc, &mpi_reply, &ioc->manu_pg0);
4111         if (ioc->ir_firmware)
4112                 mpt3sas_config_get_manufacturing_pg10(ioc, &mpi_reply,
4113                     &ioc->manu_pg10);
4114
4115         /*
4116          * Ensure correct T10 PI operation if vendor left EEDPTagMode
4117          * flag unset in NVDATA.
4118          */
4119         mpt3sas_config_get_manufacturing_pg11(ioc, &mpi_reply, &ioc->manu_pg11);
4120         if (!ioc->is_gen35_ioc && ioc->manu_pg11.EEDPTagMode == 0) {
4121                 pr_err("%s: overriding NVDATA EEDPTagMode setting\n",
4122                     ioc->name);
4123                 ioc->manu_pg11.EEDPTagMode &= ~0x3;
4124                 ioc->manu_pg11.EEDPTagMode |= 0x1;
4125                 mpt3sas_config_set_manufacturing_pg11(ioc, &mpi_reply,
4126                     &ioc->manu_pg11);
4127         }
4128         if (ioc->manu_pg11.AddlFlags2 & NVME_TASK_MNGT_CUSTOM_MASK)
4129                 ioc->tm_custom_handling = 1;
4130         else {
4131                 ioc->tm_custom_handling = 0;
4132                 if (ioc->manu_pg11.NVMeAbortTO < NVME_TASK_ABORT_MIN_TIMEOUT)
4133                         ioc->nvme_abort_timeout = NVME_TASK_ABORT_MIN_TIMEOUT;
4134                 else if (ioc->manu_pg11.NVMeAbortTO >
4135                                         NVME_TASK_ABORT_MAX_TIMEOUT)
4136                         ioc->nvme_abort_timeout = NVME_TASK_ABORT_MAX_TIMEOUT;
4137                 else
4138                         ioc->nvme_abort_timeout = ioc->manu_pg11.NVMeAbortTO;
4139         }
4140
4141         mpt3sas_config_get_bios_pg2(ioc, &mpi_reply, &ioc->bios_pg2);
4142         mpt3sas_config_get_bios_pg3(ioc, &mpi_reply, &ioc->bios_pg3);
4143         mpt3sas_config_get_ioc_pg8(ioc, &mpi_reply, &ioc->ioc_pg8);
4144         mpt3sas_config_get_iounit_pg0(ioc, &mpi_reply, &ioc->iounit_pg0);
4145         mpt3sas_config_get_iounit_pg1(ioc, &mpi_reply, &ioc->iounit_pg1);
4146         mpt3sas_config_get_iounit_pg8(ioc, &mpi_reply, &ioc->iounit_pg8);
4147         _base_display_ioc_capabilities(ioc);
4148
4149         /*
4150          * Enable task_set_full handling in iounit_pg1 when the
4151          * facts capabilities indicate that its supported.
4152          */
4153         iounit_pg1_flags = le32_to_cpu(ioc->iounit_pg1.Flags);
4154         if ((ioc->facts.IOCCapabilities &
4155             MPI2_IOCFACTS_CAPABILITY_TASK_SET_FULL_HANDLING))
4156                 iounit_pg1_flags &=
4157                     ~MPI2_IOUNITPAGE1_DISABLE_TASK_SET_FULL_HANDLING;
4158         else
4159                 iounit_pg1_flags |=
4160                     MPI2_IOUNITPAGE1_DISABLE_TASK_SET_FULL_HANDLING;
4161         ioc->iounit_pg1.Flags = cpu_to_le32(iounit_pg1_flags);
4162         mpt3sas_config_set_iounit_pg1(ioc, &mpi_reply, &ioc->iounit_pg1);
4163
4164         if (ioc->iounit_pg8.NumSensors)
4165                 ioc->temp_sensors_count = ioc->iounit_pg8.NumSensors;
4166 }
4167
4168 /**
4169  * mpt3sas_free_enclosure_list - release memory
4170  * @ioc: per adapter object
4171  *
4172  * Free memory allocated during encloure add.
4173  */
4174 void
4175 mpt3sas_free_enclosure_list(struct MPT3SAS_ADAPTER *ioc)
4176 {
4177         struct _enclosure_node *enclosure_dev, *enclosure_dev_next;
4178
4179         /* Free enclosure list */
4180         list_for_each_entry_safe(enclosure_dev,
4181                         enclosure_dev_next, &ioc->enclosure_list, list) {
4182                 list_del(&enclosure_dev->list);
4183                 kfree(enclosure_dev);
4184         }
4185 }
4186
4187 /**
4188  * _base_release_memory_pools - release memory
4189  * @ioc: per adapter object
4190  *
4191  * Free memory allocated from _base_allocate_memory_pools.
4192  */
4193 static void
4194 _base_release_memory_pools(struct MPT3SAS_ADAPTER *ioc)
4195 {
4196         int i = 0;
4197         int j = 0;
4198         struct chain_tracker *ct;
4199         struct reply_post_struct *rps;
4200
4201         dexitprintk(ioc, pr_info(MPT3SAS_FMT "%s\n", ioc->name,
4202             __func__));
4203
4204         if (ioc->request) {
4205                 pci_free_consistent(ioc->pdev, ioc->request_dma_sz,
4206                     ioc->request,  ioc->request_dma);
4207                 dexitprintk(ioc, pr_info(MPT3SAS_FMT
4208                         "request_pool(0x%p): free\n",
4209                         ioc->name, ioc->request));
4210                 ioc->request = NULL;
4211         }
4212
4213         if (ioc->sense) {
4214                 dma_pool_free(ioc->sense_dma_pool, ioc->sense, ioc->sense_dma);
4215                 dma_pool_destroy(ioc->sense_dma_pool);
4216                 dexitprintk(ioc, pr_info(MPT3SAS_FMT
4217                         "sense_pool(0x%p): free\n",
4218                         ioc->name, ioc->sense));
4219                 ioc->sense = NULL;
4220         }
4221
4222         if (ioc->reply) {
4223                 dma_pool_free(ioc->reply_dma_pool, ioc->reply, ioc->reply_dma);
4224                 dma_pool_destroy(ioc->reply_dma_pool);
4225                 dexitprintk(ioc, pr_info(MPT3SAS_FMT
4226                         "reply_pool(0x%p): free\n",
4227                         ioc->name, ioc->reply));
4228                 ioc->reply = NULL;
4229         }
4230
4231         if (ioc->reply_free) {
4232                 dma_pool_free(ioc->reply_free_dma_pool, ioc->reply_free,
4233                     ioc->reply_free_dma);
4234                 dma_pool_destroy(ioc->reply_free_dma_pool);
4235                 dexitprintk(ioc, pr_info(MPT3SAS_FMT
4236                         "reply_free_pool(0x%p): free\n",
4237                         ioc->name, ioc->reply_free));
4238                 ioc->reply_free = NULL;
4239         }
4240
4241         if (ioc->reply_post) {
4242                 do {
4243                         rps = &ioc->reply_post[i];
4244                         if (rps->reply_post_free) {
4245                                 dma_pool_free(
4246                                     ioc->reply_post_free_dma_pool,
4247                                     rps->reply_post_free,
4248                                     rps->reply_post_free_dma);
4249                                 dexitprintk(ioc, pr_info(MPT3SAS_FMT
4250                                     "reply_post_free_pool(0x%p): free\n",
4251                                     ioc->name, rps->reply_post_free));
4252                                 rps->reply_post_free = NULL;
4253                         }
4254                 } while (ioc->rdpq_array_enable &&
4255                            (++i < ioc->reply_queue_count));
4256                 if (ioc->reply_post_free_array &&
4257                         ioc->rdpq_array_enable) {
4258                         dma_pool_free(ioc->reply_post_free_array_dma_pool,
4259                                 ioc->reply_post_free_array,
4260                                 ioc->reply_post_free_array_dma);
4261                         ioc->reply_post_free_array = NULL;
4262                 }
4263                 dma_pool_destroy(ioc->reply_post_free_array_dma_pool);
4264                 dma_pool_destroy(ioc->reply_post_free_dma_pool);
4265                 kfree(ioc->reply_post);
4266         }
4267
4268         if (ioc->pcie_sgl_dma_pool) {
4269                 for (i = 0; i < ioc->scsiio_depth; i++) {
4270                         dma_pool_free(ioc->pcie_sgl_dma_pool,
4271                                         ioc->pcie_sg_lookup[i].pcie_sgl,
4272                                         ioc->pcie_sg_lookup[i].pcie_sgl_dma);
4273                 }
4274                 if (ioc->pcie_sgl_dma_pool)
4275                         dma_pool_destroy(ioc->pcie_sgl_dma_pool);
4276         }
4277
4278         if (ioc->config_page) {
4279                 dexitprintk(ioc, pr_info(MPT3SAS_FMT
4280                     "config_page(0x%p): free\n", ioc->name,
4281                     ioc->config_page));
4282                 pci_free_consistent(ioc->pdev, ioc->config_page_sz,
4283                     ioc->config_page, ioc->config_page_dma);
4284         }
4285
4286         kfree(ioc->hpr_lookup);
4287         ioc->hpr_lookup = NULL;
4288         kfree(ioc->internal_lookup);
4289         ioc->internal_lookup = NULL;
4290         if (ioc->chain_lookup) {
4291                 for (i = 0; i < ioc->scsiio_depth; i++) {
4292                         for (j = ioc->chains_per_prp_buffer;
4293                             j < ioc->chains_needed_per_io; j++) {
4294                                 ct = &ioc->chain_lookup[i].chains_per_smid[j];
4295                                 if (ct && ct->chain_buffer)
4296                                         dma_pool_free(ioc->chain_dma_pool,
4297                                                 ct->chain_buffer,
4298                                                 ct->chain_buffer_dma);
4299                         }
4300                         kfree(ioc->chain_lookup[i].chains_per_smid);
4301                 }
4302                 dma_pool_destroy(ioc->chain_dma_pool);
4303                 kfree(ioc->chain_lookup);
4304                 ioc->chain_lookup = NULL;
4305         }
4306 }
4307
4308 /**
4309  * is_MSB_are_same - checks whether all reply queues in a set are
4310  *      having same upper 32bits in their base memory address.
4311  * @reply_pool_start_address: Base address of a reply queue set
4312  * @pool_sz: Size of single Reply Descriptor Post Queues pool size
4313  *
4314  * Return: 1 if reply queues in a set have a same upper 32bits in their base
4315  * memory address, else 0.
4316  */
4317
4318 static int
4319 is_MSB_are_same(long reply_pool_start_address, u32 pool_sz)
4320 {
4321         long reply_pool_end_address;
4322
4323         reply_pool_end_address = reply_pool_start_address + pool_sz;
4324
4325         if (upper_32_bits(reply_pool_start_address) ==
4326                 upper_32_bits(reply_pool_end_address))
4327                 return 1;
4328         else
4329                 return 0;
4330 }
4331
4332 /**
4333  * _base_allocate_memory_pools - allocate start of day memory pools
4334  * @ioc: per adapter object
4335  *
4336  * Return: 0 success, anything else error.
4337  */
4338 static int
4339 _base_allocate_memory_pools(struct MPT3SAS_ADAPTER *ioc)
4340 {
4341         struct mpt3sas_facts *facts;
4342         u16 max_sge_elements;
4343         u16 chains_needed_per_io;
4344         u32 sz, total_sz, reply_post_free_sz, reply_post_free_array_sz;
4345         u32 retry_sz;
4346         u16 max_request_credit, nvme_blocks_needed;
4347         unsigned short sg_tablesize;
4348         u16 sge_size;
4349         int i, j;
4350         struct chain_tracker *ct;
4351
4352         dinitprintk(ioc, pr_info(MPT3SAS_FMT "%s\n", ioc->name,
4353             __func__));
4354
4355
4356         retry_sz = 0;
4357         facts = &ioc->facts;
4358
4359         /* command line tunables for max sgl entries */
4360         if (max_sgl_entries != -1)
4361                 sg_tablesize = max_sgl_entries;
4362         else {
4363                 if (ioc->hba_mpi_version_belonged == MPI2_VERSION)
4364                         sg_tablesize = MPT2SAS_SG_DEPTH;
4365                 else
4366                         sg_tablesize = MPT3SAS_SG_DEPTH;
4367         }
4368
4369         /* max sgl entries <= MPT_KDUMP_MIN_PHYS_SEGMENTS in KDUMP mode */
4370         if (reset_devices)
4371                 sg_tablesize = min_t(unsigned short, sg_tablesize,
4372                    MPT_KDUMP_MIN_PHYS_SEGMENTS);
4373
4374         if (ioc->is_mcpu_endpoint)
4375                 ioc->shost->sg_tablesize = MPT_MIN_PHYS_SEGMENTS;
4376         else {
4377                 if (sg_tablesize < MPT_MIN_PHYS_SEGMENTS)
4378                         sg_tablesize = MPT_MIN_PHYS_SEGMENTS;
4379                 else if (sg_tablesize > MPT_MAX_PHYS_SEGMENTS) {
4380                         sg_tablesize = min_t(unsigned short, sg_tablesize,
4381                                         SG_MAX_SEGMENTS);
4382                         pr_warn(MPT3SAS_FMT
4383                                 "sg_tablesize(%u) is bigger than kernel "
4384                                 "defined SG_CHUNK_SIZE(%u)\n", ioc->name,
4385                                 sg_tablesize, MPT_MAX_PHYS_SEGMENTS);
4386                 }
4387                 ioc->shost->sg_tablesize = sg_tablesize;
4388         }
4389
4390         ioc->internal_depth = min_t(int, (facts->HighPriorityCredit + (5)),
4391                 (facts->RequestCredit / 4));
4392         if (ioc->internal_depth < INTERNAL_CMDS_COUNT) {
4393                 if (facts->RequestCredit <= (INTERNAL_CMDS_COUNT +
4394                                 INTERNAL_SCSIIO_CMDS_COUNT)) {
4395                         pr_err(MPT3SAS_FMT "IOC doesn't have enough Request \
4396                             Credits, it has just %d number of credits\n",
4397                             ioc->name, facts->RequestCredit);
4398                         return -ENOMEM;
4399                 }
4400                 ioc->internal_depth = 10;
4401         }
4402
4403         ioc->hi_priority_depth = ioc->internal_depth - (5);
4404         /* command line tunables  for max controller queue depth */
4405         if (max_queue_depth != -1 && max_queue_depth != 0) {
4406                 max_request_credit = min_t(u16, max_queue_depth +
4407                         ioc->internal_depth, facts->RequestCredit);
4408                 if (max_request_credit > MAX_HBA_QUEUE_DEPTH)
4409                         max_request_credit =  MAX_HBA_QUEUE_DEPTH;
4410         } else if (reset_devices)
4411                 max_request_credit = min_t(u16, facts->RequestCredit,
4412                     (MPT3SAS_KDUMP_SCSI_IO_DEPTH + ioc->internal_depth));
4413         else
4414                 max_request_credit = min_t(u16, facts->RequestCredit,
4415                     MAX_HBA_QUEUE_DEPTH);
4416
4417         /* Firmware maintains additional facts->HighPriorityCredit number of
4418          * credits for HiPriprity Request messages, so hba queue depth will be
4419          * sum of max_request_credit and high priority queue depth.
4420          */
4421         ioc->hba_queue_depth = max_request_credit + ioc->hi_priority_depth;
4422
4423         /* request frame size */
4424         ioc->request_sz = facts->IOCRequestFrameSize * 4;
4425
4426         /* reply frame size */
4427         ioc->reply_sz = facts->ReplyFrameSize * 4;
4428
4429         /* chain segment size */
4430         if (ioc->hba_mpi_version_belonged != MPI2_VERSION) {
4431                 if (facts->IOCMaxChainSegmentSize)
4432                         ioc->chain_segment_sz =
4433                                         facts->IOCMaxChainSegmentSize *
4434                                         MAX_CHAIN_ELEMT_SZ;
4435                 else
4436                 /* set to 128 bytes size if IOCMaxChainSegmentSize is zero */
4437                         ioc->chain_segment_sz = DEFAULT_NUM_FWCHAIN_ELEMTS *
4438                                                     MAX_CHAIN_ELEMT_SZ;
4439         } else
4440                 ioc->chain_segment_sz = ioc->request_sz;
4441
4442         /* calculate the max scatter element size */
4443         sge_size = max_t(u16, ioc->sge_size, ioc->sge_size_ieee);
4444
4445  retry_allocation:
4446         total_sz = 0;
4447         /* calculate number of sg elements left over in the 1st frame */
4448         max_sge_elements = ioc->request_sz - ((sizeof(Mpi2SCSIIORequest_t) -
4449             sizeof(Mpi2SGEIOUnion_t)) + sge_size);
4450         ioc->max_sges_in_main_message = max_sge_elements/sge_size;
4451
4452         /* now do the same for a chain buffer */
4453         max_sge_elements = ioc->chain_segment_sz - sge_size;
4454         ioc->max_sges_in_chain_message = max_sge_elements/sge_size;
4455
4456         /*
4457          *  MPT3SAS_SG_DEPTH = CONFIG_FUSION_MAX_SGE
4458          */
4459         chains_needed_per_io = ((ioc->shost->sg_tablesize -
4460            ioc->max_sges_in_main_message)/ioc->max_sges_in_chain_message)
4461             + 1;
4462         if (chains_needed_per_io > facts->MaxChainDepth) {
4463                 chains_needed_per_io = facts->MaxChainDepth;
4464                 ioc->shost->sg_tablesize = min_t(u16,
4465                 ioc->max_sges_in_main_message + (ioc->max_sges_in_chain_message
4466                 * chains_needed_per_io), ioc->shost->sg_tablesize);
4467         }
4468         ioc->chains_needed_per_io = chains_needed_per_io;
4469
4470         /* reply free queue sizing - taking into account for 64 FW events */
4471         ioc->reply_free_queue_depth = ioc->hba_queue_depth + 64;
4472
4473         /* mCPU manage single counters for simplicity */
4474         if (ioc->is_mcpu_endpoint)
4475                 ioc->reply_post_queue_depth = ioc->reply_free_queue_depth;
4476         else {
4477                 /* calculate reply descriptor post queue depth */
4478                 ioc->reply_post_queue_depth = ioc->hba_queue_depth +
4479                         ioc->reply_free_queue_depth +  1;
4480                 /* align the reply post queue on the next 16 count boundary */
4481                 if (ioc->reply_post_queue_depth % 16)
4482                         ioc->reply_post_queue_depth += 16 -
4483                                 (ioc->reply_post_queue_depth % 16);
4484         }
4485
4486         if (ioc->reply_post_queue_depth >
4487             facts->MaxReplyDescriptorPostQueueDepth) {
4488                 ioc->reply_post_queue_depth =
4489                                 facts->MaxReplyDescriptorPostQueueDepth -
4490                     (facts->MaxReplyDescriptorPostQueueDepth % 16);
4491                 ioc->hba_queue_depth =
4492                                 ((ioc->reply_post_queue_depth - 64) / 2) - 1;
4493                 ioc->reply_free_queue_depth = ioc->hba_queue_depth + 64;
4494         }
4495
4496         dinitprintk(ioc, pr_info(MPT3SAS_FMT "scatter gather: " \
4497             "sge_in_main_msg(%d), sge_per_chain(%d), sge_per_io(%d), "
4498             "chains_per_io(%d)\n", ioc->name, ioc->max_sges_in_main_message,
4499             ioc->max_sges_in_chain_message, ioc->shost->sg_tablesize,
4500             ioc->chains_needed_per_io));
4501
4502         /* reply post queue, 16 byte align */
4503         reply_post_free_sz = ioc->reply_post_queue_depth *
4504             sizeof(Mpi2DefaultReplyDescriptor_t);
4505
4506         sz = reply_post_free_sz;
4507         if (_base_is_controller_msix_enabled(ioc) && !ioc->rdpq_array_enable)
4508                 sz *= ioc->reply_queue_count;
4509
4510         ioc->reply_post = kcalloc((ioc->rdpq_array_enable) ?
4511             (ioc->reply_queue_count):1,
4512             sizeof(struct reply_post_struct), GFP_KERNEL);
4513
4514         if (!ioc->reply_post) {
4515                 pr_err(MPT3SAS_FMT "reply_post_free pool: kcalloc failed\n",
4516                         ioc->name);
4517                 goto out;
4518         }
4519         ioc->reply_post_free_dma_pool = dma_pool_create("reply_post_free pool",
4520             &ioc->pdev->dev, sz, 16, 0);
4521         if (!ioc->reply_post_free_dma_pool) {
4522                 pr_err(MPT3SAS_FMT
4523                  "reply_post_free pool: dma_pool_create failed\n",
4524                  ioc->name);
4525                 goto out;
4526         }
4527         i = 0;
4528         do {
4529                 ioc->reply_post[i].reply_post_free =
4530                     dma_pool_alloc(ioc->reply_post_free_dma_pool,
4531                     GFP_KERNEL,
4532                     &ioc->reply_post[i].reply_post_free_dma);
4533                 if (!ioc->reply_post[i].reply_post_free) {
4534                         pr_err(MPT3SAS_FMT
4535                         "reply_post_free pool: dma_pool_alloc failed\n",
4536                         ioc->name);
4537                         goto out;
4538                 }
4539                 memset(ioc->reply_post[i].reply_post_free, 0, sz);
4540                 dinitprintk(ioc, pr_info(MPT3SAS_FMT
4541                     "reply post free pool (0x%p): depth(%d),"
4542                     "element_size(%d), pool_size(%d kB)\n", ioc->name,
4543                     ioc->reply_post[i].reply_post_free,
4544                     ioc->reply_post_queue_depth, 8, sz/1024));
4545                 dinitprintk(ioc, pr_info(MPT3SAS_FMT
4546                     "reply_post_free_dma = (0x%llx)\n", ioc->name,
4547                     (unsigned long long)
4548                     ioc->reply_post[i].reply_post_free_dma));
4549                 total_sz += sz;
4550         } while (ioc->rdpq_array_enable && (++i < ioc->reply_queue_count));
4551
4552         if (ioc->dma_mask > 32) {
4553                 if (_base_change_consistent_dma_mask(ioc, ioc->pdev) != 0) {
4554                         pr_warn(MPT3SAS_FMT
4555                             "no suitable consistent DMA mask for %s\n",
4556                             ioc->name, pci_name(ioc->pdev));
4557                         goto out;
4558                 }
4559         }
4560
4561         ioc->scsiio_depth = ioc->hba_queue_depth -
4562             ioc->hi_priority_depth - ioc->internal_depth;
4563
4564         /* set the scsi host can_queue depth
4565          * with some internal commands that could be outstanding
4566          */
4567         ioc->shost->can_queue = ioc->scsiio_depth - INTERNAL_SCSIIO_CMDS_COUNT;
4568         dinitprintk(ioc, pr_info(MPT3SAS_FMT
4569                 "scsi host: can_queue depth (%d)\n",
4570                 ioc->name, ioc->shost->can_queue));
4571
4572
4573         /* contiguous pool for request and chains, 16 byte align, one extra "
4574          * "frame for smid=0
4575          */
4576         ioc->chain_depth = ioc->chains_needed_per_io * ioc->scsiio_depth;
4577         sz = ((ioc->scsiio_depth + 1) * ioc->request_sz);
4578
4579         /* hi-priority queue */
4580         sz += (ioc->hi_priority_depth * ioc->request_sz);
4581
4582         /* internal queue */
4583         sz += (ioc->internal_depth * ioc->request_sz);
4584
4585         ioc->request_dma_sz = sz;
4586         ioc->request = pci_alloc_consistent(ioc->pdev, sz, &ioc->request_dma);
4587         if (!ioc->request) {
4588                 pr_err(MPT3SAS_FMT "request pool: pci_alloc_consistent " \
4589                     "failed: hba_depth(%d), chains_per_io(%d), frame_sz(%d), "
4590                     "total(%d kB)\n", ioc->name, ioc->hba_queue_depth,
4591                     ioc->chains_needed_per_io, ioc->request_sz, sz/1024);
4592                 if (ioc->scsiio_depth < MPT3SAS_SAS_QUEUE_DEPTH)
4593                         goto out;
4594                 retry_sz = 64;
4595                 ioc->hba_queue_depth -= retry_sz;
4596                 _base_release_memory_pools(ioc);
4597                 goto retry_allocation;
4598         }
4599
4600         if (retry_sz)
4601                 pr_err(MPT3SAS_FMT "request pool: pci_alloc_consistent " \
4602                     "succeed: hba_depth(%d), chains_per_io(%d), frame_sz(%d), "
4603                     "total(%d kb)\n", ioc->name, ioc->hba_queue_depth,
4604                     ioc->chains_needed_per_io, ioc->request_sz, sz/1024);
4605
4606         /* hi-priority queue */
4607         ioc->hi_priority = ioc->request + ((ioc->scsiio_depth + 1) *
4608             ioc->request_sz);
4609         ioc->hi_priority_dma = ioc->request_dma + ((ioc->scsiio_depth + 1) *
4610             ioc->request_sz);
4611
4612         /* internal queue */
4613         ioc->internal = ioc->hi_priority + (ioc->hi_priority_depth *
4614             ioc->request_sz);
4615         ioc->internal_dma = ioc->hi_priority_dma + (ioc->hi_priority_depth *
4616             ioc->request_sz);
4617
4618         dinitprintk(ioc, pr_info(MPT3SAS_FMT
4619                 "request pool(0x%p): depth(%d), frame_size(%d), pool_size(%d kB)\n",
4620                 ioc->name, ioc->request, ioc->hba_queue_depth, ioc->request_sz,
4621             (ioc->hba_queue_depth * ioc->request_sz)/1024));
4622
4623         dinitprintk(ioc, pr_info(MPT3SAS_FMT "request pool: dma(0x%llx)\n",
4624             ioc->name, (unsigned long long) ioc->request_dma));
4625         total_sz += sz;
4626
4627         dinitprintk(ioc, pr_info(MPT3SAS_FMT "scsiio(0x%p): depth(%d)\n",
4628                 ioc->name, ioc->request, ioc->scsiio_depth));
4629
4630         ioc->chain_depth = min_t(u32, ioc->chain_depth, MAX_CHAIN_DEPTH);
4631         sz = ioc->scsiio_depth * sizeof(struct chain_lookup);
4632         ioc->chain_lookup = kzalloc(sz, GFP_KERNEL);
4633         if (!ioc->chain_lookup) {
4634                 pr_err(MPT3SAS_FMT "chain_lookup: __get_free_pages "
4635                                 "failed\n", ioc->name);
4636                 goto out;
4637         }
4638
4639         sz = ioc->chains_needed_per_io * sizeof(struct chain_tracker);
4640         for (i = 0; i < ioc->scsiio_depth; i++) {
4641                 ioc->chain_lookup[i].chains_per_smid = kzalloc(sz, GFP_KERNEL);
4642                 if (!ioc->chain_lookup[i].chains_per_smid) {
4643                         pr_err(MPT3SAS_FMT "chain_lookup: "
4644                                         " kzalloc failed\n", ioc->name);
4645                         goto out;
4646                 }
4647         }
4648
4649         /* initialize hi-priority queue smid's */
4650         ioc->hpr_lookup = kcalloc(ioc->hi_priority_depth,
4651             sizeof(struct request_tracker), GFP_KERNEL);
4652         if (!ioc->hpr_lookup) {
4653                 pr_err(MPT3SAS_FMT "hpr_lookup: kcalloc failed\n",
4654                     ioc->name);
4655                 goto out;
4656         }
4657         ioc->hi_priority_smid = ioc->scsiio_depth + 1;
4658         dinitprintk(ioc, pr_info(MPT3SAS_FMT
4659                 "hi_priority(0x%p): depth(%d), start smid(%d)\n",
4660                 ioc->name, ioc->hi_priority,
4661             ioc->hi_priority_depth, ioc->hi_priority_smid));
4662
4663         /* initialize internal queue smid's */
4664         ioc->internal_lookup = kcalloc(ioc->internal_depth,
4665             sizeof(struct request_tracker), GFP_KERNEL);
4666         if (!ioc->internal_lookup) {
4667                 pr_err(MPT3SAS_FMT "internal_lookup: kcalloc failed\n",
4668                     ioc->name);
4669                 goto out;
4670         }
4671         ioc->internal_smid = ioc->hi_priority_smid + ioc->hi_priority_depth;
4672         dinitprintk(ioc, pr_info(MPT3SAS_FMT
4673                 "internal(0x%p): depth(%d), start smid(%d)\n",
4674                 ioc->name, ioc->internal,
4675             ioc->internal_depth, ioc->internal_smid));
4676         /*
4677          * The number of NVMe page sized blocks needed is:
4678          *     (((sg_tablesize * 8) - 1) / (page_size - 8)) + 1
4679          * ((sg_tablesize * 8) - 1) is the max PRP's minus the first PRP entry
4680          * that is placed in the main message frame.  8 is the size of each PRP
4681          * entry or PRP list pointer entry.  8 is subtracted from page_size
4682          * because of the PRP list pointer entry at the end of a page, so this
4683          * is not counted as a PRP entry.  The 1 added page is a round up.
4684          *
4685          * To avoid allocation failures due to the amount of memory that could
4686          * be required for NVMe PRP's, only each set of NVMe blocks will be
4687          * contiguous, so a new set is allocated for each possible I/O.
4688          */
4689         ioc->chains_per_prp_buffer = 0;
4690         if (ioc->facts.ProtocolFlags & MPI2_IOCFACTS_PROTOCOL_NVME_DEVICES) {
4691                 nvme_blocks_needed =
4692                         (ioc->shost->sg_tablesize * NVME_PRP_SIZE) - 1;
4693                 nvme_blocks_needed /= (ioc->page_size - NVME_PRP_SIZE);
4694                 nvme_blocks_needed++;
4695
4696                 sz = sizeof(struct pcie_sg_list) * ioc->scsiio_depth;
4697                 ioc->pcie_sg_lookup = kzalloc(sz, GFP_KERNEL);
4698                 if (!ioc->pcie_sg_lookup) {
4699                         pr_info(MPT3SAS_FMT
4700                             "PCIe SGL lookup: kzalloc failed\n", ioc->name);
4701                         goto out;
4702                 }
4703                 sz = nvme_blocks_needed * ioc->page_size;
4704                 ioc->pcie_sgl_dma_pool =
4705                         dma_pool_create("PCIe SGL pool", &ioc->pdev->dev, sz, 16, 0);
4706                 if (!ioc->pcie_sgl_dma_pool) {
4707                         pr_info(MPT3SAS_FMT
4708                             "PCIe SGL pool: dma_pool_create failed\n",
4709                             ioc->name);
4710                         goto out;
4711                 }
4712
4713                 ioc->chains_per_prp_buffer = sz/ioc->chain_segment_sz;
4714                 ioc->chains_per_prp_buffer = min(ioc->chains_per_prp_buffer,
4715                                                 ioc->chains_needed_per_io);
4716
4717                 for (i = 0; i < ioc->scsiio_depth; i++) {
4718                         ioc->pcie_sg_lookup[i].pcie_sgl = dma_pool_alloc(
4719                                 ioc->pcie_sgl_dma_pool, GFP_KERNEL,
4720                                 &ioc->pcie_sg_lookup[i].pcie_sgl_dma);
4721                         if (!ioc->pcie_sg_lookup[i].pcie_sgl) {
4722                                 pr_info(MPT3SAS_FMT
4723                                     "PCIe SGL pool: dma_pool_alloc failed\n",
4724                                     ioc->name);
4725                                 goto out;
4726                         }
4727                         for (j = 0; j < ioc->chains_per_prp_buffer; j++) {
4728                                 ct = &ioc->chain_lookup[i].chains_per_smid[j];
4729                                 ct->chain_buffer =
4730                                     ioc->pcie_sg_lookup[i].pcie_sgl +
4731                                     (j * ioc->chain_segment_sz);
4732                                 ct->chain_buffer_dma =
4733                                     ioc->pcie_sg_lookup[i].pcie_sgl_dma +
4734                                     (j * ioc->chain_segment_sz);
4735                         }
4736                 }
4737
4738                 dinitprintk(ioc, pr_info(MPT3SAS_FMT "PCIe sgl pool depth(%d), "
4739                         "element_size(%d), pool_size(%d kB)\n", ioc->name,
4740                         ioc->scsiio_depth, sz, (sz * ioc->scsiio_depth)/1024));
4741                 dinitprintk(ioc, pr_info(MPT3SAS_FMT "Number of chains can "
4742                     "fit in a PRP page(%d)\n", ioc->name,
4743                     ioc->chains_per_prp_buffer));
4744                 total_sz += sz * ioc->scsiio_depth;
4745         }
4746
4747         ioc->chain_dma_pool = dma_pool_create("chain pool", &ioc->pdev->dev,
4748             ioc->chain_segment_sz, 16, 0);
4749         if (!ioc->chain_dma_pool) {
4750                 pr_err(MPT3SAS_FMT "chain_dma_pool: dma_pool_create failed\n",
4751                         ioc->name);
4752                 goto out;
4753         }
4754         for (i = 0; i < ioc->scsiio_depth; i++) {
4755                 for (j = ioc->chains_per_prp_buffer;
4756                                 j < ioc->chains_needed_per_io; j++) {
4757                         ct = &ioc->chain_lookup[i].chains_per_smid[j];
4758                         ct->chain_buffer = dma_pool_alloc(
4759                                         ioc->chain_dma_pool, GFP_KERNEL,
4760                                         &ct->chain_buffer_dma);
4761                         if (!ct->chain_buffer) {
4762                                 pr_err(MPT3SAS_FMT "chain_lookup: "
4763                                 " pci_pool_alloc failed\n", ioc->name);
4764                                 _base_release_memory_pools(ioc);
4765                                 goto out;
4766                         }
4767                 }
4768                 total_sz += ioc->chain_segment_sz;
4769         }
4770
4771         dinitprintk(ioc, pr_info(MPT3SAS_FMT
4772                 "chain pool depth(%d), frame_size(%d), pool_size(%d kB)\n",
4773                 ioc->name, ioc->chain_depth, ioc->chain_segment_sz,
4774                 ((ioc->chain_depth *  ioc->chain_segment_sz))/1024));
4775
4776         /* sense buffers, 4 byte align */
4777         sz = ioc->scsiio_depth * SCSI_SENSE_BUFFERSIZE;
4778         ioc->sense_dma_pool = dma_pool_create("sense pool", &ioc->pdev->dev, sz,
4779                                               4, 0);
4780         if (!ioc->sense_dma_pool) {
4781                 pr_err(MPT3SAS_FMT "sense pool: dma_pool_create failed\n",
4782                     ioc->name);
4783                 goto out;
4784         }
4785         ioc->sense = dma_pool_alloc(ioc->sense_dma_pool, GFP_KERNEL,
4786             &ioc->sense_dma);
4787         if (!ioc->sense) {
4788                 pr_err(MPT3SAS_FMT "sense pool: dma_pool_alloc failed\n",
4789                     ioc->name);
4790                 goto out;
4791         }
4792         /* sense buffer requires to be in same 4 gb region.
4793          * Below function will check the same.
4794          * In case of failure, new pci pool will be created with updated
4795          * alignment. Older allocation and pool will be destroyed.
4796          * Alignment will be used such a way that next allocation if
4797          * success, will always meet same 4gb region requirement.
4798          * Actual requirement is not alignment, but we need start and end of
4799          * DMA address must have same upper 32 bit address.
4800          */
4801         if (!is_MSB_are_same((long)ioc->sense, sz)) {
4802                 //Release Sense pool & Reallocate
4803                 dma_pool_free(ioc->sense_dma_pool, ioc->sense, ioc->sense_dma);
4804                 dma_pool_destroy(ioc->sense_dma_pool);
4805                 ioc->sense = NULL;
4806
4807                 ioc->sense_dma_pool =
4808                         dma_pool_create("sense pool", &ioc->pdev->dev, sz,
4809                                                 roundup_pow_of_two(sz), 0);
4810                 if (!ioc->sense_dma_pool) {
4811                         pr_err(MPT3SAS_FMT "sense pool: pci_pool_create failed\n",
4812                                         ioc->name);
4813                         goto out;
4814                 }
4815                 ioc->sense = dma_pool_alloc(ioc->sense_dma_pool, GFP_KERNEL,
4816                                 &ioc->sense_dma);
4817                 if (!ioc->sense) {
4818                         pr_err(MPT3SAS_FMT "sense pool: pci_pool_alloc failed\n",
4819                                         ioc->name);
4820                         goto out;
4821                 }
4822         }
4823         dinitprintk(ioc, pr_info(MPT3SAS_FMT
4824             "sense pool(0x%p): depth(%d), element_size(%d), pool_size"
4825             "(%d kB)\n", ioc->name, ioc->sense, ioc->scsiio_depth,
4826             SCSI_SENSE_BUFFERSIZE, sz/1024));
4827         dinitprintk(ioc, pr_info(MPT3SAS_FMT "sense_dma(0x%llx)\n",
4828             ioc->name, (unsigned long long)ioc->sense_dma));
4829         total_sz += sz;
4830
4831         /* reply pool, 4 byte align */
4832         sz = ioc->reply_free_queue_depth * ioc->reply_sz;
4833         ioc->reply_dma_pool = dma_pool_create("reply pool", &ioc->pdev->dev, sz,
4834                                               4, 0);
4835         if (!ioc->reply_dma_pool) {
4836                 pr_err(MPT3SAS_FMT "reply pool: dma_pool_create failed\n",
4837                     ioc->name);
4838                 goto out;
4839         }
4840         ioc->reply = dma_pool_alloc(ioc->reply_dma_pool, GFP_KERNEL,
4841             &ioc->reply_dma);
4842         if (!ioc->reply) {
4843                 pr_err(MPT3SAS_FMT "reply pool: dma_pool_alloc failed\n",
4844                     ioc->name);
4845                 goto out;
4846         }
4847         ioc->reply_dma_min_address = (u32)(ioc->reply_dma);
4848         ioc->reply_dma_max_address = (u32)(ioc->reply_dma) + sz;
4849         dinitprintk(ioc, pr_info(MPT3SAS_FMT
4850                 "reply pool(0x%p): depth(%d), frame_size(%d), pool_size(%d kB)\n",
4851                 ioc->name, ioc->reply,
4852             ioc->reply_free_queue_depth, ioc->reply_sz, sz/1024));
4853         dinitprintk(ioc, pr_info(MPT3SAS_FMT "reply_dma(0x%llx)\n",
4854             ioc->name, (unsigned long long)ioc->reply_dma));
4855         total_sz += sz;
4856
4857         /* reply free queue, 16 byte align */
4858         sz = ioc->reply_free_queue_depth * 4;
4859         ioc->reply_free_dma_pool = dma_pool_create("reply_free pool",
4860             &ioc->pdev->dev, sz, 16, 0);
4861         if (!ioc->reply_free_dma_pool) {
4862                 pr_err(MPT3SAS_FMT "reply_free pool: dma_pool_create failed\n",
4863                         ioc->name);
4864                 goto out;
4865         }
4866         ioc->reply_free = dma_pool_alloc(ioc->reply_free_dma_pool, GFP_KERNEL,
4867             &ioc->reply_free_dma);
4868         if (!ioc->reply_free) {
4869                 pr_err(MPT3SAS_FMT "reply_free pool: dma_pool_alloc failed\n",
4870                         ioc->name);
4871                 goto out;
4872         }
4873         memset(ioc->reply_free, 0, sz);
4874         dinitprintk(ioc, pr_info(MPT3SAS_FMT "reply_free pool(0x%p): " \
4875             "depth(%d), element_size(%d), pool_size(%d kB)\n", ioc->name,
4876             ioc->reply_free, ioc->reply_free_queue_depth, 4, sz/1024));
4877         dinitprintk(ioc, pr_info(MPT3SAS_FMT
4878                 "reply_free_dma (0x%llx)\n",
4879                 ioc->name, (unsigned long long)ioc->reply_free_dma));
4880         total_sz += sz;
4881
4882         if (ioc->rdpq_array_enable) {
4883                 reply_post_free_array_sz = ioc->reply_queue_count *
4884                     sizeof(Mpi2IOCInitRDPQArrayEntry);
4885                 ioc->reply_post_free_array_dma_pool =
4886                     dma_pool_create("reply_post_free_array pool",
4887                     &ioc->pdev->dev, reply_post_free_array_sz, 16, 0);
4888                 if (!ioc->reply_post_free_array_dma_pool) {
4889                         dinitprintk(ioc,
4890                             pr_info(MPT3SAS_FMT "reply_post_free_array pool: "
4891                             "dma_pool_create failed\n", ioc->name));
4892                         goto out;
4893                 }
4894                 ioc->reply_post_free_array =
4895                     dma_pool_alloc(ioc->reply_post_free_array_dma_pool,
4896                     GFP_KERNEL, &ioc->reply_post_free_array_dma);
4897                 if (!ioc->reply_post_free_array) {
4898                         dinitprintk(ioc,
4899                             pr_info(MPT3SAS_FMT "reply_post_free_array pool: "
4900                             "dma_pool_alloc failed\n", ioc->name));
4901                         goto out;
4902                 }
4903         }
4904         ioc->config_page_sz = 512;
4905         ioc->config_page = pci_alloc_consistent(ioc->pdev,
4906             ioc->config_page_sz, &ioc->config_page_dma);
4907         if (!ioc->config_page) {
4908                 pr_err(MPT3SAS_FMT
4909                         "config page: dma_pool_alloc failed\n",
4910                         ioc->name);
4911                 goto out;
4912         }
4913         dinitprintk(ioc, pr_info(MPT3SAS_FMT
4914                 "config page(0x%p): size(%d)\n",
4915                 ioc->name, ioc->config_page, ioc->config_page_sz));
4916         dinitprintk(ioc, pr_info(MPT3SAS_FMT "config_page_dma(0x%llx)\n",
4917                 ioc->name, (unsigned long long)ioc->config_page_dma));
4918         total_sz += ioc->config_page_sz;
4919
4920         pr_info(MPT3SAS_FMT "Allocated physical memory: size(%d kB)\n",
4921             ioc->name, total_sz/1024);
4922         pr_info(MPT3SAS_FMT
4923                 "Current Controller Queue Depth(%d),Max Controller Queue Depth(%d)\n",
4924             ioc->name, ioc->shost->can_queue, facts->RequestCredit);
4925         pr_info(MPT3SAS_FMT "Scatter Gather Elements per IO(%d)\n",
4926             ioc->name, ioc->shost->sg_tablesize);
4927         return 0;
4928
4929  out:
4930         return -ENOMEM;
4931 }
4932
4933 /**
4934  * mpt3sas_base_get_iocstate - Get the current state of a MPT adapter.
4935  * @ioc: Pointer to MPT_ADAPTER structure
4936  * @cooked: Request raw or cooked IOC state
4937  *
4938  * Return: all IOC Doorbell register bits if cooked==0, else just the
4939  * Doorbell bits in MPI_IOC_STATE_MASK.
4940  */
4941 u32
4942 mpt3sas_base_get_iocstate(struct MPT3SAS_ADAPTER *ioc, int cooked)
4943 {
4944         u32 s, sc;
4945
4946         s = readl(&ioc->chip->Doorbell);
4947         sc = s & MPI2_IOC_STATE_MASK;
4948         return cooked ? sc : s;
4949 }
4950
4951 /**
4952  * _base_wait_on_iocstate - waiting on a particular ioc state
4953  * @ioc: ?
4954  * @ioc_state: controller state { READY, OPERATIONAL, or RESET }
4955  * @timeout: timeout in second
4956  *
4957  * Return: 0 for success, non-zero for failure.
4958  */
4959 static int
4960 _base_wait_on_iocstate(struct MPT3SAS_ADAPTER *ioc, u32 ioc_state, int timeout)
4961 {
4962         u32 count, cntdn;
4963         u32 current_state;
4964
4965         count = 0;
4966         cntdn = 1000 * timeout;
4967         do {
4968                 current_state = mpt3sas_base_get_iocstate(ioc, 1);
4969                 if (current_state == ioc_state)
4970                         return 0;
4971                 if (count && current_state == MPI2_IOC_STATE_FAULT)
4972                         break;
4973
4974                 usleep_range(1000, 1500);
4975                 count++;
4976         } while (--cntdn);
4977
4978         return current_state;
4979 }
4980
4981 /**
4982  * _base_wait_for_doorbell_int - waiting for controller interrupt(generated by
4983  * a write to the doorbell)
4984  * @ioc: per adapter object
4985  *
4986  * Return: 0 for success, non-zero for failure.
4987  *
4988  * Notes: MPI2_HIS_IOC2SYS_DB_STATUS - set to one when IOC writes to doorbell.
4989  */
4990 static int
4991 _base_diag_reset(struct MPT3SAS_ADAPTER *ioc);
4992
4993 static int
4994 _base_wait_for_doorbell_int(struct MPT3SAS_ADAPTER *ioc, int timeout)
4995 {
4996         u32 cntdn, count;
4997         u32 int_status;
4998
4999         count = 0;
5000         cntdn = 1000 * timeout;
5001         do {
5002                 int_status = readl(&ioc->chip->HostInterruptStatus);
5003                 if (int_status & MPI2_HIS_IOC2SYS_DB_STATUS) {
5004                         dhsprintk(ioc, pr_info(MPT3SAS_FMT
5005                                 "%s: successful count(%d), timeout(%d)\n",
5006                                 ioc->name, __func__, count, timeout));
5007                         return 0;
5008                 }
5009
5010                 usleep_range(1000, 1500);
5011                 count++;
5012         } while (--cntdn);
5013
5014         pr_err(MPT3SAS_FMT
5015                 "%s: failed due to timeout count(%d), int_status(%x)!\n",
5016                 ioc->name, __func__, count, int_status);
5017         return -EFAULT;
5018 }
5019
5020 static int
5021 _base_spin_on_doorbell_int(struct MPT3SAS_ADAPTER *ioc, int timeout)
5022 {
5023         u32 cntdn, count;
5024         u32 int_status;
5025
5026         count = 0;
5027         cntdn = 2000 * timeout;
5028         do {
5029                 int_status = readl(&ioc->chip->HostInterruptStatus);
5030                 if (int_status & MPI2_HIS_IOC2SYS_DB_STATUS) {
5031                         dhsprintk(ioc, pr_info(MPT3SAS_FMT
5032                                 "%s: successful count(%d), timeout(%d)\n",
5033                                 ioc->name, __func__, count, timeout));
5034                         return 0;
5035                 }
5036
5037                 udelay(500);
5038                 count++;
5039         } while (--cntdn);
5040
5041         pr_err(MPT3SAS_FMT
5042                 "%s: failed due to timeout count(%d), int_status(%x)!\n",
5043                 ioc->name, __func__, count, int_status);
5044         return -EFAULT;
5045
5046 }
5047
5048 /**
5049  * _base_wait_for_doorbell_ack - waiting for controller to read the doorbell.
5050  * @ioc: per adapter object
5051  * @timeout: timeout in second
5052  *
5053  * Return: 0 for success, non-zero for failure.
5054  *
5055  * Notes: MPI2_HIS_SYS2IOC_DB_STATUS - set to one when host writes to
5056  * doorbell.
5057  */
5058 static int
5059 _base_wait_for_doorbell_ack(struct MPT3SAS_ADAPTER *ioc, int timeout)
5060 {
5061         u32 cntdn, count;
5062         u32 int_status;
5063         u32 doorbell;
5064
5065         count = 0;
5066         cntdn = 1000 * timeout;
5067         do {
5068                 int_status = readl(&ioc->chip->HostInterruptStatus);
5069                 if (!(int_status & MPI2_HIS_SYS2IOC_DB_STATUS)) {
5070                         dhsprintk(ioc, pr_info(MPT3SAS_FMT
5071                                 "%s: successful count(%d), timeout(%d)\n",
5072                                 ioc->name, __func__, count, timeout));
5073                         return 0;
5074                 } else if (int_status & MPI2_HIS_IOC2SYS_DB_STATUS) {
5075                         doorbell = readl(&ioc->chip->Doorbell);
5076                         if ((doorbell & MPI2_IOC_STATE_MASK) ==
5077                             MPI2_IOC_STATE_FAULT) {
5078                                 mpt3sas_base_fault_info(ioc , doorbell);
5079                                 return -EFAULT;
5080                         }
5081                 } else if (int_status == 0xFFFFFFFF)
5082                         goto out;
5083
5084                 usleep_range(1000, 1500);
5085                 count++;
5086         } while (--cntdn);
5087
5088  out:
5089         pr_err(MPT3SAS_FMT
5090          "%s: failed due to timeout count(%d), int_status(%x)!\n",
5091          ioc->name, __func__, count, int_status);
5092         return -EFAULT;
5093 }
5094
5095 /**
5096  * _base_wait_for_doorbell_not_used - waiting for doorbell to not be in use
5097  * @ioc: per adapter object
5098  * @timeout: timeout in second
5099  *
5100  * Return: 0 for success, non-zero for failure.
5101  */
5102 static int
5103 _base_wait_for_doorbell_not_used(struct MPT3SAS_ADAPTER *ioc, int timeout)
5104 {
5105         u32 cntdn, count;
5106         u32 doorbell_reg;
5107
5108         count = 0;
5109         cntdn = 1000 * timeout;
5110         do {
5111                 doorbell_reg = readl(&ioc->chip->Doorbell);
5112                 if (!(doorbell_reg & MPI2_DOORBELL_USED)) {
5113                         dhsprintk(ioc, pr_info(MPT3SAS_FMT
5114                                 "%s: successful count(%d), timeout(%d)\n",
5115                                 ioc->name, __func__, count, timeout));
5116                         return 0;
5117                 }
5118
5119                 usleep_range(1000, 1500);
5120                 count++;
5121         } while (--cntdn);
5122
5123         pr_err(MPT3SAS_FMT
5124                 "%s: failed due to timeout count(%d), doorbell_reg(%x)!\n",
5125                 ioc->name, __func__, count, doorbell_reg);
5126         return -EFAULT;
5127 }
5128
5129 /**
5130  * _base_send_ioc_reset - send doorbell reset
5131  * @ioc: per adapter object
5132  * @reset_type: currently only supports: MPI2_FUNCTION_IOC_MESSAGE_UNIT_RESET
5133  * @timeout: timeout in second
5134  *
5135  * Return: 0 for success, non-zero for failure.
5136  */
5137 static int
5138 _base_send_ioc_reset(struct MPT3SAS_ADAPTER *ioc, u8 reset_type, int timeout)
5139 {
5140         u32 ioc_state;
5141         int r = 0;
5142
5143         if (reset_type != MPI2_FUNCTION_IOC_MESSAGE_UNIT_RESET) {
5144                 pr_err(MPT3SAS_FMT "%s: unknown reset_type\n",
5145                     ioc->name, __func__);
5146                 return -EFAULT;
5147         }
5148
5149         if (!(ioc->facts.IOCCapabilities &
5150            MPI2_IOCFACTS_CAPABILITY_EVENT_REPLAY))
5151                 return -EFAULT;
5152
5153         pr_info(MPT3SAS_FMT "sending message unit reset !!\n", ioc->name);
5154
5155         writel(reset_type << MPI2_DOORBELL_FUNCTION_SHIFT,
5156             &ioc->chip->Doorbell);
5157         if ((_base_wait_for_doorbell_ack(ioc, 15))) {
5158                 r = -EFAULT;
5159                 goto out;
5160         }
5161         ioc_state = _base_wait_on_iocstate(ioc, MPI2_IOC_STATE_READY, timeout);
5162         if (ioc_state) {
5163                 pr_err(MPT3SAS_FMT
5164                         "%s: failed going to ready state (ioc_state=0x%x)\n",
5165                         ioc->name, __func__, ioc_state);
5166                 r = -EFAULT;
5167                 goto out;
5168         }
5169  out:
5170         pr_info(MPT3SAS_FMT "message unit reset: %s\n",
5171             ioc->name, ((r == 0) ? "SUCCESS" : "FAILED"));
5172         return r;
5173 }
5174
5175 /**
5176  * _base_handshake_req_reply_wait - send request thru doorbell interface
5177  * @ioc: per adapter object
5178  * @request_bytes: request length
5179  * @request: pointer having request payload
5180  * @reply_bytes: reply length
5181  * @reply: pointer to reply payload
5182  * @timeout: timeout in second
5183  *
5184  * Return: 0 for success, non-zero for failure.
5185  */
5186 static int
5187 _base_handshake_req_reply_wait(struct MPT3SAS_ADAPTER *ioc, int request_bytes,
5188         u32 *request, int reply_bytes, u16 *reply, int timeout)
5189 {
5190         MPI2DefaultReply_t *default_reply = (MPI2DefaultReply_t *)reply;
5191         int i;
5192         u8 failed;
5193         __le32 *mfp;
5194
5195         /* make sure doorbell is not in use */
5196         if ((readl(&ioc->chip->Doorbell) & MPI2_DOORBELL_USED)) {
5197                 pr_err(MPT3SAS_FMT
5198                         "doorbell is in use (line=%d)\n",
5199                         ioc->name, __LINE__);
5200                 return -EFAULT;
5201         }
5202
5203         /* clear pending doorbell interrupts from previous state changes */
5204         if (readl(&ioc->chip->HostInterruptStatus) &
5205             MPI2_HIS_IOC2SYS_DB_STATUS)
5206                 writel(0, &ioc->chip->HostInterruptStatus);
5207
5208         /* send message to ioc */
5209         writel(((MPI2_FUNCTION_HANDSHAKE<<MPI2_DOORBELL_FUNCTION_SHIFT) |
5210             ((request_bytes/4)<<MPI2_DOORBELL_ADD_DWORDS_SHIFT)),
5211             &ioc->chip->Doorbell);
5212
5213         if ((_base_spin_on_doorbell_int(ioc, 5))) {
5214                 pr_err(MPT3SAS_FMT
5215                         "doorbell handshake int failed (line=%d)\n",
5216                         ioc->name, __LINE__);
5217                 return -EFAULT;
5218         }
5219         writel(0, &ioc->chip->HostInterruptStatus);
5220
5221         if ((_base_wait_for_doorbell_ack(ioc, 5))) {
5222                 pr_err(MPT3SAS_FMT
5223                         "doorbell handshake ack failed (line=%d)\n",
5224                         ioc->name, __LINE__);
5225                 return -EFAULT;
5226         }
5227
5228         /* send message 32-bits at a time */
5229         for (i = 0, failed = 0; i < request_bytes/4 && !failed; i++) {
5230                 writel(cpu_to_le32(request[i]), &ioc->chip->Doorbell);
5231                 if ((_base_wait_for_doorbell_ack(ioc, 5)))
5232                         failed = 1;
5233         }
5234
5235         if (failed) {
5236                 pr_err(MPT3SAS_FMT
5237                         "doorbell handshake sending request failed (line=%d)\n",
5238                         ioc->name, __LINE__);
5239                 return -EFAULT;
5240         }
5241
5242         /* now wait for the reply */
5243         if ((_base_wait_for_doorbell_int(ioc, timeout))) {
5244                 pr_err(MPT3SAS_FMT
5245                         "doorbell handshake int failed (line=%d)\n",
5246                         ioc->name, __LINE__);
5247                 return -EFAULT;
5248         }
5249
5250         /* read the first two 16-bits, it gives the total length of the reply */
5251         reply[0] = le16_to_cpu(readl(&ioc->chip->Doorbell)
5252             & MPI2_DOORBELL_DATA_MASK);
5253         writel(0, &ioc->chip->HostInterruptStatus);
5254         if ((_base_wait_for_doorbell_int(ioc, 5))) {
5255                 pr_err(MPT3SAS_FMT
5256                         "doorbell handshake int failed (line=%d)\n",
5257                         ioc->name, __LINE__);
5258                 return -EFAULT;
5259         }
5260         reply[1] = le16_to_cpu(readl(&ioc->chip->Doorbell)
5261             & MPI2_DOORBELL_DATA_MASK);
5262         writel(0, &ioc->chip->HostInterruptStatus);
5263
5264         for (i = 2; i < default_reply->MsgLength * 2; i++)  {
5265                 if ((_base_wait_for_doorbell_int(ioc, 5))) {
5266                         pr_err(MPT3SAS_FMT
5267                                 "doorbell handshake int failed (line=%d)\n",
5268                                 ioc->name, __LINE__);
5269                         return -EFAULT;
5270                 }
5271                 if (i >=  reply_bytes/2) /* overflow case */
5272                         readl(&ioc->chip->Doorbell);
5273                 else
5274                         reply[i] = le16_to_cpu(readl(&ioc->chip->Doorbell)
5275                             & MPI2_DOORBELL_DATA_MASK);
5276                 writel(0, &ioc->chip->HostInterruptStatus);
5277         }
5278
5279         _base_wait_for_doorbell_int(ioc, 5);
5280         if (_base_wait_for_doorbell_not_used(ioc, 5) != 0) {
5281                 dhsprintk(ioc, pr_info(MPT3SAS_FMT
5282                         "doorbell is in use (line=%d)\n", ioc->name, __LINE__));
5283         }
5284         writel(0, &ioc->chip->HostInterruptStatus);
5285
5286         if (ioc->logging_level & MPT_DEBUG_INIT) {
5287                 mfp = (__le32 *)reply;
5288                 pr_info("\toffset:data\n");
5289                 for (i = 0; i < reply_bytes/4; i++)
5290                         pr_info("\t[0x%02x]:%08x\n", i*4,
5291                             le32_to_cpu(mfp[i]));
5292         }
5293         return 0;
5294 }
5295
5296 /**
5297  * mpt3sas_base_sas_iounit_control - send sas iounit control to FW
5298  * @ioc: per adapter object
5299  * @mpi_reply: the reply payload from FW
5300  * @mpi_request: the request payload sent to FW
5301  *
5302  * The SAS IO Unit Control Request message allows the host to perform low-level
5303  * operations, such as resets on the PHYs of the IO Unit, also allows the host
5304  * to obtain the IOC assigned device handles for a device if it has other
5305  * identifying information about the device, in addition allows the host to
5306  * remove IOC resources associated with the device.
5307  *
5308  * Return: 0 for success, non-zero for failure.
5309  */
5310 int
5311 mpt3sas_base_sas_iounit_control(struct MPT3SAS_ADAPTER *ioc,
5312         Mpi2SasIoUnitControlReply_t *mpi_reply,
5313         Mpi2SasIoUnitControlRequest_t *mpi_request)
5314 {
5315         u16 smid;
5316         u32 ioc_state;
5317         u8 issue_reset = 0;
5318         int rc;
5319         void *request;
5320         u16 wait_state_count;
5321
5322         dinitprintk(ioc, pr_info(MPT3SAS_FMT "%s\n", ioc->name,
5323             __func__));
5324
5325         mutex_lock(&ioc->base_cmds.mutex);
5326
5327         if (ioc->base_cmds.status != MPT3_CMD_NOT_USED) {
5328                 pr_err(MPT3SAS_FMT "%s: base_cmd in use\n",
5329                     ioc->name, __func__);
5330                 rc = -EAGAIN;
5331                 goto out;
5332         }
5333
5334         wait_state_count = 0;
5335         ioc_state = mpt3sas_base_get_iocstate(ioc, 1);
5336         while (ioc_state != MPI2_IOC_STATE_OPERATIONAL) {
5337                 if (wait_state_count++ == 10) {
5338                         pr_err(MPT3SAS_FMT
5339                             "%s: failed due to ioc not operational\n",
5340                             ioc->name, __func__);
5341                         rc = -EFAULT;
5342                         goto out;
5343                 }
5344                 ssleep(1);
5345                 ioc_state = mpt3sas_base_get_iocstate(ioc, 1);
5346                 pr_info(MPT3SAS_FMT
5347                         "%s: waiting for operational state(count=%d)\n",
5348                         ioc->name, __func__, wait_state_count);
5349         }
5350
5351         smid = mpt3sas_base_get_smid(ioc, ioc->base_cb_idx);
5352         if (!smid) {
5353                 pr_err(MPT3SAS_FMT "%s: failed obtaining a smid\n",
5354                     ioc->name, __func__);
5355                 rc = -EAGAIN;
5356                 goto out;
5357         }
5358
5359         rc = 0;
5360         ioc->base_cmds.status = MPT3_CMD_PENDING;
5361         request = mpt3sas_base_get_msg_frame(ioc, smid);
5362         ioc->base_cmds.smid = smid;
5363         memcpy(request, mpi_request, sizeof(Mpi2SasIoUnitControlRequest_t));
5364         if (mpi_request->Operation == MPI2_SAS_OP_PHY_HARD_RESET ||
5365             mpi_request->Operation == MPI2_SAS_OP_PHY_LINK_RESET)
5366                 ioc->ioc_link_reset_in_progress = 1;
5367         init_completion(&ioc->base_cmds.done);
5368         mpt3sas_base_put_smid_default(ioc, smid);
5369         wait_for_completion_timeout(&ioc->base_cmds.done,
5370             msecs_to_jiffies(10000));
5371         if ((mpi_request->Operation == MPI2_SAS_OP_PHY_HARD_RESET ||
5372             mpi_request->Operation == MPI2_SAS_OP_PHY_LINK_RESET) &&
5373             ioc->ioc_link_reset_in_progress)
5374                 ioc->ioc_link_reset_in_progress = 0;
5375         if (!(ioc->base_cmds.status & MPT3_CMD_COMPLETE)) {
5376                 issue_reset =
5377                         mpt3sas_base_check_cmd_timeout(ioc,
5378                                 ioc->base_cmds.status, mpi_request,
5379                                 sizeof(Mpi2SasIoUnitControlRequest_t)/4);
5380                 goto issue_host_reset;
5381         }
5382         if (ioc->base_cmds.status & MPT3_CMD_REPLY_VALID)
5383                 memcpy(mpi_reply, ioc->base_cmds.reply,
5384                     sizeof(Mpi2SasIoUnitControlReply_t));
5385         else
5386                 memset(mpi_reply, 0, sizeof(Mpi2SasIoUnitControlReply_t));
5387         ioc->base_cmds.status = MPT3_CMD_NOT_USED;
5388         goto out;
5389
5390  issue_host_reset:
5391         if (issue_reset)
5392                 mpt3sas_base_hard_reset_handler(ioc, FORCE_BIG_HAMMER);
5393         ioc->base_cmds.status = MPT3_CMD_NOT_USED;
5394         rc = -EFAULT;
5395  out:
5396         mutex_unlock(&ioc->base_cmds.mutex);
5397         return rc;
5398 }
5399
5400 /**
5401  * mpt3sas_base_scsi_enclosure_processor - sending request to sep device
5402  * @ioc: per adapter object
5403  * @mpi_reply: the reply payload from FW
5404  * @mpi_request: the request payload sent to FW
5405  *
5406  * The SCSI Enclosure Processor request message causes the IOC to
5407  * communicate with SES devices to control LED status signals.
5408  *
5409  * Return: 0 for success, non-zero for failure.
5410  */
5411 int
5412 mpt3sas_base_scsi_enclosure_processor(struct MPT3SAS_ADAPTER *ioc,
5413         Mpi2SepReply_t *mpi_reply, Mpi2SepRequest_t *mpi_request)
5414 {
5415         u16 smid;
5416         u32 ioc_state;
5417         u8 issue_reset = 0;
5418         int rc;
5419         void *request;
5420         u16 wait_state_count;
5421
5422         dinitprintk(ioc, pr_info(MPT3SAS_FMT "%s\n", ioc->name,
5423             __func__));
5424
5425         mutex_lock(&ioc->base_cmds.mutex);
5426
5427         if (ioc->base_cmds.status != MPT3_CMD_NOT_USED) {
5428                 pr_err(MPT3SAS_FMT "%s: base_cmd in use\n",
5429                     ioc->name, __func__);
5430                 rc = -EAGAIN;
5431                 goto out;
5432         }
5433
5434         wait_state_count = 0;
5435         ioc_state = mpt3sas_base_get_iocstate(ioc, 1);
5436         while (ioc_state != MPI2_IOC_STATE_OPERATIONAL) {
5437                 if (wait_state_count++ == 10) {
5438                         pr_err(MPT3SAS_FMT
5439                             "%s: failed due to ioc not operational\n",
5440                             ioc->name, __func__);
5441                         rc = -EFAULT;
5442                         goto out;
5443                 }
5444                 ssleep(1);
5445                 ioc_state = mpt3sas_base_get_iocstate(ioc, 1);
5446                 pr_info(MPT3SAS_FMT
5447                         "%s: waiting for operational state(count=%d)\n",
5448                         ioc->name,
5449                     __func__, wait_state_count);
5450         }
5451
5452         smid = mpt3sas_base_get_smid(ioc, ioc->base_cb_idx);
5453         if (!smid) {
5454                 pr_err(MPT3SAS_FMT "%s: failed obtaining a smid\n",
5455                     ioc->name, __func__);
5456                 rc = -EAGAIN;
5457                 goto out;
5458         }
5459
5460         rc = 0;
5461         ioc->base_cmds.status = MPT3_CMD_PENDING;
5462         request = mpt3sas_base_get_msg_frame(ioc, smid);
5463         ioc->base_cmds.smid = smid;
5464         memcpy(request, mpi_request, sizeof(Mpi2SepReply_t));
5465         init_completion(&ioc->base_cmds.done);
5466         mpt3sas_base_put_smid_default(ioc, smid);
5467         wait_for_completion_timeout(&ioc->base_cmds.done,
5468             msecs_to_jiffies(10000));
5469         if (!(ioc->base_cmds.status & MPT3_CMD_COMPLETE)) {
5470                 issue_reset =
5471                         mpt3sas_base_check_cmd_timeout(ioc,
5472                                 ioc->base_cmds.status, mpi_request,
5473                                 sizeof(Mpi2SepRequest_t)/4);
5474                 goto issue_host_reset;
5475         }
5476         if (ioc->base_cmds.status & MPT3_CMD_REPLY_VALID)
5477                 memcpy(mpi_reply, ioc->base_cmds.reply,
5478                     sizeof(Mpi2SepReply_t));
5479         else
5480                 memset(mpi_reply, 0, sizeof(Mpi2SepReply_t));
5481         ioc->base_cmds.status = MPT3_CMD_NOT_USED;
5482         goto out;
5483
5484  issue_host_reset:
5485         if (issue_reset)
5486                 mpt3sas_base_hard_reset_handler(ioc, FORCE_BIG_HAMMER);
5487         ioc->base_cmds.status = MPT3_CMD_NOT_USED;
5488         rc = -EFAULT;
5489  out:
5490         mutex_unlock(&ioc->base_cmds.mutex);
5491         return rc;
5492 }
5493
5494 /**
5495  * _base_get_port_facts - obtain port facts reply and save in ioc
5496  * @ioc: per adapter object
5497  * @port: ?
5498  *
5499  * Return: 0 for success, non-zero for failure.
5500  */
5501 static int
5502 _base_get_port_facts(struct MPT3SAS_ADAPTER *ioc, int port)
5503 {
5504         Mpi2PortFactsRequest_t mpi_request;
5505         Mpi2PortFactsReply_t mpi_reply;
5506         struct mpt3sas_port_facts *pfacts;
5507         int mpi_reply_sz, mpi_request_sz, r;
5508
5509         dinitprintk(ioc, pr_info(MPT3SAS_FMT "%s\n", ioc->name,
5510             __func__));
5511
5512         mpi_reply_sz = sizeof(Mpi2PortFactsReply_t);
5513         mpi_request_sz = sizeof(Mpi2PortFactsRequest_t);
5514         memset(&mpi_request, 0, mpi_request_sz);
5515         mpi_request.Function = MPI2_FUNCTION_PORT_FACTS;
5516         mpi_request.PortNumber = port;
5517         r = _base_handshake_req_reply_wait(ioc, mpi_request_sz,
5518             (u32 *)&mpi_request, mpi_reply_sz, (u16 *)&mpi_reply, 5);
5519
5520         if (r != 0) {
5521                 pr_err(MPT3SAS_FMT "%s: handshake failed (r=%d)\n",
5522                     ioc->name, __func__, r);
5523                 return r;
5524         }
5525
5526         pfacts = &ioc->pfacts[port];
5527         memset(pfacts, 0, sizeof(struct mpt3sas_port_facts));
5528         pfacts->PortNumber = mpi_reply.PortNumber;
5529         pfacts->VP_ID = mpi_reply.VP_ID;
5530         pfacts->VF_ID = mpi_reply.VF_ID;
5531         pfacts->MaxPostedCmdBuffers =
5532             le16_to_cpu(mpi_reply.MaxPostedCmdBuffers);
5533
5534         return 0;
5535 }
5536
5537 /**
5538  * _base_wait_for_iocstate - Wait until the card is in READY or OPERATIONAL
5539  * @ioc: per adapter object
5540  * @timeout:
5541  *
5542  * Return: 0 for success, non-zero for failure.
5543  */
5544 static int
5545 _base_wait_for_iocstate(struct MPT3SAS_ADAPTER *ioc, int timeout)
5546 {
5547         u32 ioc_state;
5548         int rc;
5549
5550         dinitprintk(ioc, printk(MPT3SAS_FMT "%s\n", ioc->name,
5551             __func__));
5552
5553         if (ioc->pci_error_recovery) {
5554                 dfailprintk(ioc, printk(MPT3SAS_FMT
5555                     "%s: host in pci error recovery\n", ioc->name, __func__));
5556                 return -EFAULT;
5557         }
5558
5559         ioc_state = mpt3sas_base_get_iocstate(ioc, 0);
5560         dhsprintk(ioc, printk(MPT3SAS_FMT "%s: ioc_state(0x%08x)\n",
5561             ioc->name, __func__, ioc_state));
5562
5563         if (((ioc_state & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_READY) ||
5564             (ioc_state & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_OPERATIONAL)
5565                 return 0;
5566
5567         if (ioc_state & MPI2_DOORBELL_USED) {
5568                 dhsprintk(ioc, printk(MPT3SAS_FMT
5569                     "unexpected doorbell active!\n", ioc->name));
5570                 goto issue_diag_reset;
5571         }
5572
5573         if ((ioc_state & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_FAULT) {
5574                 mpt3sas_base_fault_info(ioc, ioc_state &
5575                     MPI2_DOORBELL_DATA_MASK);
5576                 goto issue_diag_reset;
5577         }
5578
5579         ioc_state = _base_wait_on_iocstate(ioc, MPI2_IOC_STATE_READY, timeout);
5580         if (ioc_state) {
5581                 dfailprintk(ioc, printk(MPT3SAS_FMT
5582                     "%s: failed going to ready state (ioc_state=0x%x)\n",
5583                     ioc->name, __func__, ioc_state));
5584                 return -EFAULT;
5585         }
5586
5587  issue_diag_reset:
5588         rc = _base_diag_reset(ioc);
5589         return rc;
5590 }
5591
5592 /**
5593  * _base_get_ioc_facts - obtain ioc facts reply and save in ioc
5594  * @ioc: per adapter object
5595  *
5596  * Return: 0 for success, non-zero for failure.
5597  */
5598 static int
5599 _base_get_ioc_facts(struct MPT3SAS_ADAPTER *ioc)
5600 {
5601         Mpi2IOCFactsRequest_t mpi_request;
5602         Mpi2IOCFactsReply_t mpi_reply;
5603         struct mpt3sas_facts *facts;
5604         int mpi_reply_sz, mpi_request_sz, r;
5605
5606         dinitprintk(ioc, pr_info(MPT3SAS_FMT "%s\n", ioc->name,
5607             __func__));
5608
5609         r = _base_wait_for_iocstate(ioc, 10);
5610         if (r) {
5611                 dfailprintk(ioc, printk(MPT3SAS_FMT
5612                     "%s: failed getting to correct state\n",
5613                     ioc->name, __func__));
5614                 return r;
5615         }
5616         mpi_reply_sz = sizeof(Mpi2IOCFactsReply_t);
5617         mpi_request_sz = sizeof(Mpi2IOCFactsRequest_t);
5618         memset(&mpi_request, 0, mpi_request_sz);
5619         mpi_request.Function = MPI2_FUNCTION_IOC_FACTS;
5620         r = _base_handshake_req_reply_wait(ioc, mpi_request_sz,
5621             (u32 *)&mpi_request, mpi_reply_sz, (u16 *)&mpi_reply, 5);
5622
5623         if (r != 0) {
5624                 pr_err(MPT3SAS_FMT "%s: handshake failed (r=%d)\n",
5625                     ioc->name, __func__, r);
5626                 return r;
5627         }
5628
5629         facts = &ioc->facts;
5630         memset(facts, 0, sizeof(struct mpt3sas_facts));
5631         facts->MsgVersion = le16_to_cpu(mpi_reply.MsgVersion);
5632         facts->HeaderVersion = le16_to_cpu(mpi_reply.HeaderVersion);
5633         facts->VP_ID = mpi_reply.VP_ID;
5634         facts->VF_ID = mpi_reply.VF_ID;
5635         facts->IOCExceptions = le16_to_cpu(mpi_reply.IOCExceptions);
5636         facts->MaxChainDepth = mpi_reply.MaxChainDepth;
5637         facts->WhoInit = mpi_reply.WhoInit;
5638         facts->NumberOfPorts = mpi_reply.NumberOfPorts;
5639         facts->MaxMSIxVectors = mpi_reply.MaxMSIxVectors;
5640         if (ioc->msix_enable && (facts->MaxMSIxVectors <=
5641             MAX_COMBINED_MSIX_VECTORS(ioc->is_gen35_ioc)))
5642                 ioc->combined_reply_queue = 0;
5643         facts->RequestCredit = le16_to_cpu(mpi_reply.RequestCredit);
5644         facts->MaxReplyDescriptorPostQueueDepth =
5645             le16_to_cpu(mpi_reply.MaxReplyDescriptorPostQueueDepth);
5646         facts->ProductID = le16_to_cpu(mpi_reply.ProductID);
5647         facts->IOCCapabilities = le32_to_cpu(mpi_reply.IOCCapabilities);
5648         if ((facts->IOCCapabilities & MPI2_IOCFACTS_CAPABILITY_INTEGRATED_RAID))
5649                 ioc->ir_firmware = 1;
5650         if ((facts->IOCCapabilities &
5651               MPI2_IOCFACTS_CAPABILITY_RDPQ_ARRAY_CAPABLE) && (!reset_devices))
5652                 ioc->rdpq_array_capable = 1;
5653         facts->FWVersion.Word = le32_to_cpu(mpi_reply.FWVersion.Word);
5654         facts->IOCRequestFrameSize =
5655             le16_to_cpu(mpi_reply.IOCRequestFrameSize);
5656         if (ioc->hba_mpi_version_belonged != MPI2_VERSION) {
5657                 facts->IOCMaxChainSegmentSize =
5658                         le16_to_cpu(mpi_reply.IOCMaxChainSegmentSize);
5659         }
5660         facts->MaxInitiators = le16_to_cpu(mpi_reply.MaxInitiators);
5661         facts->MaxTargets = le16_to_cpu(mpi_reply.MaxTargets);
5662         ioc->shost->max_id = -1;
5663         facts->MaxSasExpanders = le16_to_cpu(mpi_reply.MaxSasExpanders);
5664         facts->MaxEnclosures = le16_to_cpu(mpi_reply.MaxEnclosures);
5665         facts->ProtocolFlags = le16_to_cpu(mpi_reply.ProtocolFlags);
5666         facts->HighPriorityCredit =
5667             le16_to_cpu(mpi_reply.HighPriorityCredit);
5668         facts->ReplyFrameSize = mpi_reply.ReplyFrameSize;
5669         facts->MaxDevHandle = le16_to_cpu(mpi_reply.MaxDevHandle);
5670         facts->CurrentHostPageSize = mpi_reply.CurrentHostPageSize;
5671
5672         /*
5673          * Get the Page Size from IOC Facts. If it's 0, default to 4k.
5674          */
5675         ioc->page_size = 1 << facts->CurrentHostPageSize;
5676         if (ioc->page_size == 1) {
5677                 pr_info(MPT3SAS_FMT "CurrentHostPageSize is 0: Setting "
5678                         "default host page size to 4k\n", ioc->name);
5679                 ioc->page_size = 1 << MPT3SAS_HOST_PAGE_SIZE_4K;
5680         }
5681         dinitprintk(ioc, pr_info(MPT3SAS_FMT "CurrentHostPageSize(%d)\n",
5682                 ioc->name, facts->CurrentHostPageSize));
5683
5684         dinitprintk(ioc, pr_info(MPT3SAS_FMT
5685                 "hba queue depth(%d), max chains per io(%d)\n",
5686                 ioc->name, facts->RequestCredit,
5687             facts->MaxChainDepth));
5688         dinitprintk(ioc, pr_info(MPT3SAS_FMT
5689                 "request frame size(%d), reply frame size(%d)\n", ioc->name,
5690             facts->IOCRequestFrameSize * 4, facts->ReplyFrameSize * 4));
5691         return 0;
5692 }
5693
5694 /**
5695  * _base_send_ioc_init - send ioc_init to firmware
5696  * @ioc: per adapter object
5697  *
5698  * Return: 0 for success, non-zero for failure.
5699  */
5700 static int
5701 _base_send_ioc_init(struct MPT3SAS_ADAPTER *ioc)
5702 {
5703         Mpi2IOCInitRequest_t mpi_request;
5704         Mpi2IOCInitReply_t mpi_reply;
5705         int i, r = 0;
5706         ktime_t current_time;
5707         u16 ioc_status;
5708         u32 reply_post_free_array_sz = 0;
5709
5710         dinitprintk(ioc, pr_info(MPT3SAS_FMT "%s\n", ioc->name,
5711             __func__));
5712
5713         memset(&mpi_request, 0, sizeof(Mpi2IOCInitRequest_t));
5714         mpi_request.Function = MPI2_FUNCTION_IOC_INIT;
5715         mpi_request.WhoInit = MPI2_WHOINIT_HOST_DRIVER;
5716         mpi_request.VF_ID = 0; /* TODO */
5717         mpi_request.VP_ID = 0;
5718         mpi_request.MsgVersion = cpu_to_le16(ioc->hba_mpi_version_belonged);
5719         mpi_request.HeaderVersion = cpu_to_le16(MPI2_HEADER_VERSION);
5720         mpi_request.HostPageSize = MPT3SAS_HOST_PAGE_SIZE_4K;
5721
5722         if (_base_is_controller_msix_enabled(ioc))
5723                 mpi_request.HostMSIxVectors = ioc->reply_queue_count;
5724         mpi_request.SystemRequestFrameSize = cpu_to_le16(ioc->request_sz/4);
5725         mpi_request.ReplyDescriptorPostQueueDepth =
5726             cpu_to_le16(ioc->reply_post_queue_depth);
5727         mpi_request.ReplyFreeQueueDepth =
5728             cpu_to_le16(ioc->reply_free_queue_depth);
5729
5730         mpi_request.SenseBufferAddressHigh =
5731             cpu_to_le32((u64)ioc->sense_dma >> 32);
5732         mpi_request.SystemReplyAddressHigh =
5733             cpu_to_le32((u64)ioc->reply_dma >> 32);
5734         mpi_request.SystemRequestFrameBaseAddress =
5735             cpu_to_le64((u64)ioc->request_dma);
5736         mpi_request.ReplyFreeQueueAddress =
5737             cpu_to_le64((u64)ioc->reply_free_dma);
5738
5739         if (ioc->rdpq_array_enable) {
5740                 reply_post_free_array_sz = ioc->reply_queue_count *
5741                     sizeof(Mpi2IOCInitRDPQArrayEntry);
5742                 memset(ioc->reply_post_free_array, 0, reply_post_free_array_sz);
5743                 for (i = 0; i < ioc->reply_queue_count; i++)
5744                         ioc->reply_post_free_array[i].RDPQBaseAddress =
5745                             cpu_to_le64(
5746                                 (u64)ioc->reply_post[i].reply_post_free_dma);
5747                 mpi_request.MsgFlags = MPI2_IOCINIT_MSGFLAG_RDPQ_ARRAY_MODE;
5748                 mpi_request.ReplyDescriptorPostQueueAddress =
5749                     cpu_to_le64((u64)ioc->reply_post_free_array_dma);
5750         } else {
5751                 mpi_request.ReplyDescriptorPostQueueAddress =
5752                     cpu_to_le64((u64)ioc->reply_post[0].reply_post_free_dma);
5753         }
5754
5755         /* This time stamp specifies number of milliseconds
5756          * since epoch ~ midnight January 1, 1970.
5757          */
5758         current_time = ktime_get_real();
5759         mpi_request.TimeStamp = cpu_to_le64(ktime_to_ms(current_time));
5760
5761         if (ioc->logging_level & MPT_DEBUG_INIT) {
5762                 __le32 *mfp;
5763                 int i;
5764
5765                 mfp = (__le32 *)&mpi_request;
5766                 pr_info("\toffset:data\n");
5767                 for (i = 0; i < sizeof(Mpi2IOCInitRequest_t)/4; i++)
5768                         pr_info("\t[0x%02x]:%08x\n", i*4,
5769                             le32_to_cpu(mfp[i]));
5770         }
5771
5772         r = _base_handshake_req_reply_wait(ioc,
5773             sizeof(Mpi2IOCInitRequest_t), (u32 *)&mpi_request,
5774             sizeof(Mpi2IOCInitReply_t), (u16 *)&mpi_reply, 30);
5775
5776         if (r != 0) {
5777                 pr_err(MPT3SAS_FMT "%s: handshake failed (r=%d)\n",
5778                     ioc->name, __func__, r);
5779                 return r;
5780         }
5781
5782         ioc_status = le16_to_cpu(mpi_reply.IOCStatus) & MPI2_IOCSTATUS_MASK;
5783         if (ioc_status != MPI2_IOCSTATUS_SUCCESS ||
5784             mpi_reply.IOCLogInfo) {
5785                 pr_err(MPT3SAS_FMT "%s: failed\n", ioc->name, __func__);
5786                 r = -EIO;
5787         }
5788
5789         return r;
5790 }
5791
5792 /**
5793  * mpt3sas_port_enable_done - command completion routine for port enable
5794  * @ioc: per adapter object
5795  * @smid: system request message index
5796  * @msix_index: MSIX table index supplied by the OS
5797  * @reply: reply message frame(lower 32bit addr)
5798  *
5799  * Return: 1 meaning mf should be freed from _base_interrupt
5800  *          0 means the mf is freed from this function.
5801  */
5802 u8
5803 mpt3sas_port_enable_done(struct MPT3SAS_ADAPTER *ioc, u16 smid, u8 msix_index,
5804         u32 reply)
5805 {
5806         MPI2DefaultReply_t *mpi_reply;
5807         u16 ioc_status;
5808
5809         if (ioc->port_enable_cmds.status == MPT3_CMD_NOT_USED)
5810                 return 1;
5811
5812         mpi_reply = mpt3sas_base_get_reply_virt_addr(ioc, reply);
5813         if (!mpi_reply)
5814                 return 1;
5815
5816         if (mpi_reply->Function != MPI2_FUNCTION_PORT_ENABLE)
5817                 return 1;
5818
5819         ioc->port_enable_cmds.status &= ~MPT3_CMD_PENDING;
5820         ioc->port_enable_cmds.status |= MPT3_CMD_COMPLETE;
5821         ioc->port_enable_cmds.status |= MPT3_CMD_REPLY_VALID;
5822         memcpy(ioc->port_enable_cmds.reply, mpi_reply, mpi_reply->MsgLength*4);
5823         ioc_status = le16_to_cpu(mpi_reply->IOCStatus) & MPI2_IOCSTATUS_MASK;
5824         if (ioc_status != MPI2_IOCSTATUS_SUCCESS)
5825                 ioc->port_enable_failed = 1;
5826
5827         if (ioc->is_driver_loading) {
5828                 if (ioc_status == MPI2_IOCSTATUS_SUCCESS) {
5829                         mpt3sas_port_enable_complete(ioc);
5830                         return 1;
5831                 } else {
5832                         ioc->start_scan_failed = ioc_status;
5833                         ioc->start_scan = 0;
5834                         return 1;
5835                 }
5836         }
5837         complete(&ioc->port_enable_cmds.done);
5838         return 1;
5839 }
5840
5841 /**
5842  * _base_send_port_enable - send port_enable(discovery stuff) to firmware
5843  * @ioc: per adapter object
5844  *
5845  * Return: 0 for success, non-zero for failure.
5846  */
5847 static int
5848 _base_send_port_enable(struct MPT3SAS_ADAPTER *ioc)
5849 {
5850         Mpi2PortEnableRequest_t *mpi_request;
5851         Mpi2PortEnableReply_t *mpi_reply;
5852         int r = 0;
5853         u16 smid;
5854         u16 ioc_status;
5855
5856         pr_info(MPT3SAS_FMT "sending port enable !!\n", ioc->name);
5857
5858         if (ioc->port_enable_cmds.status & MPT3_CMD_PENDING) {
5859                 pr_err(MPT3SAS_FMT "%s: internal command already in use\n",
5860                     ioc->name, __func__);
5861                 return -EAGAIN;
5862         }
5863
5864         smid = mpt3sas_base_get_smid(ioc, ioc->port_enable_cb_idx);
5865         if (!smid) {
5866                 pr_err(MPT3SAS_FMT "%s: failed obtaining a smid\n",
5867                     ioc->name, __func__);
5868                 return -EAGAIN;
5869         }
5870
5871         ioc->port_enable_cmds.status = MPT3_CMD_PENDING;
5872         mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);
5873         ioc->port_enable_cmds.smid = smid;
5874         memset(mpi_request, 0, sizeof(Mpi2PortEnableRequest_t));
5875         mpi_request->Function = MPI2_FUNCTION_PORT_ENABLE;
5876
5877         init_completion(&ioc->port_enable_cmds.done);
5878         mpt3sas_base_put_smid_default(ioc, smid);
5879         wait_for_completion_timeout(&ioc->port_enable_cmds.done, 300*HZ);
5880         if (!(ioc->port_enable_cmds.status & MPT3_CMD_COMPLETE)) {
5881                 pr_err(MPT3SAS_FMT "%s: timeout\n",
5882                     ioc->name, __func__);
5883                 _debug_dump_mf(mpi_request,
5884                     sizeof(Mpi2PortEnableRequest_t)/4);
5885                 if (ioc->port_enable_cmds.status & MPT3_CMD_RESET)
5886                         r = -EFAULT;
5887                 else
5888                         r = -ETIME;
5889                 goto out;
5890         }
5891
5892         mpi_reply = ioc->port_enable_cmds.reply;
5893         ioc_status = le16_to_cpu(mpi_reply->IOCStatus) & MPI2_IOCSTATUS_MASK;
5894         if (ioc_status != MPI2_IOCSTATUS_SUCCESS) {
5895                 pr_err(MPT3SAS_FMT "%s: failed with (ioc_status=0x%08x)\n",
5896                     ioc->name, __func__, ioc_status);
5897                 r = -EFAULT;
5898                 goto out;
5899         }
5900
5901  out:
5902         ioc->port_enable_cmds.status = MPT3_CMD_NOT_USED;
5903         pr_info(MPT3SAS_FMT "port enable: %s\n", ioc->name, ((r == 0) ?
5904             "SUCCESS" : "FAILED"));
5905         return r;
5906 }
5907
5908 /**
5909  * mpt3sas_port_enable - initiate firmware discovery (don't wait for reply)
5910  * @ioc: per adapter object
5911  *
5912  * Return: 0 for success, non-zero for failure.
5913  */
5914 int
5915 mpt3sas_port_enable(struct MPT3SAS_ADAPTER *ioc)
5916 {
5917         Mpi2PortEnableRequest_t *mpi_request;
5918         u16 smid;
5919
5920         pr_info(MPT3SAS_FMT "sending port enable !!\n", ioc->name);
5921
5922         if (ioc->port_enable_cmds.status & MPT3_CMD_PENDING) {
5923                 pr_err(MPT3SAS_FMT "%s: internal command already in use\n",
5924                     ioc->name, __func__);
5925                 return -EAGAIN;
5926         }
5927
5928         smid = mpt3sas_base_get_smid(ioc, ioc->port_enable_cb_idx);
5929         if (!smid) {
5930                 pr_err(MPT3SAS_FMT "%s: failed obtaining a smid\n",
5931                     ioc->name, __func__);
5932                 return -EAGAIN;
5933         }
5934
5935         ioc->port_enable_cmds.status = MPT3_CMD_PENDING;
5936         mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);
5937         ioc->port_enable_cmds.smid = smid;
5938         memset(mpi_request, 0, sizeof(Mpi2PortEnableRequest_t));
5939         mpi_request->Function = MPI2_FUNCTION_PORT_ENABLE;
5940
5941         mpt3sas_base_put_smid_default(ioc, smid);
5942         return 0;
5943 }
5944
5945 /**
5946  * _base_determine_wait_on_discovery - desposition
5947  * @ioc: per adapter object
5948  *
5949  * Decide whether to wait on discovery to complete. Used to either
5950  * locate boot device, or report volumes ahead of physical devices.
5951  *
5952  * Return: 1 for wait, 0 for don't wait.
5953  */
5954 static int
5955 _base_determine_wait_on_discovery(struct MPT3SAS_ADAPTER *ioc)
5956 {
5957         /* We wait for discovery to complete if IR firmware is loaded.
5958          * The sas topology events arrive before PD events, so we need time to
5959          * turn on the bit in ioc->pd_handles to indicate PD
5960          * Also, it maybe required to report Volumes ahead of physical
5961          * devices when MPI2_IOCPAGE8_IRFLAGS_LOW_VOLUME_MAPPING is set.
5962          */
5963         if (ioc->ir_firmware)
5964                 return 1;
5965
5966         /* if no Bios, then we don't need to wait */
5967         if (!ioc->bios_pg3.BiosVersion)
5968                 return 0;
5969
5970         /* Bios is present, then we drop down here.
5971          *
5972          * If there any entries in the Bios Page 2, then we wait
5973          * for discovery to complete.
5974          */
5975
5976         /* Current Boot Device */
5977         if ((ioc->bios_pg2.CurrentBootDeviceForm &
5978             MPI2_BIOSPAGE2_FORM_MASK) ==
5979             MPI2_BIOSPAGE2_FORM_NO_DEVICE_SPECIFIED &&
5980         /* Request Boot Device */
5981            (ioc->bios_pg2.ReqBootDeviceForm &
5982             MPI2_BIOSPAGE2_FORM_MASK) ==
5983             MPI2_BIOSPAGE2_FORM_NO_DEVICE_SPECIFIED &&
5984         /* Alternate Request Boot Device */
5985            (ioc->bios_pg2.ReqAltBootDeviceForm &
5986             MPI2_BIOSPAGE2_FORM_MASK) ==
5987             MPI2_BIOSPAGE2_FORM_NO_DEVICE_SPECIFIED)
5988                 return 0;
5989
5990         return 1;
5991 }
5992
5993 /**
5994  * _base_unmask_events - turn on notification for this event
5995  * @ioc: per adapter object
5996  * @event: firmware event
5997  *
5998  * The mask is stored in ioc->event_masks.
5999  */
6000 static void
6001 _base_unmask_events(struct MPT3SAS_ADAPTER *ioc, u16 event)
6002 {
6003         u32 desired_event;
6004
6005         if (event >= 128)
6006                 return;
6007
6008         desired_event = (1 << (event % 32));
6009
6010         if (event < 32)
6011                 ioc->event_masks[0] &= ~desired_event;
6012         else if (event < 64)
6013                 ioc->event_masks[1] &= ~desired_event;
6014         else if (event < 96)
6015                 ioc->event_masks[2] &= ~desired_event;
6016         else if (event < 128)
6017                 ioc->event_masks[3] &= ~desired_event;
6018 }
6019
6020 /**
6021  * _base_event_notification - send event notification
6022  * @ioc: per adapter object
6023  *
6024  * Return: 0 for success, non-zero for failure.
6025  */
6026 static int
6027 _base_event_notification(struct MPT3SAS_ADAPTER *ioc)
6028 {
6029         Mpi2EventNotificationRequest_t *mpi_request;
6030         u16 smid;
6031         int r = 0;
6032         int i;
6033
6034         dinitprintk(ioc, pr_info(MPT3SAS_FMT "%s\n", ioc->name,
6035             __func__));
6036
6037         if (ioc->base_cmds.status & MPT3_CMD_PENDING) {
6038                 pr_err(MPT3SAS_FMT "%s: internal command already in use\n",
6039                     ioc->name, __func__);
6040                 return -EAGAIN;
6041         }
6042
6043         smid = mpt3sas_base_get_smid(ioc, ioc->base_cb_idx);
6044         if (!smid) {
6045                 pr_err(MPT3SAS_FMT "%s: failed obtaining a smid\n",
6046                     ioc->name, __func__);
6047                 return -EAGAIN;
6048         }
6049         ioc->base_cmds.status = MPT3_CMD_PENDING;
6050         mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);
6051         ioc->base_cmds.smid = smid;
6052         memset(mpi_request, 0, sizeof(Mpi2EventNotificationRequest_t));
6053         mpi_request->Function = MPI2_FUNCTION_EVENT_NOTIFICATION;
6054         mpi_request->VF_ID = 0; /* TODO */
6055         mpi_request->VP_ID = 0;
6056         for (i = 0; i < MPI2_EVENT_NOTIFY_EVENTMASK_WORDS; i++)
6057                 mpi_request->EventMasks[i] =
6058                     cpu_to_le32(ioc->event_masks[i]);
6059         init_completion(&ioc->base_cmds.done);
6060         mpt3sas_base_put_smid_default(ioc, smid);
6061         wait_for_completion_timeout(&ioc->base_cmds.done, 30*HZ);
6062         if (!(ioc->base_cmds.status & MPT3_CMD_COMPLETE)) {
6063                 pr_err(MPT3SAS_FMT "%s: timeout\n",
6064                     ioc->name, __func__);
6065                 _debug_dump_mf(mpi_request,
6066                     sizeof(Mpi2EventNotificationRequest_t)/4);
6067                 if (ioc->base_cmds.status & MPT3_CMD_RESET)
6068                         r = -EFAULT;
6069                 else
6070                         r = -ETIME;
6071         } else
6072                 dinitprintk(ioc, pr_info(MPT3SAS_FMT "%s: complete\n",
6073                     ioc->name, __func__));
6074         ioc->base_cmds.status = MPT3_CMD_NOT_USED;
6075         return r;
6076 }
6077
6078 /**
6079  * mpt3sas_base_validate_event_type - validating event types
6080  * @ioc: per adapter object
6081  * @event_type: firmware event
6082  *
6083  * This will turn on firmware event notification when application
6084  * ask for that event. We don't mask events that are already enabled.
6085  */
6086 void
6087 mpt3sas_base_validate_event_type(struct MPT3SAS_ADAPTER *ioc, u32 *event_type)
6088 {
6089         int i, j;
6090         u32 event_mask, desired_event;
6091         u8 send_update_to_fw;
6092
6093         for (i = 0, send_update_to_fw = 0; i <
6094             MPI2_EVENT_NOTIFY_EVENTMASK_WORDS; i++) {
6095                 event_mask = ~event_type[i];
6096                 desired_event = 1;
6097                 for (j = 0; j < 32; j++) {
6098                         if (!(event_mask & desired_event) &&
6099                             (ioc->event_masks[i] & desired_event)) {
6100                                 ioc->event_masks[i] &= ~desired_event;
6101                                 send_update_to_fw = 1;
6102                         }
6103                         desired_event = (desired_event << 1);
6104                 }
6105         }
6106
6107         if (!send_update_to_fw)
6108                 return;
6109
6110         mutex_lock(&ioc->base_cmds.mutex);
6111         _base_event_notification(ioc);
6112         mutex_unlock(&ioc->base_cmds.mutex);
6113 }
6114
6115 /**
6116  * _base_diag_reset - the "big hammer" start of day reset
6117  * @ioc: per adapter object
6118  *
6119  * Return: 0 for success, non-zero for failure.
6120  */
6121 static int
6122 _base_diag_reset(struct MPT3SAS_ADAPTER *ioc)
6123 {
6124         u32 host_diagnostic;
6125         u32 ioc_state;
6126         u32 count;
6127         u32 hcb_size;
6128
6129         pr_info(MPT3SAS_FMT "sending diag reset !!\n", ioc->name);
6130
6131         drsprintk(ioc, pr_info(MPT3SAS_FMT "clear interrupts\n",
6132             ioc->name));
6133
6134         count = 0;
6135         do {
6136                 /* Write magic sequence to WriteSequence register
6137                  * Loop until in diagnostic mode
6138                  */
6139                 drsprintk(ioc, pr_info(MPT3SAS_FMT
6140                         "write magic sequence\n", ioc->name));
6141                 writel(MPI2_WRSEQ_FLUSH_KEY_VALUE, &ioc->chip->WriteSequence);
6142                 writel(MPI2_WRSEQ_1ST_KEY_VALUE, &ioc->chip->WriteSequence);
6143                 writel(MPI2_WRSEQ_2ND_KEY_VALUE, &ioc->chip->WriteSequence);
6144                 writel(MPI2_WRSEQ_3RD_KEY_VALUE, &ioc->chip->WriteSequence);
6145                 writel(MPI2_WRSEQ_4TH_KEY_VALUE, &ioc->chip->WriteSequence);
6146                 writel(MPI2_WRSEQ_5TH_KEY_VALUE, &ioc->chip->WriteSequence);
6147                 writel(MPI2_WRSEQ_6TH_KEY_VALUE, &ioc->chip->WriteSequence);
6148
6149                 /* wait 100 msec */
6150                 msleep(100);
6151
6152                 if (count++ > 20)
6153                         goto out;
6154
6155                 host_diagnostic = readl(&ioc->chip->HostDiagnostic);
6156                 drsprintk(ioc, pr_info(MPT3SAS_FMT
6157                         "wrote magic sequence: count(%d), host_diagnostic(0x%08x)\n",
6158                     ioc->name, count, host_diagnostic));
6159
6160         } while ((host_diagnostic & MPI2_DIAG_DIAG_WRITE_ENABLE) == 0);
6161
6162         hcb_size = readl(&ioc->chip->HCBSize);
6163
6164         drsprintk(ioc, pr_info(MPT3SAS_FMT "diag reset: issued\n",
6165             ioc->name));
6166         writel(host_diagnostic | MPI2_DIAG_RESET_ADAPTER,
6167              &ioc->chip->HostDiagnostic);
6168
6169         /*This delay allows the chip PCIe hardware time to finish reset tasks*/
6170         msleep(MPI2_HARD_RESET_PCIE_FIRST_READ_DELAY_MICRO_SEC/1000);
6171
6172         /* Approximately 300 second max wait */
6173         for (count = 0; count < (300000000 /
6174                 MPI2_HARD_RESET_PCIE_SECOND_READ_DELAY_MICRO_SEC); count++) {
6175
6176                 host_diagnostic = readl(&ioc->chip->HostDiagnostic);
6177
6178                 if (host_diagnostic == 0xFFFFFFFF)
6179                         goto out;
6180                 if (!(host_diagnostic & MPI2_DIAG_RESET_ADAPTER))
6181                         break;
6182
6183                 msleep(MPI2_HARD_RESET_PCIE_SECOND_READ_DELAY_MICRO_SEC / 1000);
6184         }
6185
6186         if (host_diagnostic & MPI2_DIAG_HCB_MODE) {
6187
6188                 drsprintk(ioc, pr_info(MPT3SAS_FMT
6189                 "restart the adapter assuming the HCB Address points to good F/W\n",
6190                     ioc->name));
6191                 host_diagnostic &= ~MPI2_DIAG_BOOT_DEVICE_SELECT_MASK;
6192                 host_diagnostic |= MPI2_DIAG_BOOT_DEVICE_SELECT_HCDW;
6193                 writel(host_diagnostic, &ioc->chip->HostDiagnostic);
6194
6195                 drsprintk(ioc, pr_info(MPT3SAS_FMT
6196                     "re-enable the HCDW\n", ioc->name));
6197                 writel(hcb_size | MPI2_HCB_SIZE_HCB_ENABLE,
6198                     &ioc->chip->HCBSize);
6199         }
6200
6201         drsprintk(ioc, pr_info(MPT3SAS_FMT "restart the adapter\n",
6202             ioc->name));
6203         writel(host_diagnostic & ~MPI2_DIAG_HOLD_IOC_RESET,
6204             &ioc->chip->HostDiagnostic);
6205
6206         drsprintk(ioc, pr_info(MPT3SAS_FMT
6207                 "disable writes to the diagnostic register\n", ioc->name));
6208         writel(MPI2_WRSEQ_FLUSH_KEY_VALUE, &ioc->chip->WriteSequence);
6209
6210         drsprintk(ioc, pr_info(MPT3SAS_FMT
6211                 "Wait for FW to go to the READY state\n", ioc->name));
6212         ioc_state = _base_wait_on_iocstate(ioc, MPI2_IOC_STATE_READY, 20);
6213         if (ioc_state) {
6214                 pr_err(MPT3SAS_FMT
6215                         "%s: failed going to ready state (ioc_state=0x%x)\n",
6216                         ioc->name, __func__, ioc_state);
6217                 goto out;
6218         }
6219
6220         pr_info(MPT3SAS_FMT "diag reset: SUCCESS\n", ioc->name);
6221         return 0;
6222
6223  out:
6224         pr_err(MPT3SAS_FMT "diag reset: FAILED\n", ioc->name);
6225         return -EFAULT;
6226 }
6227
6228 /**
6229  * _base_make_ioc_ready - put controller in READY state
6230  * @ioc: per adapter object
6231  * @type: FORCE_BIG_HAMMER or SOFT_RESET
6232  *
6233  * Return: 0 for success, non-zero for failure.
6234  */
6235 static int
6236 _base_make_ioc_ready(struct MPT3SAS_ADAPTER *ioc, enum reset_type type)
6237 {
6238         u32 ioc_state;
6239         int rc;
6240         int count;
6241
6242         dinitprintk(ioc, pr_info(MPT3SAS_FMT "%s\n", ioc->name,
6243             __func__));
6244
6245         if (ioc->pci_error_recovery)
6246                 return 0;
6247
6248         ioc_state = mpt3sas_base_get_iocstate(ioc, 0);
6249         dhsprintk(ioc, pr_info(MPT3SAS_FMT "%s: ioc_state(0x%08x)\n",
6250             ioc->name, __func__, ioc_state));
6251
6252         /* if in RESET state, it should move to READY state shortly */
6253         count = 0;
6254         if ((ioc_state & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_RESET) {
6255                 while ((ioc_state & MPI2_IOC_STATE_MASK) !=
6256                     MPI2_IOC_STATE_READY) {
6257                         if (count++ == 10) {
6258                                 pr_err(MPT3SAS_FMT
6259                                         "%s: failed going to ready state (ioc_state=0x%x)\n",
6260                                     ioc->name, __func__, ioc_state);
6261                                 return -EFAULT;
6262                         }
6263                         ssleep(1);
6264                         ioc_state = mpt3sas_base_get_iocstate(ioc, 0);
6265                 }
6266         }
6267
6268         if ((ioc_state & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_READY)
6269                 return 0;
6270
6271         if (ioc_state & MPI2_DOORBELL_USED) {
6272                 dhsprintk(ioc, pr_info(MPT3SAS_FMT
6273                         "unexpected doorbell active!\n",
6274                         ioc->name));
6275                 goto issue_diag_reset;
6276         }
6277
6278         if ((ioc_state & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_FAULT) {
6279                 mpt3sas_base_fault_info(ioc, ioc_state &
6280                     MPI2_DOORBELL_DATA_MASK);
6281                 goto issue_diag_reset;
6282         }
6283
6284         if (type == FORCE_BIG_HAMMER)
6285                 goto issue_diag_reset;
6286
6287         if ((ioc_state & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_OPERATIONAL)
6288                 if (!(_base_send_ioc_reset(ioc,
6289                     MPI2_FUNCTION_IOC_MESSAGE_UNIT_RESET, 15))) {
6290                         return 0;
6291         }
6292
6293  issue_diag_reset:
6294         rc = _base_diag_reset(ioc);
6295         return rc;
6296 }
6297
6298 /**
6299  * _base_make_ioc_operational - put controller in OPERATIONAL state
6300  * @ioc: per adapter object
6301  *
6302  * Return: 0 for success, non-zero for failure.
6303  */
6304 static int
6305 _base_make_ioc_operational(struct MPT3SAS_ADAPTER *ioc)
6306 {
6307         int r, i, index;
6308         unsigned long   flags;
6309         u32 reply_address;
6310         u16 smid;
6311         struct _tr_list *delayed_tr, *delayed_tr_next;
6312         struct _sc_list *delayed_sc, *delayed_sc_next;
6313         struct _event_ack_list *delayed_event_ack, *delayed_event_ack_next;
6314         u8 hide_flag;
6315         struct adapter_reply_queue *reply_q;
6316         Mpi2ReplyDescriptorsUnion_t *reply_post_free_contig;
6317
6318         dinitprintk(ioc, pr_info(MPT3SAS_FMT "%s\n", ioc->name,
6319             __func__));
6320
6321         /* clean the delayed target reset list */
6322         list_for_each_entry_safe(delayed_tr, delayed_tr_next,
6323             &ioc->delayed_tr_list, list) {
6324                 list_del(&delayed_tr->list);
6325                 kfree(delayed_tr);
6326         }
6327
6328
6329         list_for_each_entry_safe(delayed_tr, delayed_tr_next,
6330             &ioc->delayed_tr_volume_list, list) {
6331                 list_del(&delayed_tr->list);
6332                 kfree(delayed_tr);
6333         }
6334
6335         list_for_each_entry_safe(delayed_sc, delayed_sc_next,
6336             &ioc->delayed_sc_list, list) {
6337                 list_del(&delayed_sc->list);
6338                 kfree(delayed_sc);
6339         }
6340
6341         list_for_each_entry_safe(delayed_event_ack, delayed_event_ack_next,
6342             &ioc->delayed_event_ack_list, list) {
6343                 list_del(&delayed_event_ack->list);
6344                 kfree(delayed_event_ack);
6345         }
6346
6347         spin_lock_irqsave(&ioc->scsi_lookup_lock, flags);
6348
6349         /* hi-priority queue */
6350         INIT_LIST_HEAD(&ioc->hpr_free_list);
6351         smid = ioc->hi_priority_smid;
6352         for (i = 0; i < ioc->hi_priority_depth; i++, smid++) {
6353                 ioc->hpr_lookup[i].cb_idx = 0xFF;
6354                 ioc->hpr_lookup[i].smid = smid;
6355                 list_add_tail(&ioc->hpr_lookup[i].tracker_list,
6356                     &ioc->hpr_free_list);
6357         }
6358
6359         /* internal queue */
6360         INIT_LIST_HEAD(&ioc->internal_free_list);
6361         smid = ioc->internal_smid;
6362         for (i = 0; i < ioc->internal_depth; i++, smid++) {
6363                 ioc->internal_lookup[i].cb_idx = 0xFF;
6364                 ioc->internal_lookup[i].smid = smid;
6365                 list_add_tail(&ioc->internal_lookup[i].tracker_list,
6366                     &ioc->internal_free_list);
6367         }
6368
6369         spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags);
6370
6371         /* initialize Reply Free Queue */
6372         for (i = 0, reply_address = (u32)ioc->reply_dma ;
6373             i < ioc->reply_free_queue_depth ; i++, reply_address +=
6374             ioc->reply_sz) {
6375                 ioc->reply_free[i] = cpu_to_le32(reply_address);
6376                 if (ioc->is_mcpu_endpoint)
6377                         _base_clone_reply_to_sys_mem(ioc,
6378                                         reply_address, i);
6379         }
6380
6381         /* initialize reply queues */
6382         if (ioc->is_driver_loading)
6383                 _base_assign_reply_queues(ioc);
6384
6385         /* initialize Reply Post Free Queue */
6386         index = 0;
6387         reply_post_free_contig = ioc->reply_post[0].reply_post_free;
6388         list_for_each_entry(reply_q, &ioc->reply_queue_list, list) {
6389                 /*
6390                  * If RDPQ is enabled, switch to the next allocation.
6391                  * Otherwise advance within the contiguous region.
6392                  */
6393                 if (ioc->rdpq_array_enable) {
6394                         reply_q->reply_post_free =
6395                                 ioc->reply_post[index++].reply_post_free;
6396                 } else {
6397                         reply_q->reply_post_free = reply_post_free_contig;
6398                         reply_post_free_contig += ioc->reply_post_queue_depth;
6399                 }
6400
6401                 reply_q->reply_post_host_index = 0;
6402                 for (i = 0; i < ioc->reply_post_queue_depth; i++)
6403                         reply_q->reply_post_free[i].Words =
6404                             cpu_to_le64(ULLONG_MAX);
6405                 if (!_base_is_controller_msix_enabled(ioc))
6406                         goto skip_init_reply_post_free_queue;
6407         }
6408  skip_init_reply_post_free_queue:
6409
6410         r = _base_send_ioc_init(ioc);
6411         if (r)
6412                 return r;
6413
6414         /* initialize reply free host index */
6415         ioc->reply_free_host_index = ioc->reply_free_queue_depth - 1;
6416         writel(ioc->reply_free_host_index, &ioc->chip->ReplyFreeHostIndex);
6417
6418         /* initialize reply post host index */
6419         list_for_each_entry(reply_q, &ioc->reply_queue_list, list) {
6420                 if (ioc->combined_reply_queue)
6421                         writel((reply_q->msix_index & 7)<<
6422                            MPI2_RPHI_MSIX_INDEX_SHIFT,
6423                            ioc->replyPostRegisterIndex[reply_q->msix_index/8]);
6424                 else
6425                         writel(reply_q->msix_index <<
6426                                 MPI2_RPHI_MSIX_INDEX_SHIFT,
6427                                 &ioc->chip->ReplyPostHostIndex);
6428
6429                 if (!_base_is_controller_msix_enabled(ioc))
6430                         goto skip_init_reply_post_host_index;
6431         }
6432
6433  skip_init_reply_post_host_index:
6434
6435         _base_unmask_interrupts(ioc);
6436
6437         if (ioc->hba_mpi_version_belonged != MPI2_VERSION) {
6438                 r = _base_display_fwpkg_version(ioc);
6439                 if (r)
6440                         return r;
6441         }
6442
6443         _base_static_config_pages(ioc);
6444         r = _base_event_notification(ioc);
6445         if (r)
6446                 return r;
6447
6448         if (ioc->is_driver_loading) {
6449
6450                 if (ioc->is_warpdrive && ioc->manu_pg10.OEMIdentifier
6451                     == 0x80) {
6452                         hide_flag = (u8) (
6453                             le32_to_cpu(ioc->manu_pg10.OEMSpecificFlags0) &
6454                             MFG_PAGE10_HIDE_SSDS_MASK);
6455                         if (hide_flag != MFG_PAGE10_HIDE_SSDS_MASK)
6456                                 ioc->mfg_pg10_hide_flag = hide_flag;
6457                 }
6458
6459                 ioc->wait_for_discovery_to_complete =
6460                     _base_determine_wait_on_discovery(ioc);
6461
6462                 return r; /* scan_start and scan_finished support */
6463         }
6464
6465         r = _base_send_port_enable(ioc);
6466         if (r)
6467                 return r;
6468
6469         return r;
6470 }
6471
6472 /**
6473  * mpt3sas_base_free_resources - free resources controller resources
6474  * @ioc: per adapter object
6475  */
6476 void
6477 mpt3sas_base_free_resources(struct MPT3SAS_ADAPTER *ioc)
6478 {
6479         dexitprintk(ioc, pr_info(MPT3SAS_FMT "%s\n", ioc->name,
6480             __func__));
6481
6482         /* synchronizing freeing resource with pci_access_mutex lock */
6483         mutex_lock(&ioc->pci_access_mutex);
6484         if (ioc->chip_phys && ioc->chip) {
6485                 _base_mask_interrupts(ioc);
6486                 ioc->shost_recovery = 1;
6487                 _base_make_ioc_ready(ioc, SOFT_RESET);
6488                 ioc->shost_recovery = 0;
6489         }
6490
6491         mpt3sas_base_unmap_resources(ioc);
6492         mutex_unlock(&ioc->pci_access_mutex);
6493         return;
6494 }
6495
6496 /**
6497  * mpt3sas_base_attach - attach controller instance
6498  * @ioc: per adapter object
6499  *
6500  * Return: 0 for success, non-zero for failure.
6501  */
6502 int
6503 mpt3sas_base_attach(struct MPT3SAS_ADAPTER *ioc)
6504 {
6505         int r, i;
6506         int cpu_id, last_cpu_id = 0;
6507
6508         dinitprintk(ioc, pr_info(MPT3SAS_FMT "%s\n", ioc->name,
6509             __func__));
6510
6511         /* setup cpu_msix_table */
6512         ioc->cpu_count = num_online_cpus();
6513         for_each_online_cpu(cpu_id)
6514                 last_cpu_id = cpu_id;
6515         ioc->cpu_msix_table_sz = last_cpu_id + 1;
6516         ioc->cpu_msix_table = kzalloc(ioc->cpu_msix_table_sz, GFP_KERNEL);
6517         ioc->reply_queue_count = 1;
6518         if (!ioc->cpu_msix_table) {
6519                 dfailprintk(ioc, pr_info(MPT3SAS_FMT
6520                         "allocation for cpu_msix_table failed!!!\n",
6521                         ioc->name));
6522                 r = -ENOMEM;
6523                 goto out_free_resources;
6524         }
6525
6526         if (ioc->is_warpdrive) {
6527                 ioc->reply_post_host_index = kcalloc(ioc->cpu_msix_table_sz,
6528                     sizeof(resource_size_t *), GFP_KERNEL);
6529                 if (!ioc->reply_post_host_index) {
6530                         dfailprintk(ioc, pr_info(MPT3SAS_FMT "allocation "
6531                                 "for reply_post_host_index failed!!!\n",
6532                                 ioc->name));
6533                         r = -ENOMEM;
6534                         goto out_free_resources;
6535                 }
6536         }
6537
6538         ioc->rdpq_array_enable_assigned = 0;
6539         ioc->dma_mask = 0;
6540         r = mpt3sas_base_map_resources(ioc);
6541         if (r)
6542                 goto out_free_resources;
6543
6544         pci_set_drvdata(ioc->pdev, ioc->shost);
6545         r = _base_get_ioc_facts(ioc);
6546         if (r)
6547                 goto out_free_resources;
6548
6549         switch (ioc->hba_mpi_version_belonged) {
6550         case MPI2_VERSION:
6551                 ioc->build_sg_scmd = &_base_build_sg_scmd;
6552                 ioc->build_sg = &_base_build_sg;
6553                 ioc->build_zero_len_sge = &_base_build_zero_len_sge;
6554                 break;
6555         case MPI25_VERSION:
6556         case MPI26_VERSION:
6557                 /*
6558                  * In SAS3.0,
6559                  * SCSI_IO, SMP_PASSTHRU, SATA_PASSTHRU, Target Assist, and
6560                  * Target Status - all require the IEEE formated scatter gather
6561                  * elements.
6562                  */
6563                 ioc->build_sg_scmd = &_base_build_sg_scmd_ieee;
6564                 ioc->build_sg = &_base_build_sg_ieee;
6565                 ioc->build_nvme_prp = &_base_build_nvme_prp;
6566                 ioc->build_zero_len_sge = &_base_build_zero_len_sge_ieee;
6567                 ioc->sge_size_ieee = sizeof(Mpi2IeeeSgeSimple64_t);
6568
6569                 break;
6570         }
6571
6572         if (ioc->is_mcpu_endpoint)
6573                 ioc->put_smid_scsi_io = &_base_put_smid_mpi_ep_scsi_io;
6574         else
6575                 ioc->put_smid_scsi_io = &_base_put_smid_scsi_io;
6576
6577         /*
6578          * These function pointers for other requests that don't
6579          * the require IEEE scatter gather elements.
6580          *
6581          * For example Configuration Pages and SAS IOUNIT Control don't.
6582          */
6583         ioc->build_sg_mpi = &_base_build_sg;
6584         ioc->build_zero_len_sge_mpi = &_base_build_zero_len_sge;
6585
6586         r = _base_make_ioc_ready(ioc, SOFT_RESET);
6587         if (r)
6588                 goto out_free_resources;
6589
6590         ioc->pfacts = kcalloc(ioc->facts.NumberOfPorts,
6591             sizeof(struct mpt3sas_port_facts), GFP_KERNEL);
6592         if (!ioc->pfacts) {
6593                 r = -ENOMEM;
6594                 goto out_free_resources;
6595         }
6596
6597         for (i = 0 ; i < ioc->facts.NumberOfPorts; i++) {
6598                 r = _base_get_port_facts(ioc, i);
6599                 if (r)
6600                         goto out_free_resources;
6601         }
6602
6603         r = _base_allocate_memory_pools(ioc);
6604         if (r)
6605                 goto out_free_resources;
6606
6607         init_waitqueue_head(&ioc->reset_wq);
6608
6609         /* allocate memory pd handle bitmask list */
6610         ioc->pd_handles_sz = (ioc->facts.MaxDevHandle / 8);
6611         if (ioc->facts.MaxDevHandle % 8)
6612                 ioc->pd_handles_sz++;
6613         ioc->pd_handles = kzalloc(ioc->pd_handles_sz,
6614             GFP_KERNEL);
6615         if (!ioc->pd_handles) {
6616                 r = -ENOMEM;
6617                 goto out_free_resources;
6618         }
6619         ioc->blocking_handles = kzalloc(ioc->pd_handles_sz,
6620             GFP_KERNEL);
6621         if (!ioc->blocking_handles) {
6622                 r = -ENOMEM;
6623                 goto out_free_resources;
6624         }
6625
6626         /* allocate memory for pending OS device add list */
6627         ioc->pend_os_device_add_sz = (ioc->facts.MaxDevHandle / 8);
6628         if (ioc->facts.MaxDevHandle % 8)
6629                 ioc->pend_os_device_add_sz++;
6630         ioc->pend_os_device_add = kzalloc(ioc->pend_os_device_add_sz,
6631             GFP_KERNEL);
6632         if (!ioc->pend_os_device_add) {
6633                 r = -ENOMEM;
6634                 goto out_free_resources;
6635         }
6636
6637         ioc->device_remove_in_progress_sz = ioc->pend_os_device_add_sz;
6638         ioc->device_remove_in_progress =
6639                 kzalloc(ioc->device_remove_in_progress_sz, GFP_KERNEL);
6640         if (!ioc->device_remove_in_progress) {
6641                 r = -ENOMEM;
6642                 goto out_free_resources;
6643         }
6644
6645         ioc->fwfault_debug = mpt3sas_fwfault_debug;
6646
6647         /* base internal command bits */
6648         mutex_init(&ioc->base_cmds.mutex);
6649         ioc->base_cmds.reply = kzalloc(ioc->reply_sz, GFP_KERNEL);
6650         ioc->base_cmds.status = MPT3_CMD_NOT_USED;
6651
6652         /* port_enable command bits */
6653         ioc->port_enable_cmds.reply = kzalloc(ioc->reply_sz, GFP_KERNEL);
6654         ioc->port_enable_cmds.status = MPT3_CMD_NOT_USED;
6655
6656         /* transport internal command bits */
6657         ioc->transport_cmds.reply = kzalloc(ioc->reply_sz, GFP_KERNEL);
6658         ioc->transport_cmds.status = MPT3_CMD_NOT_USED;
6659         mutex_init(&ioc->transport_cmds.mutex);
6660
6661         /* scsih internal command bits */
6662         ioc->scsih_cmds.reply = kzalloc(ioc->reply_sz, GFP_KERNEL);
6663         ioc->scsih_cmds.status = MPT3_CMD_NOT_USED;
6664         mutex_init(&ioc->scsih_cmds.mutex);
6665
6666         /* task management internal command bits */
6667         ioc->tm_cmds.reply = kzalloc(ioc->reply_sz, GFP_KERNEL);
6668         ioc->tm_cmds.status = MPT3_CMD_NOT_USED;
6669         mutex_init(&ioc->tm_cmds.mutex);
6670
6671         /* config page internal command bits */
6672         ioc->config_cmds.reply = kzalloc(ioc->reply_sz, GFP_KERNEL);
6673         ioc->config_cmds.status = MPT3_CMD_NOT_USED;
6674         mutex_init(&ioc->config_cmds.mutex);
6675
6676         /* ctl module internal command bits */
6677         ioc->ctl_cmds.reply = kzalloc(ioc->reply_sz, GFP_KERNEL);
6678         ioc->ctl_cmds.sense = kzalloc(SCSI_SENSE_BUFFERSIZE, GFP_KERNEL);
6679         ioc->ctl_cmds.status = MPT3_CMD_NOT_USED;
6680         mutex_init(&ioc->ctl_cmds.mutex);
6681
6682         if (!ioc->base_cmds.reply || !ioc->port_enable_cmds.reply ||
6683             !ioc->transport_cmds.reply || !ioc->scsih_cmds.reply ||
6684             !ioc->tm_cmds.reply || !ioc->config_cmds.reply ||
6685             !ioc->ctl_cmds.reply || !ioc->ctl_cmds.sense) {
6686                 r = -ENOMEM;
6687                 goto out_free_resources;
6688         }
6689
6690         for (i = 0; i < MPI2_EVENT_NOTIFY_EVENTMASK_WORDS; i++)
6691                 ioc->event_masks[i] = -1;
6692
6693         /* here we enable the events we care about */
6694         _base_unmask_events(ioc, MPI2_EVENT_SAS_DISCOVERY);
6695         _base_unmask_events(ioc, MPI2_EVENT_SAS_BROADCAST_PRIMITIVE);
6696         _base_unmask_events(ioc, MPI2_EVENT_SAS_TOPOLOGY_CHANGE_LIST);
6697         _base_unmask_events(ioc, MPI2_EVENT_SAS_DEVICE_STATUS_CHANGE);
6698         _base_unmask_events(ioc, MPI2_EVENT_SAS_ENCL_DEVICE_STATUS_CHANGE);
6699         _base_unmask_events(ioc, MPI2_EVENT_IR_CONFIGURATION_CHANGE_LIST);
6700         _base_unmask_events(ioc, MPI2_EVENT_IR_VOLUME);
6701         _base_unmask_events(ioc, MPI2_EVENT_IR_PHYSICAL_DISK);
6702         _base_unmask_events(ioc, MPI2_EVENT_IR_OPERATION_STATUS);
6703         _base_unmask_events(ioc, MPI2_EVENT_LOG_ENTRY_ADDED);
6704         _base_unmask_events(ioc, MPI2_EVENT_TEMP_THRESHOLD);
6705         _base_unmask_events(ioc, MPI2_EVENT_ACTIVE_CABLE_EXCEPTION);
6706         _base_unmask_events(ioc, MPI2_EVENT_SAS_DEVICE_DISCOVERY_ERROR);
6707         if (ioc->hba_mpi_version_belonged == MPI26_VERSION) {
6708                 if (ioc->is_gen35_ioc) {
6709                         _base_unmask_events(ioc,
6710                                 MPI2_EVENT_PCIE_DEVICE_STATUS_CHANGE);
6711                         _base_unmask_events(ioc, MPI2_EVENT_PCIE_ENUMERATION);
6712                         _base_unmask_events(ioc,
6713                                 MPI2_EVENT_PCIE_TOPOLOGY_CHANGE_LIST);
6714                 }
6715         }
6716         r = _base_make_ioc_operational(ioc);
6717         if (r)
6718                 goto out_free_resources;
6719
6720         ioc->non_operational_loop = 0;
6721         ioc->got_task_abort_from_ioctl = 0;
6722         return 0;
6723
6724  out_free_resources:
6725
6726         ioc->remove_host = 1;
6727
6728         mpt3sas_base_free_resources(ioc);
6729         _base_release_memory_pools(ioc);
6730         pci_set_drvdata(ioc->pdev, NULL);
6731         kfree(ioc->cpu_msix_table);
6732         if (ioc->is_warpdrive)
6733                 kfree(ioc->reply_post_host_index);
6734         kfree(ioc->pd_handles);
6735         kfree(ioc->blocking_handles);
6736         kfree(ioc->device_remove_in_progress);
6737         kfree(ioc->pend_os_device_add);
6738         kfree(ioc->tm_cmds.reply);
6739         kfree(ioc->transport_cmds.reply);
6740         kfree(ioc->scsih_cmds.reply);
6741         kfree(ioc->config_cmds.reply);
6742         kfree(ioc->base_cmds.reply);
6743         kfree(ioc->port_enable_cmds.reply);
6744         kfree(ioc->ctl_cmds.reply);
6745         kfree(ioc->ctl_cmds.sense);
6746         kfree(ioc->pfacts);
6747         ioc->ctl_cmds.reply = NULL;
6748         ioc->base_cmds.reply = NULL;
6749         ioc->tm_cmds.reply = NULL;
6750         ioc->scsih_cmds.reply = NULL;
6751         ioc->transport_cmds.reply = NULL;
6752         ioc->config_cmds.reply = NULL;
6753         ioc->pfacts = NULL;
6754         return r;
6755 }
6756
6757
6758 /**
6759  * mpt3sas_base_detach - remove controller instance
6760  * @ioc: per adapter object
6761  */
6762 void
6763 mpt3sas_base_detach(struct MPT3SAS_ADAPTER *ioc)
6764 {
6765         dexitprintk(ioc, pr_info(MPT3SAS_FMT "%s\n", ioc->name,
6766             __func__));
6767
6768         mpt3sas_base_stop_watchdog(ioc);
6769         mpt3sas_base_free_resources(ioc);
6770         _base_release_memory_pools(ioc);
6771         mpt3sas_free_enclosure_list(ioc);
6772         pci_set_drvdata(ioc->pdev, NULL);
6773         kfree(ioc->cpu_msix_table);
6774         if (ioc->is_warpdrive)
6775                 kfree(ioc->reply_post_host_index);
6776         kfree(ioc->pd_handles);
6777         kfree(ioc->blocking_handles);
6778         kfree(ioc->device_remove_in_progress);
6779         kfree(ioc->pend_os_device_add);
6780         kfree(ioc->pfacts);
6781         kfree(ioc->ctl_cmds.reply);
6782         kfree(ioc->ctl_cmds.sense);
6783         kfree(ioc->base_cmds.reply);
6784         kfree(ioc->port_enable_cmds.reply);
6785         kfree(ioc->tm_cmds.reply);
6786         kfree(ioc->transport_cmds.reply);
6787         kfree(ioc->scsih_cmds.reply);
6788         kfree(ioc->config_cmds.reply);
6789 }
6790
6791 /**
6792  * _base_pre_reset_handler - pre reset handler
6793  * @ioc: per adapter object
6794  */
6795 static void _base_pre_reset_handler(struct MPT3SAS_ADAPTER *ioc)
6796 {
6797         mpt3sas_scsih_pre_reset_handler(ioc);
6798         mpt3sas_ctl_pre_reset_handler(ioc);
6799         dtmprintk(ioc, pr_info(MPT3SAS_FMT
6800                         "%s: MPT3_IOC_PRE_RESET\n", ioc->name, __func__));
6801 }
6802
6803 /**
6804  * _base_after_reset_handler - after reset handler
6805  * @ioc: per adapter object
6806  */
6807 static void _base_after_reset_handler(struct MPT3SAS_ADAPTER *ioc)
6808 {
6809         mpt3sas_scsih_after_reset_handler(ioc);
6810         mpt3sas_ctl_after_reset_handler(ioc);
6811         dtmprintk(ioc, pr_info(MPT3SAS_FMT
6812                         "%s: MPT3_IOC_AFTER_RESET\n", ioc->name, __func__));
6813         if (ioc->transport_cmds.status & MPT3_CMD_PENDING) {
6814                 ioc->transport_cmds.status |= MPT3_CMD_RESET;
6815                 mpt3sas_base_free_smid(ioc, ioc->transport_cmds.smid);
6816                 complete(&ioc->transport_cmds.done);
6817         }
6818         if (ioc->base_cmds.status & MPT3_CMD_PENDING) {
6819                 ioc->base_cmds.status |= MPT3_CMD_RESET;
6820                 mpt3sas_base_free_smid(ioc, ioc->base_cmds.smid);
6821                 complete(&ioc->base_cmds.done);
6822         }
6823         if (ioc->port_enable_cmds.status & MPT3_CMD_PENDING) {
6824                 ioc->port_enable_failed = 1;
6825                 ioc->port_enable_cmds.status |= MPT3_CMD_RESET;
6826                 mpt3sas_base_free_smid(ioc, ioc->port_enable_cmds.smid);
6827                 if (ioc->is_driver_loading) {
6828                         ioc->start_scan_failed =
6829                                 MPI2_IOCSTATUS_INTERNAL_ERROR;
6830                         ioc->start_scan = 0;
6831                         ioc->port_enable_cmds.status =
6832                                 MPT3_CMD_NOT_USED;
6833                 } else {
6834                         complete(&ioc->port_enable_cmds.done);
6835                 }
6836         }
6837         if (ioc->config_cmds.status & MPT3_CMD_PENDING) {
6838                 ioc->config_cmds.status |= MPT3_CMD_RESET;
6839                 mpt3sas_base_free_smid(ioc, ioc->config_cmds.smid);
6840                 ioc->config_cmds.smid = USHRT_MAX;
6841                 complete(&ioc->config_cmds.done);
6842         }
6843 }
6844
6845 /**
6846  * _base_reset_done_handler - reset done handler
6847  * @ioc: per adapter object
6848  */
6849 static void _base_reset_done_handler(struct MPT3SAS_ADAPTER *ioc)
6850 {
6851         mpt3sas_scsih_reset_done_handler(ioc);
6852         mpt3sas_ctl_reset_done_handler(ioc);
6853         dtmprintk(ioc, pr_info(MPT3SAS_FMT
6854                         "%s: MPT3_IOC_DONE_RESET\n", ioc->name, __func__));
6855 }
6856
6857 /**
6858  * mpt3sas_wait_for_commands_to_complete - reset controller
6859  * @ioc: Pointer to MPT_ADAPTER structure
6860  *
6861  * This function is waiting 10s for all pending commands to complete
6862  * prior to putting controller in reset.
6863  */
6864 void
6865 mpt3sas_wait_for_commands_to_complete(struct MPT3SAS_ADAPTER *ioc)
6866 {
6867         u32 ioc_state;
6868
6869         ioc->pending_io_count = 0;
6870
6871         ioc_state = mpt3sas_base_get_iocstate(ioc, 0);
6872         if ((ioc_state & MPI2_IOC_STATE_MASK) != MPI2_IOC_STATE_OPERATIONAL)
6873                 return;
6874
6875         /* pending command count */
6876         ioc->pending_io_count = scsi_host_busy(ioc->shost);
6877
6878         if (!ioc->pending_io_count)
6879                 return;
6880
6881         /* wait for pending commands to complete */
6882         wait_event_timeout(ioc->reset_wq, ioc->pending_io_count == 0, 10 * HZ);
6883 }
6884
6885 /**
6886  * mpt3sas_base_hard_reset_handler - reset controller
6887  * @ioc: Pointer to MPT_ADAPTER structure
6888  * @type: FORCE_BIG_HAMMER or SOFT_RESET
6889  *
6890  * Return: 0 for success, non-zero for failure.
6891  */
6892 int
6893 mpt3sas_base_hard_reset_handler(struct MPT3SAS_ADAPTER *ioc,
6894         enum reset_type type)
6895 {
6896         int r;
6897         unsigned long flags;
6898         u32 ioc_state;
6899         u8 is_fault = 0, is_trigger = 0;
6900
6901         dtmprintk(ioc, pr_info(MPT3SAS_FMT "%s: enter\n", ioc->name,
6902             __func__));
6903
6904         if (ioc->pci_error_recovery) {
6905                 pr_err(MPT3SAS_FMT "%s: pci error recovery reset\n",
6906                     ioc->name, __func__);
6907                 r = 0;
6908                 goto out_unlocked;
6909         }
6910
6911         if (mpt3sas_fwfault_debug)
6912                 mpt3sas_halt_firmware(ioc);
6913
6914         /* wait for an active reset in progress to complete */
6915         mutex_lock(&ioc->reset_in_progress_mutex);
6916
6917         spin_lock_irqsave(&ioc->ioc_reset_in_progress_lock, flags);
6918         ioc->shost_recovery = 1;
6919         spin_unlock_irqrestore(&ioc->ioc_reset_in_progress_lock, flags);
6920
6921         if ((ioc->diag_buffer_status[MPI2_DIAG_BUF_TYPE_TRACE] &
6922             MPT3_DIAG_BUFFER_IS_REGISTERED) &&
6923             (!(ioc->diag_buffer_status[MPI2_DIAG_BUF_TYPE_TRACE] &
6924             MPT3_DIAG_BUFFER_IS_RELEASED))) {
6925                 is_trigger = 1;
6926                 ioc_state = mpt3sas_base_get_iocstate(ioc, 0);
6927                 if ((ioc_state & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_FAULT)
6928                         is_fault = 1;
6929         }
6930         _base_pre_reset_handler(ioc);
6931         mpt3sas_wait_for_commands_to_complete(ioc);
6932         _base_mask_interrupts(ioc);
6933         r = _base_make_ioc_ready(ioc, type);
6934         if (r)
6935                 goto out;
6936         _base_after_reset_handler(ioc);
6937
6938         /* If this hard reset is called while port enable is active, then
6939          * there is no reason to call make_ioc_operational
6940          */
6941         if (ioc->is_driver_loading && ioc->port_enable_failed) {
6942                 ioc->remove_host = 1;
6943                 r = -EFAULT;
6944                 goto out;
6945         }
6946         r = _base_get_ioc_facts(ioc);
6947         if (r)
6948                 goto out;
6949
6950         if (ioc->rdpq_array_enable && !ioc->rdpq_array_capable)
6951                 panic("%s: Issue occurred with flashing controller firmware."
6952                       "Please reboot the system and ensure that the correct"
6953                       " firmware version is running\n", ioc->name);
6954
6955         r = _base_make_ioc_operational(ioc);
6956         if (!r)
6957                 _base_reset_done_handler(ioc);
6958
6959  out:
6960         dtmprintk(ioc, pr_info(MPT3SAS_FMT "%s: %s\n",
6961             ioc->name, __func__, ((r == 0) ? "SUCCESS" : "FAILED")));
6962
6963         spin_lock_irqsave(&ioc->ioc_reset_in_progress_lock, flags);
6964         ioc->shost_recovery = 0;
6965         spin_unlock_irqrestore(&ioc->ioc_reset_in_progress_lock, flags);
6966         ioc->ioc_reset_count++;
6967         mutex_unlock(&ioc->reset_in_progress_mutex);
6968
6969  out_unlocked:
6970         if ((r == 0) && is_trigger) {
6971                 if (is_fault)
6972                         mpt3sas_trigger_master(ioc, MASTER_TRIGGER_FW_FAULT);
6973                 else
6974                         mpt3sas_trigger_master(ioc,
6975                             MASTER_TRIGGER_ADAPTER_RESET);
6976         }
6977         dtmprintk(ioc, pr_info(MPT3SAS_FMT "%s: exit\n", ioc->name,
6978             __func__));
6979         return r;
6980 }