drivers/infiniband/hw/hfi1/chip.c

   1 /*
   2  * Copyright(c) 2015, 2016 Intel Corporation.
   3  *
   4  * This file is provided under a dual BSD/GPLv2 license.  When using or
   5  * redistributing this file, you may do so under either license.
   6  *
   7  * GPL LICENSE SUMMARY
   8  *
   9  * This program is free software; you can redistribute it and/or modify
  10  * it under the terms of version 2 of the GNU General Public License as
  11  * published by the Free Software Foundation.
  12  *
  13  * This program is distributed in the hope that it will be useful, but
  14  * WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  16  * General Public License for more details.
  17  *
  18  * BSD LICENSE
  19  *
  20  * Redistribution and use in source and binary forms, with or without
  21  * modification, are permitted provided that the following conditions
  22  * are met:
  23  *
  24  *  - Redistributions of source code must retain the above copyright
  25  *    notice, this list of conditions and the following disclaimer.
  26  *  - Redistributions in binary form must reproduce the above copyright
  27  *    notice, this list of conditions and the following disclaimer in
  28  *    the documentation and/or other materials provided with the
  29  *    distribution.
  30  *  - Neither the name of Intel Corporation nor the names of its
  31  *    contributors may be used to endorse or promote products derived
  32  *    from this software without specific prior written permission.
  33  *
  34  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  35  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  36  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  37  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  38  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  39  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  40  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  41  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  42  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  43  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  44  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  45  *
  46  */
  47
  48 /*
  49  * This file contains all of the code that is specific to the HFI chip
  50  */
  51
  52 #include <linux/pci.h>
  53 #include <linux/delay.h>
  54 #include <linux/interrupt.h>
  55 #include <linux/module.h>
  56
  57 #include "hfi.h"
  58 #include "trace.h"
  59 #include "mad.h"
  60 #include "pio.h"
  61 #include "sdma.h"
  62 #include "eprom.h"
  63 #include "efivar.h"
  64 #include "platform.h"
  65 #include "aspm.h"
  66 #include "affinity.h"
  67
  68 #define NUM_IB_PORTS 1
  69
  70 uint kdeth_qp;
  71 module_param_named(kdeth_qp, kdeth_qp, uint, S_IRUGO);
  72 MODULE_PARM_DESC(kdeth_qp, "Set the KDETH queue pair prefix");
  73
  74 uint num_vls = HFI1_MAX_VLS_SUPPORTED;
  75 module_param(num_vls, uint, S_IRUGO);
  76 MODULE_PARM_DESC(num_vls, "Set number of Virtual Lanes to use (1-8)");
  77
  78 /*
  79  * Default time to aggregate two 10K packets from the idle state
  80  * (timer not running). The timer starts at the end of the first packet,
  81  * so only the time for one 10K packet and header plus a bit extra is needed.
  82  * 10 * 1024 + 64 header byte = 10304 byte
  83  * 10304 byte / 12.5 GB/s = 824.32ns
  84  */
  85 uint rcv_intr_timeout = (824 + 16); /* 16 is for coalescing interrupt */
  86 module_param(rcv_intr_timeout, uint, S_IRUGO);
  87 MODULE_PARM_DESC(rcv_intr_timeout, "Receive interrupt mitigation timeout in ns");
  88
  89 uint rcv_intr_count = 16; /* same as qib */
  90 module_param(rcv_intr_count, uint, S_IRUGO);
  91 MODULE_PARM_DESC(rcv_intr_count, "Receive interrupt mitigation count");
  92
  93 ushort link_crc_mask = SUPPORTED_CRCS;
  94 module_param(link_crc_mask, ushort, S_IRUGO);
  95 MODULE_PARM_DESC(link_crc_mask, "CRCs to use on the link");
  96
  97 uint loopback;
  98 module_param_named(loopback, loopback, uint, S_IRUGO);
  99 MODULE_PARM_DESC(loopback, "Put into loopback mode (1 = serdes, 3 = external cable");
 100
 101 /* Other driver tunables */
 102 uint rcv_intr_dynamic = 1; /* enable dynamic mode for rcv int mitigation*/
 103 static ushort crc_14b_sideband = 1;
 104 static uint use_flr = 1;
 105 uint quick_linkup; /* skip LNI */
 106
 107 struct flag_table {
 108         u64 flag;       /* the flag */
 109         char *str;      /* description string */
 110         u16 extra;      /* extra information */
 111         u16 unused0;
 112         u32 unused1;
 113 };
 114
 115 /* str must be a string constant */
 116 #define FLAG_ENTRY(str, extra, flag) {flag, str, extra}
 117 #define FLAG_ENTRY0(str, flag) {flag, str, 0}
 118
 119 /* Send Error Consequences */
 120 #define SEC_WRITE_DROPPED       0x1
 121 #define SEC_PACKET_DROPPED      0x2
 122 #define SEC_SC_HALTED           0x4     /* per-context only */
 123 #define SEC_SPC_FREEZE          0x8     /* per-HFI only */
 124
 125 #define DEFAULT_KRCVQS            2
 126 #define MIN_KERNEL_KCTXTS         2
 127 #define FIRST_KERNEL_KCTXT        1
 128 /* sizes for both the QP and RSM map tables */
 129 #define NUM_MAP_ENTRIES         256
 130 #define NUM_MAP_REGS             32
 131
 132 /* Bit offset into the GUID which carries HFI id information */
 133 #define GUID_HFI_INDEX_SHIFT     39
 134
 135 /* extract the emulation revision */
 136 #define emulator_rev(dd) ((dd)->irev >> 8)
 137 /* parallel and serial emulation versions are 3 and 4 respectively */
 138 #define is_emulator_p(dd) ((((dd)->irev) & 0xf) == 3)
 139 #define is_emulator_s(dd) ((((dd)->irev) & 0xf) == 4)
 140
 141 /* RSM fields */
 142
 143 /* packet type */
 144 #define IB_PACKET_TYPE         2ull
 145 #define QW_SHIFT               6ull
 146 /* QPN[7..1] */
 147 #define QPN_WIDTH              7ull
 148
 149 /* LRH.BTH: QW 0, OFFSET 48 - for match */
 150 #define LRH_BTH_QW             0ull
 151 #define LRH_BTH_BIT_OFFSET     48ull
 152 #define LRH_BTH_OFFSET(off)    ((LRH_BTH_QW << QW_SHIFT) | (off))
 153 #define LRH_BTH_MATCH_OFFSET   LRH_BTH_OFFSET(LRH_BTH_BIT_OFFSET)
 154 #define LRH_BTH_SELECT
 155 #define LRH_BTH_MASK           3ull
 156 #define LRH_BTH_VALUE          2ull
 157
 158 /* LRH.SC[3..0] QW 0, OFFSET 56 - for match */
 159 #define LRH_SC_QW              0ull
 160 #define LRH_SC_BIT_OFFSET      56ull
 161 #define LRH_SC_OFFSET(off)     ((LRH_SC_QW << QW_SHIFT) | (off))
 162 #define LRH_SC_MATCH_OFFSET    LRH_SC_OFFSET(LRH_SC_BIT_OFFSET)
 163 #define LRH_SC_MASK            128ull
 164 #define LRH_SC_VALUE           0ull
 165
 166 /* SC[n..0] QW 0, OFFSET 60 - for select */
 167 #define LRH_SC_SELECT_OFFSET  ((LRH_SC_QW << QW_SHIFT) | (60ull))
 168
 169 /* QPN[m+n:1] QW 1, OFFSET 1 */
 170 #define QPN_SELECT_OFFSET      ((1ull << QW_SHIFT) | (1ull))
 171
 172 /* defines to build power on SC2VL table */
 173 #define SC2VL_VAL( \
 174         num, \
 175         sc0, sc0val, \
 176         sc1, sc1val, \
 177         sc2, sc2val, \
 178         sc3, sc3val, \
 179         sc4, sc4val, \
 180         sc5, sc5val, \
 181         sc6, sc6val, \
 182         sc7, sc7val) \
 183 ( \
 184         ((u64)(sc0val) << SEND_SC2VLT##num##_SC##sc0##_SHIFT) | \
 185         ((u64)(sc1val) << SEND_SC2VLT##num##_SC##sc1##_SHIFT) | \
 186         ((u64)(sc2val) << SEND_SC2VLT##num##_SC##sc2##_SHIFT) | \
 187         ((u64)(sc3val) << SEND_SC2VLT##num##_SC##sc3##_SHIFT) | \
 188         ((u64)(sc4val) << SEND_SC2VLT##num##_SC##sc4##_SHIFT) | \
 189         ((u64)(sc5val) << SEND_SC2VLT##num##_SC##sc5##_SHIFT) | \
 190         ((u64)(sc6val) << SEND_SC2VLT##num##_SC##sc6##_SHIFT) | \
 191         ((u64)(sc7val) << SEND_SC2VLT##num##_SC##sc7##_SHIFT)   \
 192 )
 193
 194 #define DC_SC_VL_VAL( \
 195         range, \
 196         e0, e0val, \
 197         e1, e1val, \
 198         e2, e2val, \
 199         e3, e3val, \
 200         e4, e4val, \
 201         e5, e5val, \
 202         e6, e6val, \
 203         e7, e7val, \
 204         e8, e8val, \
 205         e9, e9val, \
 206         e10, e10val, \
 207         e11, e11val, \
 208         e12, e12val, \
 209         e13, e13val, \
 210         e14, e14val, \
 211         e15, e15val) \
 212 ( \
 213         ((u64)(e0val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e0##_SHIFT) | \
 214         ((u64)(e1val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e1##_SHIFT) | \
 215         ((u64)(e2val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e2##_SHIFT) | \
 216         ((u64)(e3val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e3##_SHIFT) | \
 217         ((u64)(e4val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e4##_SHIFT) | \
 218         ((u64)(e5val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e5##_SHIFT) | \
 219         ((u64)(e6val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e6##_SHIFT) | \
 220         ((u64)(e7val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e7##_SHIFT) | \
 221         ((u64)(e8val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e8##_SHIFT) | \
 222         ((u64)(e9val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e9##_SHIFT) | \
 223         ((u64)(e10val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e10##_SHIFT) | \
 224         ((u64)(e11val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e11##_SHIFT) | \
 225         ((u64)(e12val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e12##_SHIFT) | \
 226         ((u64)(e13val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e13##_SHIFT) | \
 227         ((u64)(e14val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e14##_SHIFT) | \
 228         ((u64)(e15val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e15##_SHIFT) \
 229 )
 230
 231 /* all CceStatus sub-block freeze bits */
 232 #define ALL_FROZE (CCE_STATUS_SDMA_FROZE_SMASK \
 233                         | CCE_STATUS_RXE_FROZE_SMASK \
 234                         | CCE_STATUS_TXE_FROZE_SMASK \
 235                         | CCE_STATUS_TXE_PIO_FROZE_SMASK)
 236 /* all CceStatus sub-block TXE pause bits */
 237 #define ALL_TXE_PAUSE (CCE_STATUS_TXE_PIO_PAUSED_SMASK \
 238                         | CCE_STATUS_TXE_PAUSED_SMASK \
 239                         | CCE_STATUS_SDMA_PAUSED_SMASK)
 240 /* all CceStatus sub-block RXE pause bits */
 241 #define ALL_RXE_PAUSE CCE_STATUS_RXE_PAUSED_SMASK
 242
 243 #define CNTR_MAX 0xFFFFFFFFFFFFFFFFULL
 244 #define CNTR_32BIT_MAX 0x00000000FFFFFFFF
 245
 246 /*
 247  * CCE Error flags.
 248  */
 249 static struct flag_table cce_err_status_flags[] = {
 250 /* 0*/  FLAG_ENTRY0("CceCsrParityErr",
 251                 CCE_ERR_STATUS_CCE_CSR_PARITY_ERR_SMASK),
 252 /* 1*/  FLAG_ENTRY0("CceCsrReadBadAddrErr",
 253                 CCE_ERR_STATUS_CCE_CSR_READ_BAD_ADDR_ERR_SMASK),
 254 /* 2*/  FLAG_ENTRY0("CceCsrWriteBadAddrErr",
 255                 CCE_ERR_STATUS_CCE_CSR_WRITE_BAD_ADDR_ERR_SMASK),
 256 /* 3*/  FLAG_ENTRY0("CceTrgtAsyncFifoParityErr",
 257                 CCE_ERR_STATUS_CCE_TRGT_ASYNC_FIFO_PARITY_ERR_SMASK),
 258 /* 4*/  FLAG_ENTRY0("CceTrgtAccessErr",
 259                 CCE_ERR_STATUS_CCE_TRGT_ACCESS_ERR_SMASK),
 260 /* 5*/  FLAG_ENTRY0("CceRspdDataParityErr",
 261                 CCE_ERR_STATUS_CCE_RSPD_DATA_PARITY_ERR_SMASK),
 262 /* 6*/  FLAG_ENTRY0("CceCli0AsyncFifoParityErr",
 263                 CCE_ERR_STATUS_CCE_CLI0_ASYNC_FIFO_PARITY_ERR_SMASK),
 264 /* 7*/  FLAG_ENTRY0("CceCsrCfgBusParityErr",
 265                 CCE_ERR_STATUS_CCE_CSR_CFG_BUS_PARITY_ERR_SMASK),
 266 /* 8*/  FLAG_ENTRY0("CceCli2AsyncFifoParityErr",
 267                 CCE_ERR_STATUS_CCE_CLI2_ASYNC_FIFO_PARITY_ERR_SMASK),
 268 /* 9*/  FLAG_ENTRY0("CceCli1AsyncFifoPioCrdtParityErr",
 269             CCE_ERR_STATUS_CCE_CLI1_ASYNC_FIFO_PIO_CRDT_PARITY_ERR_SMASK),
 270 /*10*/  FLAG_ENTRY0("CceCli1AsyncFifoPioCrdtParityErr",
 271             CCE_ERR_STATUS_CCE_CLI1_ASYNC_FIFO_SDMA_HD_PARITY_ERR_SMASK),
 272 /*11*/  FLAG_ENTRY0("CceCli1AsyncFifoRxdmaParityError",
 273             CCE_ERR_STATUS_CCE_CLI1_ASYNC_FIFO_RXDMA_PARITY_ERROR_SMASK),
 274 /*12*/  FLAG_ENTRY0("CceCli1AsyncFifoDbgParityError",
 275                 CCE_ERR_STATUS_CCE_CLI1_ASYNC_FIFO_DBG_PARITY_ERROR_SMASK),
 276 /*13*/  FLAG_ENTRY0("PcicRetryMemCorErr",
 277                 CCE_ERR_STATUS_PCIC_RETRY_MEM_COR_ERR_SMASK),
 278 /*14*/  FLAG_ENTRY0("PcicRetryMemCorErr",
 279                 CCE_ERR_STATUS_PCIC_RETRY_SOT_MEM_COR_ERR_SMASK),
 280 /*15*/  FLAG_ENTRY0("PcicPostHdQCorErr",
 281                 CCE_ERR_STATUS_PCIC_POST_HD_QCOR_ERR_SMASK),
 282 /*16*/  FLAG_ENTRY0("PcicPostHdQCorErr",
 283                 CCE_ERR_STATUS_PCIC_POST_DAT_QCOR_ERR_SMASK),
 284 /*17*/  FLAG_ENTRY0("PcicPostHdQCorErr",
 285                 CCE_ERR_STATUS_PCIC_CPL_HD_QCOR_ERR_SMASK),
 286 /*18*/  FLAG_ENTRY0("PcicCplDatQCorErr",
 287                 CCE_ERR_STATUS_PCIC_CPL_DAT_QCOR_ERR_SMASK),
 288 /*19*/  FLAG_ENTRY0("PcicNPostHQParityErr",
 289                 CCE_ERR_STATUS_PCIC_NPOST_HQ_PARITY_ERR_SMASK),
 290 /*20*/  FLAG_ENTRY0("PcicNPostDatQParityErr",
 291                 CCE_ERR_STATUS_PCIC_NPOST_DAT_QPARITY_ERR_SMASK),
 292 /*21*/  FLAG_ENTRY0("PcicRetryMemUncErr",
 293                 CCE_ERR_STATUS_PCIC_RETRY_MEM_UNC_ERR_SMASK),
 294 /*22*/  FLAG_ENTRY0("PcicRetrySotMemUncErr",
 295                 CCE_ERR_STATUS_PCIC_RETRY_SOT_MEM_UNC_ERR_SMASK),
 296 /*23*/  FLAG_ENTRY0("PcicPostHdQUncErr",
 297                 CCE_ERR_STATUS_PCIC_POST_HD_QUNC_ERR_SMASK),
 298 /*24*/  FLAG_ENTRY0("PcicPostDatQUncErr",
 299                 CCE_ERR_STATUS_PCIC_POST_DAT_QUNC_ERR_SMASK),
 300 /*25*/  FLAG_ENTRY0("PcicCplHdQUncErr",
 301                 CCE_ERR_STATUS_PCIC_CPL_HD_QUNC_ERR_SMASK),
 302 /*26*/  FLAG_ENTRY0("PcicCplDatQUncErr",
 303                 CCE_ERR_STATUS_PCIC_CPL_DAT_QUNC_ERR_SMASK),
 304 /*27*/  FLAG_ENTRY0("PcicTransmitFrontParityErr",
 305                 CCE_ERR_STATUS_PCIC_TRANSMIT_FRONT_PARITY_ERR_SMASK),
 306 /*28*/  FLAG_ENTRY0("PcicTransmitBackParityErr",
 307                 CCE_ERR_STATUS_PCIC_TRANSMIT_BACK_PARITY_ERR_SMASK),
 308 /*29*/  FLAG_ENTRY0("PcicReceiveParityErr",
 309                 CCE_ERR_STATUS_PCIC_RECEIVE_PARITY_ERR_SMASK),
 310 /*30*/  FLAG_ENTRY0("CceTrgtCplTimeoutErr",
 311                 CCE_ERR_STATUS_CCE_TRGT_CPL_TIMEOUT_ERR_SMASK),
 312 /*31*/  FLAG_ENTRY0("LATriggered",
 313                 CCE_ERR_STATUS_LA_TRIGGERED_SMASK),
 314 /*32*/  FLAG_ENTRY0("CceSegReadBadAddrErr",
 315                 CCE_ERR_STATUS_CCE_SEG_READ_BAD_ADDR_ERR_SMASK),
 316 /*33*/  FLAG_ENTRY0("CceSegWriteBadAddrErr",
 317                 CCE_ERR_STATUS_CCE_SEG_WRITE_BAD_ADDR_ERR_SMASK),
 318 /*34*/  FLAG_ENTRY0("CceRcplAsyncFifoParityErr",
 319                 CCE_ERR_STATUS_CCE_RCPL_ASYNC_FIFO_PARITY_ERR_SMASK),
 320 /*35*/  FLAG_ENTRY0("CceRxdmaConvFifoParityErr",
 321                 CCE_ERR_STATUS_CCE_RXDMA_CONV_FIFO_PARITY_ERR_SMASK),
 322 /*36*/  FLAG_ENTRY0("CceMsixTableCorErr",
 323                 CCE_ERR_STATUS_CCE_MSIX_TABLE_COR_ERR_SMASK),
 324 /*37*/  FLAG_ENTRY0("CceMsixTableUncErr",
 325                 CCE_ERR_STATUS_CCE_MSIX_TABLE_UNC_ERR_SMASK),
 326 /*38*/  FLAG_ENTRY0("CceIntMapCorErr",
 327                 CCE_ERR_STATUS_CCE_INT_MAP_COR_ERR_SMASK),
 328 /*39*/  FLAG_ENTRY0("CceIntMapUncErr",
 329                 CCE_ERR_STATUS_CCE_INT_MAP_UNC_ERR_SMASK),
 330 /*40*/  FLAG_ENTRY0("CceMsixCsrParityErr",
 331                 CCE_ERR_STATUS_CCE_MSIX_CSR_PARITY_ERR_SMASK),
 332 /*41-63 reserved*/
 333 };
 334
 335 /*
 336  * Misc Error flags
 337  */
 338 #define MES(text) MISC_ERR_STATUS_MISC_##text##_ERR_SMASK
 339 static struct flag_table misc_err_status_flags[] = {
 340 /* 0*/  FLAG_ENTRY0("CSR_PARITY", MES(CSR_PARITY)),
 341 /* 1*/  FLAG_ENTRY0("CSR_READ_BAD_ADDR", MES(CSR_READ_BAD_ADDR)),
 342 /* 2*/  FLAG_ENTRY0("CSR_WRITE_BAD_ADDR", MES(CSR_WRITE_BAD_ADDR)),
 343 /* 3*/  FLAG_ENTRY0("SBUS_WRITE_FAILED", MES(SBUS_WRITE_FAILED)),
 344 /* 4*/  FLAG_ENTRY0("KEY_MISMATCH", MES(KEY_MISMATCH)),
 345 /* 5*/  FLAG_ENTRY0("FW_AUTH_FAILED", MES(FW_AUTH_FAILED)),
 346 /* 6*/  FLAG_ENTRY0("EFUSE_CSR_PARITY", MES(EFUSE_CSR_PARITY)),
 347 /* 7*/  FLAG_ENTRY0("EFUSE_READ_BAD_ADDR", MES(EFUSE_READ_BAD_ADDR)),
 348 /* 8*/  FLAG_ENTRY0("EFUSE_WRITE", MES(EFUSE_WRITE)),
 349 /* 9*/  FLAG_ENTRY0("EFUSE_DONE_PARITY", MES(EFUSE_DONE_PARITY)),
 350 /*10*/  FLAG_ENTRY0("INVALID_EEP_CMD", MES(INVALID_EEP_CMD)),
 351 /*11*/  FLAG_ENTRY0("MBIST_FAIL", MES(MBIST_FAIL)),
 352 /*12*/  FLAG_ENTRY0("PLL_LOCK_FAIL", MES(PLL_LOCK_FAIL))
 353 };
 354
 355 /*
 356  * TXE PIO Error flags and consequences
 357  */
 358 static struct flag_table pio_err_status_flags[] = {
 359 /* 0*/  FLAG_ENTRY("PioWriteBadCtxt",
 360         SEC_WRITE_DROPPED,
 361         SEND_PIO_ERR_STATUS_PIO_WRITE_BAD_CTXT_ERR_SMASK),
 362 /* 1*/  FLAG_ENTRY("PioWriteAddrParity",
 363         SEC_SPC_FREEZE,
 364         SEND_PIO_ERR_STATUS_PIO_WRITE_ADDR_PARITY_ERR_SMASK),
 365 /* 2*/  FLAG_ENTRY("PioCsrParity",
 366         SEC_SPC_FREEZE,
 367         SEND_PIO_ERR_STATUS_PIO_CSR_PARITY_ERR_SMASK),
 368 /* 3*/  FLAG_ENTRY("PioSbMemFifo0",
 369         SEC_SPC_FREEZE,
 370         SEND_PIO_ERR_STATUS_PIO_SB_MEM_FIFO0_ERR_SMASK),
 371 /* 4*/  FLAG_ENTRY("PioSbMemFifo1",
 372         SEC_SPC_FREEZE,
 373         SEND_PIO_ERR_STATUS_PIO_SB_MEM_FIFO1_ERR_SMASK),
 374 /* 5*/  FLAG_ENTRY("PioPccFifoParity",
 375         SEC_SPC_FREEZE,
 376         SEND_PIO_ERR_STATUS_PIO_PCC_FIFO_PARITY_ERR_SMASK),
 377 /* 6*/  FLAG_ENTRY("PioPecFifoParity",
 378         SEC_SPC_FREEZE,
 379         SEND_PIO_ERR_STATUS_PIO_PEC_FIFO_PARITY_ERR_SMASK),
 380 /* 7*/  FLAG_ENTRY("PioSbrdctlCrrelParity",
 381         SEC_SPC_FREEZE,
 382         SEND_PIO_ERR_STATUS_PIO_SBRDCTL_CRREL_PARITY_ERR_SMASK),
 383 /* 8*/  FLAG_ENTRY("PioSbrdctrlCrrelFifoParity",
 384         SEC_SPC_FREEZE,
 385         SEND_PIO_ERR_STATUS_PIO_SBRDCTRL_CRREL_FIFO_PARITY_ERR_SMASK),
 386 /* 9*/  FLAG_ENTRY("PioPktEvictFifoParityErr",
 387         SEC_SPC_FREEZE,
 388         SEND_PIO_ERR_STATUS_PIO_PKT_EVICT_FIFO_PARITY_ERR_SMASK),
 389 /*10*/  FLAG_ENTRY("PioSmPktResetParity",
 390         SEC_SPC_FREEZE,
 391         SEND_PIO_ERR_STATUS_PIO_SM_PKT_RESET_PARITY_ERR_SMASK),
 392 /*11*/  FLAG_ENTRY("PioVlLenMemBank0Unc",
 393         SEC_SPC_FREEZE,
 394         SEND_PIO_ERR_STATUS_PIO_VL_LEN_MEM_BANK0_UNC_ERR_SMASK),
 395 /*12*/  FLAG_ENTRY("PioVlLenMemBank1Unc",
 396         SEC_SPC_FREEZE,
 397         SEND_PIO_ERR_STATUS_PIO_VL_LEN_MEM_BANK1_UNC_ERR_SMASK),
 398 /*13*/  FLAG_ENTRY("PioVlLenMemBank0Cor",
 399         0,
 400         SEND_PIO_ERR_STATUS_PIO_VL_LEN_MEM_BANK0_COR_ERR_SMASK),
 401 /*14*/  FLAG_ENTRY("PioVlLenMemBank1Cor",
 402         0,
 403         SEND_PIO_ERR_STATUS_PIO_VL_LEN_MEM_BANK1_COR_ERR_SMASK),
 404 /*15*/  FLAG_ENTRY("PioCreditRetFifoParity",
 405         SEC_SPC_FREEZE,
 406         SEND_PIO_ERR_STATUS_PIO_CREDIT_RET_FIFO_PARITY_ERR_SMASK),
 407 /*16*/  FLAG_ENTRY("PioPpmcPblFifo",
 408         SEC_SPC_FREEZE,
 409         SEND_PIO_ERR_STATUS_PIO_PPMC_PBL_FIFO_ERR_SMASK),
 410 /*17*/  FLAG_ENTRY("PioInitSmIn",
 411         0,
 412         SEND_PIO_ERR_STATUS_PIO_INIT_SM_IN_ERR_SMASK),
 413 /*18*/  FLAG_ENTRY("PioPktEvictSmOrArbSm",
 414         SEC_SPC_FREEZE,
 415         SEND_PIO_ERR_STATUS_PIO_PKT_EVICT_SM_OR_ARB_SM_ERR_SMASK),
 416 /*19*/  FLAG_ENTRY("PioHostAddrMemUnc",
 417         SEC_SPC_FREEZE,
 418         SEND_PIO_ERR_STATUS_PIO_HOST_ADDR_MEM_UNC_ERR_SMASK),
 419 /*20*/  FLAG_ENTRY("PioHostAddrMemCor",
 420         0,
 421         SEND_PIO_ERR_STATUS_PIO_HOST_ADDR_MEM_COR_ERR_SMASK),
 422 /*21*/  FLAG_ENTRY("PioWriteDataParity",
 423         SEC_SPC_FREEZE,
 424         SEND_PIO_ERR_STATUS_PIO_WRITE_DATA_PARITY_ERR_SMASK),
 425 /*22*/  FLAG_ENTRY("PioStateMachine",
 426         SEC_SPC_FREEZE,
 427         SEND_PIO_ERR_STATUS_PIO_STATE_MACHINE_ERR_SMASK),
 428 /*23*/  FLAG_ENTRY("PioWriteQwValidParity",
 429         SEC_WRITE_DROPPED | SEC_SPC_FREEZE,
 430         SEND_PIO_ERR_STATUS_PIO_WRITE_QW_VALID_PARITY_ERR_SMASK),
 431 /*24*/  FLAG_ENTRY("PioBlockQwCountParity",
 432         SEC_WRITE_DROPPED | SEC_SPC_FREEZE,
 433         SEND_PIO_ERR_STATUS_PIO_BLOCK_QW_COUNT_PARITY_ERR_SMASK),
 434 /*25*/  FLAG_ENTRY("PioVlfVlLenParity",
 435         SEC_SPC_FREEZE,
 436         SEND_PIO_ERR_STATUS_PIO_VLF_VL_LEN_PARITY_ERR_SMASK),
 437 /*26*/  FLAG_ENTRY("PioVlfSopParity",
 438         SEC_SPC_FREEZE,
 439         SEND_PIO_ERR_STATUS_PIO_VLF_SOP_PARITY_ERR_SMASK),
 440 /*27*/  FLAG_ENTRY("PioVlFifoParity",
 441         SEC_SPC_FREEZE,
 442         SEND_PIO_ERR_STATUS_PIO_VL_FIFO_PARITY_ERR_SMASK),
 443 /*28*/  FLAG_ENTRY("PioPpmcBqcMemParity",
 444         SEC_SPC_FREEZE,
 445         SEND_PIO_ERR_STATUS_PIO_PPMC_BQC_MEM_PARITY_ERR_SMASK),
 446 /*29*/  FLAG_ENTRY("PioPpmcSopLen",
 447         SEC_SPC_FREEZE,
 448         SEND_PIO_ERR_STATUS_PIO_PPMC_SOP_LEN_ERR_SMASK),
 449 /*30-31 reserved*/
 450 /*32*/  FLAG_ENTRY("PioCurrentFreeCntParity",
 451         SEC_SPC_FREEZE,
 452         SEND_PIO_ERR_STATUS_PIO_CURRENT_FREE_CNT_PARITY_ERR_SMASK),
 453 /*33*/  FLAG_ENTRY("PioLastReturnedCntParity",
 454         SEC_SPC_FREEZE,
 455         SEND_PIO_ERR_STATUS_PIO_LAST_RETURNED_CNT_PARITY_ERR_SMASK),
 456 /*34*/  FLAG_ENTRY("PioPccSopHeadParity",
 457         SEC_SPC_FREEZE,
 458         SEND_PIO_ERR_STATUS_PIO_PCC_SOP_HEAD_PARITY_ERR_SMASK),
 459 /*35*/  FLAG_ENTRY("PioPecSopHeadParityErr",
 460         SEC_SPC_FREEZE,
 461         SEND_PIO_ERR_STATUS_PIO_PEC_SOP_HEAD_PARITY_ERR_SMASK),
 462 /*36-63 reserved*/
 463 };
 464
 465 /* TXE PIO errors that cause an SPC freeze */
 466 #define ALL_PIO_FREEZE_ERR \
 467         (SEND_PIO_ERR_STATUS_PIO_WRITE_ADDR_PARITY_ERR_SMASK \
 468         | SEND_PIO_ERR_STATUS_PIO_CSR_PARITY_ERR_SMASK \
 469         | SEND_PIO_ERR_STATUS_PIO_SB_MEM_FIFO0_ERR_SMASK \
 470         | SEND_PIO_ERR_STATUS_PIO_SB_MEM_FIFO1_ERR_SMASK \
 471         | SEND_PIO_ERR_STATUS_PIO_PCC_FIFO_PARITY_ERR_SMASK \
 472         | SEND_PIO_ERR_STATUS_PIO_PEC_FIFO_PARITY_ERR_SMASK \
 473         | SEND_PIO_ERR_STATUS_PIO_SBRDCTL_CRREL_PARITY_ERR_SMASK \
 474         | SEND_PIO_ERR_STATUS_PIO_SBRDCTRL_CRREL_FIFO_PARITY_ERR_SMASK \
 475         | SEND_PIO_ERR_STATUS_PIO_PKT_EVICT_FIFO_PARITY_ERR_SMASK \
 476         | SEND_PIO_ERR_STATUS_PIO_SM_PKT_RESET_PARITY_ERR_SMASK \
 477         | SEND_PIO_ERR_STATUS_PIO_VL_LEN_MEM_BANK0_UNC_ERR_SMASK \
 478         | SEND_PIO_ERR_STATUS_PIO_VL_LEN_MEM_BANK1_UNC_ERR_SMASK \
 479         | SEND_PIO_ERR_STATUS_PIO_CREDIT_RET_FIFO_PARITY_ERR_SMASK \
 480         | SEND_PIO_ERR_STATUS_PIO_PPMC_PBL_FIFO_ERR_SMASK \
 481         | SEND_PIO_ERR_STATUS_PIO_PKT_EVICT_SM_OR_ARB_SM_ERR_SMASK \
 482         | SEND_PIO_ERR_STATUS_PIO_HOST_ADDR_MEM_UNC_ERR_SMASK \
 483         | SEND_PIO_ERR_STATUS_PIO_WRITE_DATA_PARITY_ERR_SMASK \
 484         | SEND_PIO_ERR_STATUS_PIO_STATE_MACHINE_ERR_SMASK \
 485         | SEND_PIO_ERR_STATUS_PIO_WRITE_QW_VALID_PARITY_ERR_SMASK \
 486         | SEND_PIO_ERR_STATUS_PIO_BLOCK_QW_COUNT_PARITY_ERR_SMASK \
 487         | SEND_PIO_ERR_STATUS_PIO_VLF_VL_LEN_PARITY_ERR_SMASK \
 488         | SEND_PIO_ERR_STATUS_PIO_VLF_SOP_PARITY_ERR_SMASK \
 489         | SEND_PIO_ERR_STATUS_PIO_VL_FIFO_PARITY_ERR_SMASK \
 490         | SEND_PIO_ERR_STATUS_PIO_PPMC_BQC_MEM_PARITY_ERR_SMASK \
 491         | SEND_PIO_ERR_STATUS_PIO_PPMC_SOP_LEN_ERR_SMASK \
 492         | SEND_PIO_ERR_STATUS_PIO_CURRENT_FREE_CNT_PARITY_ERR_SMASK \
 493         | SEND_PIO_ERR_STATUS_PIO_LAST_RETURNED_CNT_PARITY_ERR_SMASK \
 494         | SEND_PIO_ERR_STATUS_PIO_PCC_SOP_HEAD_PARITY_ERR_SMASK \
 495         | SEND_PIO_ERR_STATUS_PIO_PEC_SOP_HEAD_PARITY_ERR_SMASK)
 496
 497 /*
 498  * TXE SDMA Error flags
 499  */
 500 static struct flag_table sdma_err_status_flags[] = {
 501 /* 0*/  FLAG_ENTRY0("SDmaRpyTagErr",
 502                 SEND_DMA_ERR_STATUS_SDMA_RPY_TAG_ERR_SMASK),
 503 /* 1*/  FLAG_ENTRY0("SDmaCsrParityErr",
 504                 SEND_DMA_ERR_STATUS_SDMA_CSR_PARITY_ERR_SMASK),
 505 /* 2*/  FLAG_ENTRY0("SDmaPcieReqTrackingUncErr",
 506                 SEND_DMA_ERR_STATUS_SDMA_PCIE_REQ_TRACKING_UNC_ERR_SMASK),
 507 /* 3*/  FLAG_ENTRY0("SDmaPcieReqTrackingCorErr",
 508                 SEND_DMA_ERR_STATUS_SDMA_PCIE_REQ_TRACKING_COR_ERR_SMASK),
 509 /*04-63 reserved*/
 510 };
 511
 512 /* TXE SDMA errors that cause an SPC freeze */
 513 #define ALL_SDMA_FREEZE_ERR  \
 514                 (SEND_DMA_ERR_STATUS_SDMA_RPY_TAG_ERR_SMASK \
 515                 | SEND_DMA_ERR_STATUS_SDMA_CSR_PARITY_ERR_SMASK \
 516                 | SEND_DMA_ERR_STATUS_SDMA_PCIE_REQ_TRACKING_UNC_ERR_SMASK)
 517
 518 /* SendEgressErrInfo bits that correspond to a PortXmitDiscard counter */
 519 #define PORT_DISCARD_EGRESS_ERRS \
 520         (SEND_EGRESS_ERR_INFO_TOO_LONG_IB_PACKET_ERR_SMASK \
 521         | SEND_EGRESS_ERR_INFO_VL_MAPPING_ERR_SMASK \
 522         | SEND_EGRESS_ERR_INFO_VL_ERR_SMASK)
 523
 524 /*
 525  * TXE Egress Error flags
 526  */
 527 #define SEES(text) SEND_EGRESS_ERR_STATUS_##text##_ERR_SMASK
 528 static struct flag_table egress_err_status_flags[] = {
 529 /* 0*/  FLAG_ENTRY0("TxPktIntegrityMemCorErr", SEES(TX_PKT_INTEGRITY_MEM_COR)),
 530 /* 1*/  FLAG_ENTRY0("TxPktIntegrityMemUncErr", SEES(TX_PKT_INTEGRITY_MEM_UNC)),
 531 /* 2 reserved */
 532 /* 3*/  FLAG_ENTRY0("TxEgressFifoUnderrunOrParityErr",
 533                 SEES(TX_EGRESS_FIFO_UNDERRUN_OR_PARITY)),
 534 /* 4*/  FLAG_ENTRY0("TxLinkdownErr", SEES(TX_LINKDOWN)),
 535 /* 5*/  FLAG_ENTRY0("TxIncorrectLinkStateErr", SEES(TX_INCORRECT_LINK_STATE)),
 536 /* 6 reserved */
 537 /* 7*/  FLAG_ENTRY0("TxPioLaunchIntfParityErr",
 538                 SEES(TX_PIO_LAUNCH_INTF_PARITY)),
 539 /* 8*/  FLAG_ENTRY0("TxSdmaLaunchIntfParityErr",
 540                 SEES(TX_SDMA_LAUNCH_INTF_PARITY)),
 541 /* 9-10 reserved */
 542 /*11*/  FLAG_ENTRY0("TxSbrdCtlStateMachineParityErr",
 543                 SEES(TX_SBRD_CTL_STATE_MACHINE_PARITY)),
 544 /*12*/  FLAG_ENTRY0("TxIllegalVLErr", SEES(TX_ILLEGAL_VL)),
 545 /*13*/  FLAG_ENTRY0("TxLaunchCsrParityErr", SEES(TX_LAUNCH_CSR_PARITY)),
 546 /*14*/  FLAG_ENTRY0("TxSbrdCtlCsrParityErr", SEES(TX_SBRD_CTL_CSR_PARITY)),
 547 /*15*/  FLAG_ENTRY0("TxConfigParityErr", SEES(TX_CONFIG_PARITY)),
 548 /*16*/  FLAG_ENTRY0("TxSdma0DisallowedPacketErr",
 549                 SEES(TX_SDMA0_DISALLOWED_PACKET)),
 550 /*17*/  FLAG_ENTRY0("TxSdma1DisallowedPacketErr",
 551                 SEES(TX_SDMA1_DISALLOWED_PACKET)),
 552 /*18*/  FLAG_ENTRY0("TxSdma2DisallowedPacketErr",
 553                 SEES(TX_SDMA2_DISALLOWED_PACKET)),
 554 /*19*/  FLAG_ENTRY0("TxSdma3DisallowedPacketErr",
 555                 SEES(TX_SDMA3_DISALLOWED_PACKET)),
 556 /*20*/  FLAG_ENTRY0("TxSdma4DisallowedPacketErr",
 557                 SEES(TX_SDMA4_DISALLOWED_PACKET)),
 558 /*21*/  FLAG_ENTRY0("TxSdma5DisallowedPacketErr",
 559                 SEES(TX_SDMA5_DISALLOWED_PACKET)),
 560 /*22*/  FLAG_ENTRY0("TxSdma6DisallowedPacketErr",
 561                 SEES(TX_SDMA6_DISALLOWED_PACKET)),
 562 /*23*/  FLAG_ENTRY0("TxSdma7DisallowedPacketErr",
 563                 SEES(TX_SDMA7_DISALLOWED_PACKET)),
 564 /*24*/  FLAG_ENTRY0("TxSdma8DisallowedPacketErr",
 565                 SEES(TX_SDMA8_DISALLOWED_PACKET)),
 566 /*25*/  FLAG_ENTRY0("TxSdma9DisallowedPacketErr",
 567                 SEES(TX_SDMA9_DISALLOWED_PACKET)),
 568 /*26*/  FLAG_ENTRY0("TxSdma10DisallowedPacketErr",
 569                 SEES(TX_SDMA10_DISALLOWED_PACKET)),
 570 /*27*/  FLAG_ENTRY0("TxSdma11DisallowedPacketErr",
 571                 SEES(TX_SDMA11_DISALLOWED_PACKET)),
 572 /*28*/  FLAG_ENTRY0("TxSdma12DisallowedPacketErr",
 573                 SEES(TX_SDMA12_DISALLOWED_PACKET)),
 574 /*29*/  FLAG_ENTRY0("TxSdma13DisallowedPacketErr",
 575                 SEES(TX_SDMA13_DISALLOWED_PACKET)),
 576 /*30*/  FLAG_ENTRY0("TxSdma14DisallowedPacketErr",
 577                 SEES(TX_SDMA14_DISALLOWED_PACKET)),
 578 /*31*/  FLAG_ENTRY0("TxSdma15DisallowedPacketErr",
 579                 SEES(TX_SDMA15_DISALLOWED_PACKET)),
 580 /*32*/  FLAG_ENTRY0("TxLaunchFifo0UncOrParityErr",
 581                 SEES(TX_LAUNCH_FIFO0_UNC_OR_PARITY)),
 582 /*33*/  FLAG_ENTRY0("TxLaunchFifo1UncOrParityErr",
 583                 SEES(TX_LAUNCH_FIFO1_UNC_OR_PARITY)),
 584 /*34*/  FLAG_ENTRY0("TxLaunchFifo2UncOrParityErr",
 585                 SEES(TX_LAUNCH_FIFO2_UNC_OR_PARITY)),
 586 /*35*/  FLAG_ENTRY0("TxLaunchFifo3UncOrParityErr",
 587                 SEES(TX_LAUNCH_FIFO3_UNC_OR_PARITY)),
 588 /*36*/  FLAG_ENTRY0("TxLaunchFifo4UncOrParityErr",
 589                 SEES(TX_LAUNCH_FIFO4_UNC_OR_PARITY)),
 590 /*37*/  FLAG_ENTRY0("TxLaunchFifo5UncOrParityErr",
 591                 SEES(TX_LAUNCH_FIFO5_UNC_OR_PARITY)),
 592 /*38*/  FLAG_ENTRY0("TxLaunchFifo6UncOrParityErr",
 593                 SEES(TX_LAUNCH_FIFO6_UNC_OR_PARITY)),
 594 /*39*/  FLAG_ENTRY0("TxLaunchFifo7UncOrParityErr",
 595                 SEES(TX_LAUNCH_FIFO7_UNC_OR_PARITY)),
 596 /*40*/  FLAG_ENTRY0("TxLaunchFifo8UncOrParityErr",
 597                 SEES(TX_LAUNCH_FIFO8_UNC_OR_PARITY)),
 598 /*41*/  FLAG_ENTRY0("TxCreditReturnParityErr", SEES(TX_CREDIT_RETURN_PARITY)),
 599 /*42*/  FLAG_ENTRY0("TxSbHdrUncErr", SEES(TX_SB_HDR_UNC)),
 600 /*43*/  FLAG_ENTRY0("TxReadSdmaMemoryUncErr", SEES(TX_READ_SDMA_MEMORY_UNC)),
 601 /*44*/  FLAG_ENTRY0("TxReadPioMemoryUncErr", SEES(TX_READ_PIO_MEMORY_UNC)),
 602 /*45*/  FLAG_ENTRY0("TxEgressFifoUncErr", SEES(TX_EGRESS_FIFO_UNC)),
 603 /*46*/  FLAG_ENTRY0("TxHcrcInsertionErr", SEES(TX_HCRC_INSERTION)),
 604 /*47*/  FLAG_ENTRY0("TxCreditReturnVLErr", SEES(TX_CREDIT_RETURN_VL)),
 605 /*48*/  FLAG_ENTRY0("TxLaunchFifo0CorErr", SEES(TX_LAUNCH_FIFO0_COR)),
 606 /*49*/  FLAG_ENTRY0("TxLaunchFifo1CorErr", SEES(TX_LAUNCH_FIFO1_COR)),
 607 /*50*/  FLAG_ENTRY0("TxLaunchFifo2CorErr", SEES(TX_LAUNCH_FIFO2_COR)),
 608 /*51*/  FLAG_ENTRY0("TxLaunchFifo3CorErr", SEES(TX_LAUNCH_FIFO3_COR)),
 609 /*52*/  FLAG_ENTRY0("TxLaunchFifo4CorErr", SEES(TX_LAUNCH_FIFO4_COR)),
 610 /*53*/  FLAG_ENTRY0("TxLaunchFifo5CorErr", SEES(TX_LAUNCH_FIFO5_COR)),
 611 /*54*/  FLAG_ENTRY0("TxLaunchFifo6CorErr", SEES(TX_LAUNCH_FIFO6_COR)),
 612 /*55*/  FLAG_ENTRY0("TxLaunchFifo7CorErr", SEES(TX_LAUNCH_FIFO7_COR)),
 613 /*56*/  FLAG_ENTRY0("TxLaunchFifo8CorErr", SEES(TX_LAUNCH_FIFO8_COR)),
 614 /*57*/  FLAG_ENTRY0("TxCreditOverrunErr", SEES(TX_CREDIT_OVERRUN)),
 615 /*58*/  FLAG_ENTRY0("TxSbHdrCorErr", SEES(TX_SB_HDR_COR)),
 616 /*59*/  FLAG_ENTRY0("TxReadSdmaMemoryCorErr", SEES(TX_READ_SDMA_MEMORY_COR)),
 617 /*60*/  FLAG_ENTRY0("TxReadPioMemoryCorErr", SEES(TX_READ_PIO_MEMORY_COR)),
 618 /*61*/  FLAG_ENTRY0("TxEgressFifoCorErr", SEES(TX_EGRESS_FIFO_COR)),
 619 /*62*/  FLAG_ENTRY0("TxReadSdmaMemoryCsrUncErr",
 620                 SEES(TX_READ_SDMA_MEMORY_CSR_UNC)),
 621 /*63*/  FLAG_ENTRY0("TxReadPioMemoryCsrUncErr",
 622                 SEES(TX_READ_PIO_MEMORY_CSR_UNC)),
 623 };
 624
 625 /*
 626  * TXE Egress Error Info flags
 627  */
 628 #define SEEI(text) SEND_EGRESS_ERR_INFO_##text##_ERR_SMASK
 629 static struct flag_table egress_err_info_flags[] = {
 630 /* 0*/  FLAG_ENTRY0("Reserved", 0ull),
 631 /* 1*/  FLAG_ENTRY0("VLErr", SEEI(VL)),
 632 /* 2*/  FLAG_ENTRY0("JobKeyErr", SEEI(JOB_KEY)),
 633 /* 3*/  FLAG_ENTRY0("JobKeyErr", SEEI(JOB_KEY)),
 634 /* 4*/  FLAG_ENTRY0("PartitionKeyErr", SEEI(PARTITION_KEY)),
 635 /* 5*/  FLAG_ENTRY0("SLIDErr", SEEI(SLID)),
 636 /* 6*/  FLAG_ENTRY0("OpcodeErr", SEEI(OPCODE)),
 637 /* 7*/  FLAG_ENTRY0("VLMappingErr", SEEI(VL_MAPPING)),
 638 /* 8*/  FLAG_ENTRY0("RawErr", SEEI(RAW)),
 639 /* 9*/  FLAG_ENTRY0("RawIPv6Err", SEEI(RAW_IPV6)),
 640 /*10*/  FLAG_ENTRY0("GRHErr", SEEI(GRH)),
 641 /*11*/  FLAG_ENTRY0("BypassErr", SEEI(BYPASS)),
 642 /*12*/  FLAG_ENTRY0("KDETHPacketsErr", SEEI(KDETH_PACKETS)),
 643 /*13*/  FLAG_ENTRY0("NonKDETHPacketsErr", SEEI(NON_KDETH_PACKETS)),
 644 /*14*/  FLAG_ENTRY0("TooSmallIBPacketsErr", SEEI(TOO_SMALL_IB_PACKETS)),
 645 /*15*/  FLAG_ENTRY0("TooSmallBypassPacketsErr", SEEI(TOO_SMALL_BYPASS_PACKETS)),
 646 /*16*/  FLAG_ENTRY0("PbcTestErr", SEEI(PBC_TEST)),
 647 /*17*/  FLAG_ENTRY0("BadPktLenErr", SEEI(BAD_PKT_LEN)),
 648 /*18*/  FLAG_ENTRY0("TooLongIBPacketErr", SEEI(TOO_LONG_IB_PACKET)),
 649 /*19*/  FLAG_ENTRY0("TooLongBypassPacketsErr", SEEI(TOO_LONG_BYPASS_PACKETS)),
 650 /*20*/  FLAG_ENTRY0("PbcStaticRateControlErr", SEEI(PBC_STATIC_RATE_CONTROL)),
 651 /*21*/  FLAG_ENTRY0("BypassBadPktLenErr", SEEI(BAD_PKT_LEN)),
 652 };
 653
 654 /* TXE Egress errors that cause an SPC freeze */
 655 #define ALL_TXE_EGRESS_FREEZE_ERR \
 656         (SEES(TX_EGRESS_FIFO_UNDERRUN_OR_PARITY) \
 657         | SEES(TX_PIO_LAUNCH_INTF_PARITY) \
 658         | SEES(TX_SDMA_LAUNCH_INTF_PARITY) \
 659         | SEES(TX_SBRD_CTL_STATE_MACHINE_PARITY) \
 660         | SEES(TX_LAUNCH_CSR_PARITY) \
 661         | SEES(TX_SBRD_CTL_CSR_PARITY) \
 662         | SEES(TX_CONFIG_PARITY) \
 663         | SEES(TX_LAUNCH_FIFO0_UNC_OR_PARITY) \
 664         | SEES(TX_LAUNCH_FIFO1_UNC_OR_PARITY) \
 665         | SEES(TX_LAUNCH_FIFO2_UNC_OR_PARITY) \
 666         | SEES(TX_LAUNCH_FIFO3_UNC_OR_PARITY) \
 667         | SEES(TX_LAUNCH_FIFO4_UNC_OR_PARITY) \
 668         | SEES(TX_LAUNCH_FIFO5_UNC_OR_PARITY) \
 669         | SEES(TX_LAUNCH_FIFO6_UNC_OR_PARITY) \
 670         | SEES(TX_LAUNCH_FIFO7_UNC_OR_PARITY) \
 671         | SEES(TX_LAUNCH_FIFO8_UNC_OR_PARITY) \
 672         | SEES(TX_CREDIT_RETURN_PARITY))
 673
 674 /*
 675  * TXE Send error flags
 676  */
 677 #define SES(name) SEND_ERR_STATUS_SEND_##name##_ERR_SMASK
 678 static struct flag_table send_err_status_flags[] = {
 679 /* 0*/  FLAG_ENTRY0("SendCsrParityErr", SES(CSR_PARITY)),
 680 /* 1*/  FLAG_ENTRY0("SendCsrReadBadAddrErr", SES(CSR_READ_BAD_ADDR)),
 681 /* 2*/  FLAG_ENTRY0("SendCsrWriteBadAddrErr", SES(CSR_WRITE_BAD_ADDR))
 682 };
 683
 684 /*
 685  * TXE Send Context Error flags and consequences
 686  */
 687 static struct flag_table sc_err_status_flags[] = {
 688 /* 0*/  FLAG_ENTRY("InconsistentSop",
 689                 SEC_PACKET_DROPPED | SEC_SC_HALTED,
 690                 SEND_CTXT_ERR_STATUS_PIO_INCONSISTENT_SOP_ERR_SMASK),
 691 /* 1*/  FLAG_ENTRY("DisallowedPacket",
 692                 SEC_PACKET_DROPPED | SEC_SC_HALTED,
 693                 SEND_CTXT_ERR_STATUS_PIO_DISALLOWED_PACKET_ERR_SMASK),
 694 /* 2*/  FLAG_ENTRY("WriteCrossesBoundary",
 695                 SEC_WRITE_DROPPED | SEC_SC_HALTED,
 696                 SEND_CTXT_ERR_STATUS_PIO_WRITE_CROSSES_BOUNDARY_ERR_SMASK),
 697 /* 3*/  FLAG_ENTRY("WriteOverflow",
 698                 SEC_WRITE_DROPPED | SEC_SC_HALTED,
 699                 SEND_CTXT_ERR_STATUS_PIO_WRITE_OVERFLOW_ERR_SMASK),
 700 /* 4*/  FLAG_ENTRY("WriteOutOfBounds",
 701                 SEC_WRITE_DROPPED | SEC_SC_HALTED,
 702                 SEND_CTXT_ERR_STATUS_PIO_WRITE_OUT_OF_BOUNDS_ERR_SMASK),
 703 /* 5-63 reserved*/
 704 };
 705
 706 /*
 707  * RXE Receive Error flags
 708  */
 709 #define RXES(name) RCV_ERR_STATUS_RX_##name##_ERR_SMASK
 710 static struct flag_table rxe_err_status_flags[] = {
 711 /* 0*/  FLAG_ENTRY0("RxDmaCsrCorErr", RXES(DMA_CSR_COR)),
 712 /* 1*/  FLAG_ENTRY0("RxDcIntfParityErr", RXES(DC_INTF_PARITY)),
 713 /* 2*/  FLAG_ENTRY0("RxRcvHdrUncErr", RXES(RCV_HDR_UNC)),
 714 /* 3*/  FLAG_ENTRY0("RxRcvHdrCorErr", RXES(RCV_HDR_COR)),
 715 /* 4*/  FLAG_ENTRY0("RxRcvDataUncErr", RXES(RCV_DATA_UNC)),
 716 /* 5*/  FLAG_ENTRY0("RxRcvDataCorErr", RXES(RCV_DATA_COR)),
 717 /* 6*/  FLAG_ENTRY0("RxRcvQpMapTableUncErr", RXES(RCV_QP_MAP_TABLE_UNC)),
 718 /* 7*/  FLAG_ENTRY0("RxRcvQpMapTableCorErr", RXES(RCV_QP_MAP_TABLE_COR)),
 719 /* 8*/  FLAG_ENTRY0("RxRcvCsrParityErr", RXES(RCV_CSR_PARITY)),
 720 /* 9*/  FLAG_ENTRY0("RxDcSopEopParityErr", RXES(DC_SOP_EOP_PARITY)),
 721 /*10*/  FLAG_ENTRY0("RxDmaFlagUncErr", RXES(DMA_FLAG_UNC)),
 722 /*11*/  FLAG_ENTRY0("RxDmaFlagCorErr", RXES(DMA_FLAG_COR)),
 723 /*12*/  FLAG_ENTRY0("RxRcvFsmEncodingErr", RXES(RCV_FSM_ENCODING)),
 724 /*13*/  FLAG_ENTRY0("RxRbufFreeListUncErr", RXES(RBUF_FREE_LIST_UNC)),
 725 /*14*/  FLAG_ENTRY0("RxRbufFreeListCorErr", RXES(RBUF_FREE_LIST_COR)),
 726 /*15*/  FLAG_ENTRY0("RxRbufLookupDesRegUncErr", RXES(RBUF_LOOKUP_DES_REG_UNC)),
 727 /*16*/  FLAG_ENTRY0("RxRbufLookupDesRegUncCorErr",
 728                 RXES(RBUF_LOOKUP_DES_REG_UNC_COR)),
 729 /*17*/  FLAG_ENTRY0("RxRbufLookupDesUncErr", RXES(RBUF_LOOKUP_DES_UNC)),
 730 /*18*/  FLAG_ENTRY0("RxRbufLookupDesCorErr", RXES(RBUF_LOOKUP_DES_COR)),
 731 /*19*/  FLAG_ENTRY0("RxRbufBlockListReadUncErr",
 732                 RXES(RBUF_BLOCK_LIST_READ_UNC)),
 733 /*20*/  FLAG_ENTRY0("RxRbufBlockListReadCorErr",
 734                 RXES(RBUF_BLOCK_LIST_READ_COR)),
 735 /*21*/  FLAG_ENTRY0("RxRbufCsrQHeadBufNumParityErr",
 736                 RXES(RBUF_CSR_QHEAD_BUF_NUM_PARITY)),
 737 /*22*/  FLAG_ENTRY0("RxRbufCsrQEntCntParityErr",
 738                 RXES(RBUF_CSR_QENT_CNT_PARITY)),
 739 /*23*/  FLAG_ENTRY0("RxRbufCsrQNextBufParityErr",
 740                 RXES(RBUF_CSR_QNEXT_BUF_PARITY)),
 741 /*24*/  FLAG_ENTRY0("RxRbufCsrQVldBitParityErr",
 742                 RXES(RBUF_CSR_QVLD_BIT_PARITY)),
 743 /*25*/  FLAG_ENTRY0("RxRbufCsrQHdPtrParityErr", RXES(RBUF_CSR_QHD_PTR_PARITY)),
 744 /*26*/  FLAG_ENTRY0("RxRbufCsrQTlPtrParityErr", RXES(RBUF_CSR_QTL_PTR_PARITY)),
 745 /*27*/  FLAG_ENTRY0("RxRbufCsrQNumOfPktParityErr",
 746                 RXES(RBUF_CSR_QNUM_OF_PKT_PARITY)),
 747 /*28*/  FLAG_ENTRY0("RxRbufCsrQEOPDWParityErr", RXES(RBUF_CSR_QEOPDW_PARITY)),
 748 /*29*/  FLAG_ENTRY0("RxRbufCtxIdParityErr", RXES(RBUF_CTX_ID_PARITY)),
 749 /*30*/  FLAG_ENTRY0("RxRBufBadLookupErr", RXES(RBUF_BAD_LOOKUP)),
 750 /*31*/  FLAG_ENTRY0("RxRbufFullErr", RXES(RBUF_FULL)),
 751 /*32*/  FLAG_ENTRY0("RxRbufEmptyErr", RXES(RBUF_EMPTY)),
 752 /*33*/  FLAG_ENTRY0("RxRbufFlRdAddrParityErr", RXES(RBUF_FL_RD_ADDR_PARITY)),
 753 /*34*/  FLAG_ENTRY0("RxRbufFlWrAddrParityErr", RXES(RBUF_FL_WR_ADDR_PARITY)),
 754 /*35*/  FLAG_ENTRY0("RxRbufFlInitdoneParityErr",
 755                 RXES(RBUF_FL_INITDONE_PARITY)),
 756 /*36*/  FLAG_ENTRY0("RxRbufFlInitWrAddrParityErr",
 757                 RXES(RBUF_FL_INIT_WR_ADDR_PARITY)),
 758 /*37*/  FLAG_ENTRY0("RxRbufNextFreeBufUncErr", RXES(RBUF_NEXT_FREE_BUF_UNC)),
 759 /*38*/  FLAG_ENTRY0("RxRbufNextFreeBufCorErr", RXES(RBUF_NEXT_FREE_BUF_COR)),
 760 /*39*/  FLAG_ENTRY0("RxLookupDesPart1UncErr", RXES(LOOKUP_DES_PART1_UNC)),
 761 /*40*/  FLAG_ENTRY0("RxLookupDesPart1UncCorErr",
 762                 RXES(LOOKUP_DES_PART1_UNC_COR)),
 763 /*41*/  FLAG_ENTRY0("RxLookupDesPart2ParityErr",
 764                 RXES(LOOKUP_DES_PART2_PARITY)),
 765 /*42*/  FLAG_ENTRY0("RxLookupRcvArrayUncErr", RXES(LOOKUP_RCV_ARRAY_UNC)),
 766 /*43*/  FLAG_ENTRY0("RxLookupRcvArrayCorErr", RXES(LOOKUP_RCV_ARRAY_COR)),
 767 /*44*/  FLAG_ENTRY0("RxLookupCsrParityErr", RXES(LOOKUP_CSR_PARITY)),
 768 /*45*/  FLAG_ENTRY0("RxHqIntrCsrParityErr", RXES(HQ_INTR_CSR_PARITY)),
 769 /*46*/  FLAG_ENTRY0("RxHqIntrFsmErr", RXES(HQ_INTR_FSM)),
 770 /*47*/  FLAG_ENTRY0("RxRbufDescPart1UncErr", RXES(RBUF_DESC_PART1_UNC)),
 771 /*48*/  FLAG_ENTRY0("RxRbufDescPart1CorErr", RXES(RBUF_DESC_PART1_COR)),
 772 /*49*/  FLAG_ENTRY0("RxRbufDescPart2UncErr", RXES(RBUF_DESC_PART2_UNC)),
 773 /*50*/  FLAG_ENTRY0("RxRbufDescPart2CorErr", RXES(RBUF_DESC_PART2_COR)),
 774 /*51*/  FLAG_ENTRY0("RxDmaHdrFifoRdUncErr", RXES(DMA_HDR_FIFO_RD_UNC)),
 775 /*52*/  FLAG_ENTRY0("RxDmaHdrFifoRdCorErr", RXES(DMA_HDR_FIFO_RD_COR)),
 776 /*53*/  FLAG_ENTRY0("RxDmaDataFifoRdUncErr", RXES(DMA_DATA_FIFO_RD_UNC)),
 777 /*54*/  FLAG_ENTRY0("RxDmaDataFifoRdCorErr", RXES(DMA_DATA_FIFO_RD_COR)),
 778 /*55*/  FLAG_ENTRY0("RxRbufDataUncErr", RXES(RBUF_DATA_UNC)),
 779 /*56*/  FLAG_ENTRY0("RxRbufDataCorErr", RXES(RBUF_DATA_COR)),
 780 /*57*/  FLAG_ENTRY0("RxDmaCsrParityErr", RXES(DMA_CSR_PARITY)),
 781 /*58*/  FLAG_ENTRY0("RxDmaEqFsmEncodingErr", RXES(DMA_EQ_FSM_ENCODING)),
 782 /*59*/  FLAG_ENTRY0("RxDmaDqFsmEncodingErr", RXES(DMA_DQ_FSM_ENCODING)),
 783 /*60*/  FLAG_ENTRY0("RxDmaCsrUncErr", RXES(DMA_CSR_UNC)),
 784 /*61*/  FLAG_ENTRY0("RxCsrReadBadAddrErr", RXES(CSR_READ_BAD_ADDR)),
 785 /*62*/  FLAG_ENTRY0("RxCsrWriteBadAddrErr", RXES(CSR_WRITE_BAD_ADDR)),
 786 /*63*/  FLAG_ENTRY0("RxCsrParityErr", RXES(CSR_PARITY))
 787 };
 788
 789 /* RXE errors that will trigger an SPC freeze */
 790 #define ALL_RXE_FREEZE_ERR  \
 791         (RCV_ERR_STATUS_RX_RCV_QP_MAP_TABLE_UNC_ERR_SMASK \
 792         | RCV_ERR_STATUS_RX_RCV_CSR_PARITY_ERR_SMASK \
 793         | RCV_ERR_STATUS_RX_DMA_FLAG_UNC_ERR_SMASK \
 794         | RCV_ERR_STATUS_RX_RCV_FSM_ENCODING_ERR_SMASK \
 795         | RCV_ERR_STATUS_RX_RBUF_FREE_LIST_UNC_ERR_SMASK \
 796         | RCV_ERR_STATUS_RX_RBUF_LOOKUP_DES_REG_UNC_ERR_SMASK \
 797         | RCV_ERR_STATUS_RX_RBUF_LOOKUP_DES_REG_UNC_COR_ERR_SMASK \
 798         | RCV_ERR_STATUS_RX_RBUF_LOOKUP_DES_UNC_ERR_SMASK \
 799         | RCV_ERR_STATUS_RX_RBUF_BLOCK_LIST_READ_UNC_ERR_SMASK \
 800         | RCV_ERR_STATUS_RX_RBUF_CSR_QHEAD_BUF_NUM_PARITY_ERR_SMASK \
 801         | RCV_ERR_STATUS_RX_RBUF_CSR_QENT_CNT_PARITY_ERR_SMASK \
 802         | RCV_ERR_STATUS_RX_RBUF_CSR_QNEXT_BUF_PARITY_ERR_SMASK \
 803         | RCV_ERR_STATUS_RX_RBUF_CSR_QVLD_BIT_PARITY_ERR_SMASK \
 804         | RCV_ERR_STATUS_RX_RBUF_CSR_QHD_PTR_PARITY_ERR_SMASK \
 805         | RCV_ERR_STATUS_RX_RBUF_CSR_QTL_PTR_PARITY_ERR_SMASK \
 806         | RCV_ERR_STATUS_RX_RBUF_CSR_QNUM_OF_PKT_PARITY_ERR_SMASK \
 807         | RCV_ERR_STATUS_RX_RBUF_CSR_QEOPDW_PARITY_ERR_SMASK \
 808         | RCV_ERR_STATUS_RX_RBUF_CTX_ID_PARITY_ERR_SMASK \
 809         | RCV_ERR_STATUS_RX_RBUF_BAD_LOOKUP_ERR_SMASK \
 810         | RCV_ERR_STATUS_RX_RBUF_FULL_ERR_SMASK \
 811         | RCV_ERR_STATUS_RX_RBUF_EMPTY_ERR_SMASK \
 812         | RCV_ERR_STATUS_RX_RBUF_FL_RD_ADDR_PARITY_ERR_SMASK \
 813         | RCV_ERR_STATUS_RX_RBUF_FL_WR_ADDR_PARITY_ERR_SMASK \
 814         | RCV_ERR_STATUS_RX_RBUF_FL_INITDONE_PARITY_ERR_SMASK \
 815         | RCV_ERR_STATUS_RX_RBUF_FL_INIT_WR_ADDR_PARITY_ERR_SMASK \
 816         | RCV_ERR_STATUS_RX_RBUF_NEXT_FREE_BUF_UNC_ERR_SMASK \
 817         | RCV_ERR_STATUS_RX_LOOKUP_DES_PART1_UNC_ERR_SMASK \
 818         | RCV_ERR_STATUS_RX_LOOKUP_DES_PART1_UNC_COR_ERR_SMASK \
 819         | RCV_ERR_STATUS_RX_LOOKUP_DES_PART2_PARITY_ERR_SMASK \
 820         | RCV_ERR_STATUS_RX_LOOKUP_RCV_ARRAY_UNC_ERR_SMASK \
 821         | RCV_ERR_STATUS_RX_LOOKUP_CSR_PARITY_ERR_SMASK \
 822         | RCV_ERR_STATUS_RX_HQ_INTR_CSR_PARITY_ERR_SMASK \
 823         | RCV_ERR_STATUS_RX_HQ_INTR_FSM_ERR_SMASK \
 824         | RCV_ERR_STATUS_RX_RBUF_DESC_PART1_UNC_ERR_SMASK \
 825         | RCV_ERR_STATUS_RX_RBUF_DESC_PART1_COR_ERR_SMASK \
 826         | RCV_ERR_STATUS_RX_RBUF_DESC_PART2_UNC_ERR_SMASK \
 827         | RCV_ERR_STATUS_RX_DMA_HDR_FIFO_RD_UNC_ERR_SMASK \
 828         | RCV_ERR_STATUS_RX_DMA_DATA_FIFO_RD_UNC_ERR_SMASK \
 829         | RCV_ERR_STATUS_RX_RBUF_DATA_UNC_ERR_SMASK \
 830         | RCV_ERR_STATUS_RX_DMA_CSR_PARITY_ERR_SMASK \
 831         | RCV_ERR_STATUS_RX_DMA_EQ_FSM_ENCODING_ERR_SMASK \
 832         | RCV_ERR_STATUS_RX_DMA_DQ_FSM_ENCODING_ERR_SMASK \
 833         | RCV_ERR_STATUS_RX_DMA_CSR_UNC_ERR_SMASK \
 834         | RCV_ERR_STATUS_RX_CSR_PARITY_ERR_SMASK)
 835
 836 #define RXE_FREEZE_ABORT_MASK \
 837         (RCV_ERR_STATUS_RX_DMA_CSR_UNC_ERR_SMASK | \
 838         RCV_ERR_STATUS_RX_DMA_HDR_FIFO_RD_UNC_ERR_SMASK | \
 839         RCV_ERR_STATUS_RX_DMA_DATA_FIFO_RD_UNC_ERR_SMASK)
 840
 841 /*
 842  * DCC Error Flags
 843  */
 844 #define DCCE(name) DCC_ERR_FLG_##name##_SMASK
 845 static struct flag_table dcc_err_flags[] = {
 846         FLAG_ENTRY0("bad_l2_err", DCCE(BAD_L2_ERR)),
 847         FLAG_ENTRY0("bad_sc_err", DCCE(BAD_SC_ERR)),
 848         FLAG_ENTRY0("bad_mid_tail_err", DCCE(BAD_MID_TAIL_ERR)),
 849         FLAG_ENTRY0("bad_preemption_err", DCCE(BAD_PREEMPTION_ERR)),
 850         FLAG_ENTRY0("preemption_err", DCCE(PREEMPTION_ERR)),
 851         FLAG_ENTRY0("preemptionvl15_err", DCCE(PREEMPTIONVL15_ERR)),
 852         FLAG_ENTRY0("bad_vl_marker_err", DCCE(BAD_VL_MARKER_ERR)),
 853         FLAG_ENTRY0("bad_dlid_target_err", DCCE(BAD_DLID_TARGET_ERR)),
 854         FLAG_ENTRY0("bad_lver_err", DCCE(BAD_LVER_ERR)),
 855         FLAG_ENTRY0("uncorrectable_err", DCCE(UNCORRECTABLE_ERR)),
 856         FLAG_ENTRY0("bad_crdt_ack_err", DCCE(BAD_CRDT_ACK_ERR)),
 857         FLAG_ENTRY0("unsup_pkt_type", DCCE(UNSUP_PKT_TYPE)),
 858         FLAG_ENTRY0("bad_ctrl_flit_err", DCCE(BAD_CTRL_FLIT_ERR)),
 859         FLAG_ENTRY0("event_cntr_parity_err", DCCE(EVENT_CNTR_PARITY_ERR)),
 860         FLAG_ENTRY0("event_cntr_rollover_err", DCCE(EVENT_CNTR_ROLLOVER_ERR)),
 861         FLAG_ENTRY0("link_err", DCCE(LINK_ERR)),
 862         FLAG_ENTRY0("misc_cntr_rollover_err", DCCE(MISC_CNTR_ROLLOVER_ERR)),
 863         FLAG_ENTRY0("bad_ctrl_dist_err", DCCE(BAD_CTRL_DIST_ERR)),
 864         FLAG_ENTRY0("bad_tail_dist_err", DCCE(BAD_TAIL_DIST_ERR)),
 865         FLAG_ENTRY0("bad_head_dist_err", DCCE(BAD_HEAD_DIST_ERR)),
 866         FLAG_ENTRY0("nonvl15_state_err", DCCE(NONVL15_STATE_ERR)),
 867         FLAG_ENTRY0("vl15_multi_err", DCCE(VL15_MULTI_ERR)),
 868         FLAG_ENTRY0("bad_pkt_length_err", DCCE(BAD_PKT_LENGTH_ERR)),
 869         FLAG_ENTRY0("unsup_vl_err", DCCE(UNSUP_VL_ERR)),
 870         FLAG_ENTRY0("perm_nvl15_err", DCCE(PERM_NVL15_ERR)),
 871         FLAG_ENTRY0("slid_zero_err", DCCE(SLID_ZERO_ERR)),
 872         FLAG_ENTRY0("dlid_zero_err", DCCE(DLID_ZERO_ERR)),
 873         FLAG_ENTRY0("length_mtu_err", DCCE(LENGTH_MTU_ERR)),
 874         FLAG_ENTRY0("rx_early_drop_err", DCCE(RX_EARLY_DROP_ERR)),
 875         FLAG_ENTRY0("late_short_err", DCCE(LATE_SHORT_ERR)),
 876         FLAG_ENTRY0("late_long_err", DCCE(LATE_LONG_ERR)),
 877         FLAG_ENTRY0("late_ebp_err", DCCE(LATE_EBP_ERR)),
 878         FLAG_ENTRY0("fpe_tx_fifo_ovflw_err", DCCE(FPE_TX_FIFO_OVFLW_ERR)),
 879         FLAG_ENTRY0("fpe_tx_fifo_unflw_err", DCCE(FPE_TX_FIFO_UNFLW_ERR)),
 880         FLAG_ENTRY0("csr_access_blocked_host", DCCE(CSR_ACCESS_BLOCKED_HOST)),
 881         FLAG_ENTRY0("csr_access_blocked_uc", DCCE(CSR_ACCESS_BLOCKED_UC)),
 882         FLAG_ENTRY0("tx_ctrl_parity_err", DCCE(TX_CTRL_PARITY_ERR)),
 883         FLAG_ENTRY0("tx_ctrl_parity_mbe_err", DCCE(TX_CTRL_PARITY_MBE_ERR)),
 884         FLAG_ENTRY0("tx_sc_parity_err", DCCE(TX_SC_PARITY_ERR)),
 885         FLAG_ENTRY0("rx_ctrl_parity_mbe_err", DCCE(RX_CTRL_PARITY_MBE_ERR)),
 886         FLAG_ENTRY0("csr_parity_err", DCCE(CSR_PARITY_ERR)),
 887         FLAG_ENTRY0("csr_inval_addr", DCCE(CSR_INVAL_ADDR)),
 888         FLAG_ENTRY0("tx_byte_shft_parity_err", DCCE(TX_BYTE_SHFT_PARITY_ERR)),
 889         FLAG_ENTRY0("rx_byte_shft_parity_err", DCCE(RX_BYTE_SHFT_PARITY_ERR)),
 890         FLAG_ENTRY0("fmconfig_err", DCCE(FMCONFIG_ERR)),
 891         FLAG_ENTRY0("rcvport_err", DCCE(RCVPORT_ERR)),
 892 };
 893
 894 /*
 895  * LCB error flags
 896  */
 897 #define LCBE(name) DC_LCB_ERR_FLG_##name##_SMASK
 898 static struct flag_table lcb_err_flags[] = {
 899 /* 0*/  FLAG_ENTRY0("CSR_PARITY_ERR", LCBE(CSR_PARITY_ERR)),
 900 /* 1*/  FLAG_ENTRY0("INVALID_CSR_ADDR", LCBE(INVALID_CSR_ADDR)),
 901 /* 2*/  FLAG_ENTRY0("RST_FOR_FAILED_DESKEW", LCBE(RST_FOR_FAILED_DESKEW)),
 902 /* 3*/  FLAG_ENTRY0("ALL_LNS_FAILED_REINIT_TEST",
 903                 LCBE(ALL_LNS_FAILED_REINIT_TEST)),
 904 /* 4*/  FLAG_ENTRY0("LOST_REINIT_STALL_OR_TOS", LCBE(LOST_REINIT_STALL_OR_TOS)),
 905 /* 5*/  FLAG_ENTRY0("TX_LESS_THAN_FOUR_LNS", LCBE(TX_LESS_THAN_FOUR_LNS)),
 906 /* 6*/  FLAG_ENTRY0("RX_LESS_THAN_FOUR_LNS", LCBE(RX_LESS_THAN_FOUR_LNS)),
 907 /* 7*/  FLAG_ENTRY0("SEQ_CRC_ERR", LCBE(SEQ_CRC_ERR)),
 908 /* 8*/  FLAG_ENTRY0("REINIT_FROM_PEER", LCBE(REINIT_FROM_PEER)),
 909 /* 9*/  FLAG_ENTRY0("REINIT_FOR_LN_DEGRADE", LCBE(REINIT_FOR_LN_DEGRADE)),
 910 /*10*/  FLAG_ENTRY0("CRC_ERR_CNT_HIT_LIMIT", LCBE(CRC_ERR_CNT_HIT_LIMIT)),
 911 /*11*/  FLAG_ENTRY0("RCLK_STOPPED", LCBE(RCLK_STOPPED)),
 912 /*12*/  FLAG_ENTRY0("UNEXPECTED_REPLAY_MARKER", LCBE(UNEXPECTED_REPLAY_MARKER)),
 913 /*13*/  FLAG_ENTRY0("UNEXPECTED_ROUND_TRIP_MARKER",
 914                 LCBE(UNEXPECTED_ROUND_TRIP_MARKER)),
 915 /*14*/  FLAG_ENTRY0("ILLEGAL_NULL_LTP", LCBE(ILLEGAL_NULL_LTP)),
 916 /*15*/  FLAG_ENTRY0("ILLEGAL_FLIT_ENCODING", LCBE(ILLEGAL_FLIT_ENCODING)),
 917 /*16*/  FLAG_ENTRY0("FLIT_INPUT_BUF_OFLW", LCBE(FLIT_INPUT_BUF_OFLW)),
 918 /*17*/  FLAG_ENTRY0("VL_ACK_INPUT_BUF_OFLW", LCBE(VL_ACK_INPUT_BUF_OFLW)),
 919 /*18*/  FLAG_ENTRY0("VL_ACK_INPUT_PARITY_ERR", LCBE(VL_ACK_INPUT_PARITY_ERR)),
 920 /*19*/  FLAG_ENTRY0("VL_ACK_INPUT_WRONG_CRC_MODE",
 921                 LCBE(VL_ACK_INPUT_WRONG_CRC_MODE)),
 922 /*20*/  FLAG_ENTRY0("FLIT_INPUT_BUF_MBE", LCBE(FLIT_INPUT_BUF_MBE)),
 923 /*21*/  FLAG_ENTRY0("FLIT_INPUT_BUF_SBE", LCBE(FLIT_INPUT_BUF_SBE)),
 924 /*22*/  FLAG_ENTRY0("REPLAY_BUF_MBE", LCBE(REPLAY_BUF_MBE)),
 925 /*23*/  FLAG_ENTRY0("REPLAY_BUF_SBE", LCBE(REPLAY_BUF_SBE)),
 926 /*24*/  FLAG_ENTRY0("CREDIT_RETURN_FLIT_MBE", LCBE(CREDIT_RETURN_FLIT_MBE)),
 927 /*25*/  FLAG_ENTRY0("RST_FOR_LINK_TIMEOUT", LCBE(RST_FOR_LINK_TIMEOUT)),
 928 /*26*/  FLAG_ENTRY0("RST_FOR_INCOMPLT_RND_TRIP",
 929                 LCBE(RST_FOR_INCOMPLT_RND_TRIP)),
 930 /*27*/  FLAG_ENTRY0("HOLD_REINIT", LCBE(HOLD_REINIT)),
 931 /*28*/  FLAG_ENTRY0("NEG_EDGE_LINK_TRANSFER_ACTIVE",
 932                 LCBE(NEG_EDGE_LINK_TRANSFER_ACTIVE)),
 933 /*29*/  FLAG_ENTRY0("REDUNDANT_FLIT_PARITY_ERR",
 934                 LCBE(REDUNDANT_FLIT_PARITY_ERR))
 935 };
 936
 937 /*
 938  * DC8051 Error Flags
 939  */
 940 #define D8E(name) DC_DC8051_ERR_FLG_##name##_SMASK
 941 static struct flag_table dc8051_err_flags[] = {
 942         FLAG_ENTRY0("SET_BY_8051", D8E(SET_BY_8051)),
 943         FLAG_ENTRY0("LOST_8051_HEART_BEAT", D8E(LOST_8051_HEART_BEAT)),
 944         FLAG_ENTRY0("CRAM_MBE", D8E(CRAM_MBE)),
 945         FLAG_ENTRY0("CRAM_SBE", D8E(CRAM_SBE)),
 946         FLAG_ENTRY0("DRAM_MBE", D8E(DRAM_MBE)),
 947         FLAG_ENTRY0("DRAM_SBE", D8E(DRAM_SBE)),
 948         FLAG_ENTRY0("IRAM_MBE", D8E(IRAM_MBE)),
 949         FLAG_ENTRY0("IRAM_SBE", D8E(IRAM_SBE)),
 950         FLAG_ENTRY0("UNMATCHED_SECURE_MSG_ACROSS_BCC_LANES",
 951                     D8E(UNMATCHED_SECURE_MSG_ACROSS_BCC_LANES)),
 952         FLAG_ENTRY0("INVALID_CSR_ADDR", D8E(INVALID_CSR_ADDR)),
 953 };
 954
 955 /*
 956  * DC8051 Information Error flags
 957  *
 958  * Flags in DC8051_DBG_ERR_INFO_SET_BY_8051.ERROR field.
 959  */
 960 static struct flag_table dc8051_info_err_flags[] = {
 961         FLAG_ENTRY0("Spico ROM check failed",  SPICO_ROM_FAILED),
 962         FLAG_ENTRY0("Unknown frame received",  UNKNOWN_FRAME),
 963         FLAG_ENTRY0("Target BER not met",      TARGET_BER_NOT_MET),
 964         FLAG_ENTRY0("Serdes internal loopback failure",
 965                     FAILED_SERDES_INTERNAL_LOOPBACK),
 966         FLAG_ENTRY0("Failed SerDes init",      FAILED_SERDES_INIT),
 967         FLAG_ENTRY0("Failed LNI(Polling)",     FAILED_LNI_POLLING),
 968         FLAG_ENTRY0("Failed LNI(Debounce)",    FAILED_LNI_DEBOUNCE),
 969         FLAG_ENTRY0("Failed LNI(EstbComm)",    FAILED_LNI_ESTBCOMM),
 970         FLAG_ENTRY0("Failed LNI(OptEq)",       FAILED_LNI_OPTEQ),
 971         FLAG_ENTRY0("Failed LNI(VerifyCap_1)", FAILED_LNI_VERIFY_CAP1),
 972         FLAG_ENTRY0("Failed LNI(VerifyCap_2)", FAILED_LNI_VERIFY_CAP2),
 973         FLAG_ENTRY0("Failed LNI(ConfigLT)",    FAILED_LNI_CONFIGLT),
 974         FLAG_ENTRY0("Host Handshake Timeout",  HOST_HANDSHAKE_TIMEOUT),
 975         FLAG_ENTRY0("External Device Request Timeout",
 976                     EXTERNAL_DEVICE_REQ_TIMEOUT),
 977 };
 978
 979 /*
 980  * DC8051 Information Host Information flags
 981  *
 982  * Flags in DC8051_DBG_ERR_INFO_SET_BY_8051.HOST_MSG field.
 983  */
 984 static struct flag_table dc8051_info_host_msg_flags[] = {
 985         FLAG_ENTRY0("Host request done", 0x0001),
 986         FLAG_ENTRY0("BC SMA message", 0x0002),
 987         FLAG_ENTRY0("BC PWR_MGM message", 0x0004),
 988         FLAG_ENTRY0("BC Unknown message (BCC)", 0x0008),
 989         FLAG_ENTRY0("BC Unknown message (LCB)", 0x0010),
 990         FLAG_ENTRY0("External device config request", 0x0020),
 991         FLAG_ENTRY0("VerifyCap all frames received", 0x0040),
 992         FLAG_ENTRY0("LinkUp achieved", 0x0080),
 993         FLAG_ENTRY0("Link going down", 0x0100),
 994 };
 995
 996 static u32 encoded_size(u32 size);
 997 static u32 chip_to_opa_lstate(struct hfi1_devdata *dd, u32 chip_lstate);
 998 static int set_physical_link_state(struct hfi1_devdata *dd, u64 state);
 999 static void read_vc_remote_phy(struct hfi1_devdata *dd, u8 *power_management,
1000                                u8 *continuous);
1001 static void read_vc_remote_fabric(struct hfi1_devdata *dd, u8 *vau, u8 *z,
1002                                   u8 *vcu, u16 *vl15buf, u8 *crc_sizes);
1003 static void read_vc_remote_link_width(struct hfi1_devdata *dd,
1004                                       u8 *remote_tx_rate, u16 *link_widths);
1005 static void read_vc_local_link_width(struct hfi1_devdata *dd, u8 *misc_bits,
1006                                      u8 *flag_bits, u16 *link_widths);
1007 static void read_remote_device_id(struct hfi1_devdata *dd, u16 *device_id,
1008                                   u8 *device_rev);
1009 static void read_mgmt_allowed(struct hfi1_devdata *dd, u8 *mgmt_allowed);
1010 static void read_local_lni(struct hfi1_devdata *dd, u8 *enable_lane_rx);
1011 static int read_tx_settings(struct hfi1_devdata *dd, u8 *enable_lane_tx,
1012                             u8 *tx_polarity_inversion,
1013                             u8 *rx_polarity_inversion, u8 *max_rate);
1014 static void handle_sdma_eng_err(struct hfi1_devdata *dd,
1015                                 unsigned int context, u64 err_status);
1016 static void handle_qsfp_int(struct hfi1_devdata *dd, u32 source, u64 reg);
1017 static void handle_dcc_err(struct hfi1_devdata *dd,
1018                            unsigned int context, u64 err_status);
1019 static void handle_lcb_err(struct hfi1_devdata *dd,
1020                            unsigned int context, u64 err_status);
1021 static void handle_8051_interrupt(struct hfi1_devdata *dd, u32 unused, u64 reg);
1022 static void handle_cce_err(struct hfi1_devdata *dd, u32 unused, u64 reg);
1023 static void handle_rxe_err(struct hfi1_devdata *dd, u32 unused, u64 reg);
1024 static void handle_misc_err(struct hfi1_devdata *dd, u32 unused, u64 reg);
1025 static void handle_pio_err(struct hfi1_devdata *dd, u32 unused, u64 reg);
1026 static void handle_sdma_err(struct hfi1_devdata *dd, u32 unused, u64 reg);
1027 static void handle_egress_err(struct hfi1_devdata *dd, u32 unused, u64 reg);
1028 static void handle_txe_err(struct hfi1_devdata *dd, u32 unused, u64 reg);
1029 static void set_partition_keys(struct hfi1_pportdata *);
1030 static const char *link_state_name(u32 state);
1031 static const char *link_state_reason_name(struct hfi1_pportdata *ppd,
1032                                           u32 state);
1033 static int do_8051_command(struct hfi1_devdata *dd, u32 type, u64 in_data,
1034                            u64 *out_data);
1035 static int read_idle_sma(struct hfi1_devdata *dd, u64 *data);
1036 static int thermal_init(struct hfi1_devdata *dd);
1037
1038 static int wait_logical_linkstate(struct hfi1_pportdata *ppd, u32 state,
1039                                   int msecs);
1040 static void read_planned_down_reason_code(struct hfi1_devdata *dd, u8 *pdrrc);
1041 static void read_link_down_reason(struct hfi1_devdata *dd, u8 *ldr);
1042 static void handle_temp_err(struct hfi1_devdata *);
1043 static void dc_shutdown(struct hfi1_devdata *);
1044 static void dc_start(struct hfi1_devdata *);
1045 static int qos_rmt_entries(struct hfi1_devdata *dd, unsigned int *mp,
1046                            unsigned int *np);
1047 static void clear_full_mgmt_pkey(struct hfi1_pportdata *ppd);
1048
1049 /*
1050  * Error interrupt table entry.  This is used as input to the interrupt
1051  * "clear down" routine used for all second tier error interrupt register.
1052  * Second tier interrupt registers have a single bit representing them
1053  * in the top-level CceIntStatus.
1054  */
1055 struct err_reg_info {
1056         u32 status;             /* status CSR offset */
1057         u32 clear;              /* clear CSR offset */
1058         u32 mask;               /* mask CSR offset */
1059         void (*handler)(struct hfi1_devdata *dd, u32 source, u64 reg);
1060         const char *desc;
1061 };
1062
1063 #define NUM_MISC_ERRS (IS_GENERAL_ERR_END - IS_GENERAL_ERR_START)
1064 #define NUM_DC_ERRS (IS_DC_END - IS_DC_START)
1065 #define NUM_VARIOUS (IS_VARIOUS_END - IS_VARIOUS_START)
1066
1067 /*
1068  * Helpers for building HFI and DC error interrupt table entries.  Different
1069  * helpers are needed because of inconsistent register names.
1070  */
1071 #define EE(reg, handler, desc) \
1072         { reg##_STATUS, reg##_CLEAR, reg##_MASK, \
1073                 handler, desc }
1074 #define DC_EE1(reg, handler, desc) \
1075         { reg##_FLG, reg##_FLG_CLR, reg##_FLG_EN, handler, desc }
1076 #define DC_EE2(reg, handler, desc) \
1077         { reg##_FLG, reg##_CLR, reg##_EN, handler, desc }
1078
1079 /*
1080  * Table of the "misc" grouping of error interrupts.  Each entry refers to
1081  * another register containing more information.
1082  */
1083 static const struct err_reg_info misc_errs[NUM_MISC_ERRS] = {
1084 /* 0*/  EE(CCE_ERR,             handle_cce_err,    "CceErr"),
1085 /* 1*/  EE(RCV_ERR,             handle_rxe_err,    "RxeErr"),
1086 /* 2*/  EE(MISC_ERR,    handle_misc_err,   "MiscErr"),
1087 /* 3*/  { 0, 0, 0, NULL }, /* reserved */
1088 /* 4*/  EE(SEND_PIO_ERR,    handle_pio_err,    "PioErr"),
1089 /* 5*/  EE(SEND_DMA_ERR,    handle_sdma_err,   "SDmaErr"),
1090 /* 6*/  EE(SEND_EGRESS_ERR, handle_egress_err, "EgressErr"),
1091 /* 7*/  EE(SEND_ERR,    handle_txe_err,    "TxeErr")
1092         /* the rest are reserved */
1093 };
1094
1095 /*
1096  * Index into the Various section of the interrupt sources
1097  * corresponding to the Critical Temperature interrupt.
1098  */
1099 #define TCRIT_INT_SOURCE 4
1100
1101 /*
1102  * SDMA error interrupt entry - refers to another register containing more
1103  * information.
1104  */
1105 static const struct err_reg_info sdma_eng_err =
1106         EE(SEND_DMA_ENG_ERR, handle_sdma_eng_err, "SDmaEngErr");
1107
1108 static const struct err_reg_info various_err[NUM_VARIOUS] = {
1109 /* 0*/  { 0, 0, 0, NULL }, /* PbcInt */
1110 /* 1*/  { 0, 0, 0, NULL }, /* GpioAssertInt */
1111 /* 2*/  EE(ASIC_QSFP1,  handle_qsfp_int,        "QSFP1"),
1112 /* 3*/  EE(ASIC_QSFP2,  handle_qsfp_int,        "QSFP2"),
1113 /* 4*/  { 0, 0, 0, NULL }, /* TCritInt */
1114         /* rest are reserved */
1115 };
1116
1117 /*
1118  * The DC encoding of mtu_cap for 10K MTU in the DCC_CFG_PORT_CONFIG
1119  * register can not be derived from the MTU value because 10K is not
1120  * a power of 2. Therefore, we need a constant. Everything else can
1121  * be calculated.
1122  */
1123 #define DCC_CFG_PORT_MTU_CAP_10240 7
1124
1125 /*
1126  * Table of the DC grouping of error interrupts.  Each entry refers to
1127  * another register containing more information.
1128  */
1129 static const struct err_reg_info dc_errs[NUM_DC_ERRS] = {
1130 /* 0*/  DC_EE1(DCC_ERR,         handle_dcc_err,        "DCC Err"),
1131 /* 1*/  DC_EE2(DC_LCB_ERR,      handle_lcb_err,        "LCB Err"),
1132 /* 2*/  DC_EE2(DC_DC8051_ERR,   handle_8051_interrupt, "DC8051 Interrupt"),
1133 /* 3*/  /* dc_lbm_int - special, see is_dc_int() */
1134         /* the rest are reserved */
1135 };
1136
1137 struct cntr_entry {
1138         /*
1139          * counter name
1140          */
1141         char *name;
1142
1143         /*
1144          * csr to read for name (if applicable)
1145          */
1146         u64 csr;
1147
1148         /*
1149          * offset into dd or ppd to store the counter's value
1150          */
1151         int offset;
1152
1153         /*
1154          * flags
1155          */
1156         u8 flags;
1157
1158         /*
1159          * accessor for stat element, context either dd or ppd
1160          */
1161         u64 (*rw_cntr)(const struct cntr_entry *, void *context, int vl,
1162                        int mode, u64 data);
1163 };
1164
1165 #define C_RCV_HDR_OVF_FIRST C_RCV_HDR_OVF_0
1166 #define C_RCV_HDR_OVF_LAST C_RCV_HDR_OVF_159
1167
1168 #define CNTR_ELEM(name, csr, offset, flags, accessor) \
1169 { \
1170         name, \
1171         csr, \
1172         offset, \
1173         flags, \
1174         accessor \
1175 }
1176
1177 /* 32bit RXE */
1178 #define RXE32_PORT_CNTR_ELEM(name, counter, flags) \
1179 CNTR_ELEM(#name, \
1180           (counter * 8 + RCV_COUNTER_ARRAY32), \
1181           0, flags | CNTR_32BIT, \
1182           port_access_u32_csr)
1183
1184 #define RXE32_DEV_CNTR_ELEM(name, counter, flags) \
1185 CNTR_ELEM(#name, \
1186           (counter * 8 + RCV_COUNTER_ARRAY32), \
1187           0, flags | CNTR_32BIT, \
1188           dev_access_u32_csr)
1189
1190 /* 64bit RXE */
1191 #define RXE64_PORT_CNTR_ELEM(name, counter, flags) \
1192 CNTR_ELEM(#name, \
1193           (counter * 8 + RCV_COUNTER_ARRAY64), \
1194           0, flags, \
1195           port_access_u64_csr)
1196
1197 #define RXE64_DEV_CNTR_ELEM(name, counter, flags) \
1198 CNTR_ELEM(#name, \
1199           (counter * 8 + RCV_COUNTER_ARRAY64), \
1200           0, flags, \
1201           dev_access_u64_csr)
1202
1203 #define OVR_LBL(ctx) C_RCV_HDR_OVF_ ## ctx
1204 #define OVR_ELM(ctx) \
1205 CNTR_ELEM("RcvHdrOvr" #ctx, \
1206           (RCV_HDR_OVFL_CNT + ctx * 0x100), \
1207           0, CNTR_NORMAL, port_access_u64_csr)
1208
1209 /* 32bit TXE */
1210 #define TXE32_PORT_CNTR_ELEM(name, counter, flags) \
1211 CNTR_ELEM(#name, \
1212           (counter * 8 + SEND_COUNTER_ARRAY32), \
1213           0, flags | CNTR_32BIT, \
1214           port_access_u32_csr)
1215
1216 /* 64bit TXE */
1217 #define TXE64_PORT_CNTR_ELEM(name, counter, flags) \
1218 CNTR_ELEM(#name, \
1219           (counter * 8 + SEND_COUNTER_ARRAY64), \
1220           0, flags, \
1221           port_access_u64_csr)
1222
1223 # define TX64_DEV_CNTR_ELEM(name, counter, flags) \
1224 CNTR_ELEM(#name,\
1225           counter * 8 + SEND_COUNTER_ARRAY64, \
1226           0, \
1227           flags, \
1228           dev_access_u64_csr)
1229
1230 /* CCE */
1231 #define CCE_PERF_DEV_CNTR_ELEM(name, counter, flags) \
1232 CNTR_ELEM(#name, \
1233           (counter * 8 + CCE_COUNTER_ARRAY32), \
1234           0, flags | CNTR_32BIT, \
1235           dev_access_u32_csr)
1236
1237 #define CCE_INT_DEV_CNTR_ELEM(name, counter, flags) \
1238 CNTR_ELEM(#name, \
1239           (counter * 8 + CCE_INT_COUNTER_ARRAY32), \
1240           0, flags | CNTR_32BIT, \
1241           dev_access_u32_csr)
1242
1243 /* DC */
1244 #define DC_PERF_CNTR(name, counter, flags) \
1245 CNTR_ELEM(#name, \
1246           counter, \
1247           0, \
1248           flags, \
1249           dev_access_u64_csr)
1250
1251 #define DC_PERF_CNTR_LCB(name, counter, flags) \
1252 CNTR_ELEM(#name, \
1253           counter, \
1254           0, \
1255           flags, \
1256           dc_access_lcb_cntr)
1257
1258 /* ibp counters */
1259 #define SW_IBP_CNTR(name, cntr) \
1260 CNTR_ELEM(#name, \
1261           0, \
1262           0, \
1263           CNTR_SYNTH, \
1264           access_ibp_##cntr)
1265
1266 u64 read_csr(const struct hfi1_devdata *dd, u32 offset)
1267 {
1268         if (dd->flags & HFI1_PRESENT) {
1269                 return readq((void __iomem *)dd->kregbase + offset);
1270         }
1271         return -1;
1272 }
1273
1274 void write_csr(const struct hfi1_devdata *dd, u32 offset, u64 value)
1275 {
1276         if (dd->flags & HFI1_PRESENT)
1277                 writeq(value, (void __iomem *)dd->kregbase + offset);
1278 }
1279
1280 void __iomem *get_csr_addr(
1281         struct hfi1_devdata *dd,
1282         u32 offset)
1283 {
1284         return (void __iomem *)dd->kregbase + offset;
1285 }
1286
1287 static inline u64 read_write_csr(const struct hfi1_devdata *dd, u32 csr,
1288                                  int mode, u64 value)
1289 {
1290         u64 ret;
1291
1292         if (mode == CNTR_MODE_R) {
1293                 ret = read_csr(dd, csr);
1294         } else if (mode == CNTR_MODE_W) {
1295                 write_csr(dd, csr, value);
1296                 ret = value;
1297         } else {
1298                 dd_dev_err(dd, "Invalid cntr register access mode");
1299                 return 0;
1300         }
1301
1302         hfi1_cdbg(CNTR, "csr 0x%x val 0x%llx mode %d", csr, ret, mode);
1303         return ret;
1304 }
1305
1306 /* Dev Access */
1307 static u64 dev_access_u32_csr(const struct cntr_entry *entry,
1308                               void *context, int vl, int mode, u64 data)
1309 {
1310         struct hfi1_devdata *dd = context;
1311         u64 csr = entry->csr;
1312
1313         if (entry->flags & CNTR_SDMA) {
1314                 if (vl == CNTR_INVALID_VL)
1315                         return 0;
1316                 csr += 0x100 * vl;
1317         } else {
1318                 if (vl != CNTR_INVALID_VL)
1319                         return 0;
1320         }
1321         return read_write_csr(dd, csr, mode, data);
1322 }
1323
1324 static u64 access_sde_err_cnt(const struct cntr_entry *entry,
1325                               void *context, int idx, int mode, u64 data)
1326 {
1327         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1328
1329         if (dd->per_sdma && idx < dd->num_sdma)
1330                 return dd->per_sdma[idx].err_cnt;
1331         return 0;
1332 }
1333
1334 static u64 access_sde_int_cnt(const struct cntr_entry *entry,
1335                               void *context, int idx, int mode, u64 data)
1336 {
1337         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1338
1339         if (dd->per_sdma && idx < dd->num_sdma)
1340                 return dd->per_sdma[idx].sdma_int_cnt;
1341         return 0;
1342 }
1343
1344 static u64 access_sde_idle_int_cnt(const struct cntr_entry *entry,
1345                                    void *context, int idx, int mode, u64 data)
1346 {
1347         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1348
1349         if (dd->per_sdma && idx < dd->num_sdma)
1350                 return dd->per_sdma[idx].idle_int_cnt;
1351         return 0;
1352 }
1353
1354 static u64 access_sde_progress_int_cnt(const struct cntr_entry *entry,
1355                                        void *context, int idx, int mode,
1356                                        u64 data)
1357 {
1358         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1359
1360         if (dd->per_sdma && idx < dd->num_sdma)
1361                 return dd->per_sdma[idx].progress_int_cnt;
1362         return 0;
1363 }
1364
1365 static u64 dev_access_u64_csr(const struct cntr_entry *entry, void *context,
1366                               int vl, int mode, u64 data)
1367 {
1368         struct hfi1_devdata *dd = context;
1369
1370         u64 val = 0;
1371         u64 csr = entry->csr;
1372
1373         if (entry->flags & CNTR_VL) {
1374                 if (vl == CNTR_INVALID_VL)
1375                         return 0;
1376                 csr += 8 * vl;
1377         } else {
1378                 if (vl != CNTR_INVALID_VL)
1379                         return 0;
1380         }
1381
1382         val = read_write_csr(dd, csr, mode, data);
1383         return val;
1384 }
1385
1386 static u64 dc_access_lcb_cntr(const struct cntr_entry *entry, void *context,
1387                               int vl, int mode, u64 data)
1388 {
1389         struct hfi1_devdata *dd = context;
1390         u32 csr = entry->csr;
1391         int ret = 0;
1392
1393         if (vl != CNTR_INVALID_VL)
1394                 return 0;
1395         if (mode == CNTR_MODE_R)
1396                 ret = read_lcb_csr(dd, csr, &data);
1397         else if (mode == CNTR_MODE_W)
1398                 ret = write_lcb_csr(dd, csr, data);
1399
1400         if (ret) {
1401                 dd_dev_err(dd, "Could not acquire LCB for counter 0x%x", csr);
1402                 return 0;
1403         }
1404
1405         hfi1_cdbg(CNTR, "csr 0x%x val 0x%llx mode %d", csr, data, mode);
1406         return data;
1407 }
1408
1409 /* Port Access */
1410 static u64 port_access_u32_csr(const struct cntr_entry *entry, void *context,
1411                                int vl, int mode, u64 data)
1412 {
1413         struct hfi1_pportdata *ppd = context;
1414
1415         if (vl != CNTR_INVALID_VL)
1416                 return 0;
1417         return read_write_csr(ppd->dd, entry->csr, mode, data);
1418 }
1419
1420 static u64 port_access_u64_csr(const struct cntr_entry *entry,
1421                                void *context, int vl, int mode, u64 data)
1422 {
1423         struct hfi1_pportdata *ppd = context;
1424         u64 val;
1425         u64 csr = entry->csr;
1426
1427         if (entry->flags & CNTR_VL) {
1428                 if (vl == CNTR_INVALID_VL)
1429                         return 0;
1430                 csr += 8 * vl;
1431         } else {
1432                 if (vl != CNTR_INVALID_VL)
1433                         return 0;
1434         }
1435         val = read_write_csr(ppd->dd, csr, mode, data);
1436         return val;
1437 }
1438
1439 /* Software defined */
1440 static inline u64 read_write_sw(struct hfi1_devdata *dd, u64 *cntr, int mode,
1441                                 u64 data)
1442 {
1443         u64 ret;
1444
1445         if (mode == CNTR_MODE_R) {
1446                 ret = *cntr;
1447         } else if (mode == CNTR_MODE_W) {
1448                 *cntr = data;
1449                 ret = data;
1450         } else {
1451                 dd_dev_err(dd, "Invalid cntr sw access mode");
1452                 return 0;
1453         }
1454
1455         hfi1_cdbg(CNTR, "val 0x%llx mode %d", ret, mode);
1456
1457         return ret;
1458 }
1459
1460 static u64 access_sw_link_dn_cnt(const struct cntr_entry *entry, void *context,
1461                                  int vl, int mode, u64 data)
1462 {
1463         struct hfi1_pportdata *ppd = context;
1464
1465         if (vl != CNTR_INVALID_VL)
1466                 return 0;
1467         return read_write_sw(ppd->dd, &ppd->link_downed, mode, data);
1468 }
1469
1470 static u64 access_sw_link_up_cnt(const struct cntr_entry *entry, void *context,
1471                                  int vl, int mode, u64 data)
1472 {
1473         struct hfi1_pportdata *ppd = context;
1474
1475         if (vl != CNTR_INVALID_VL)
1476                 return 0;
1477         return read_write_sw(ppd->dd, &ppd->link_up, mode, data);
1478 }
1479
1480 static u64 access_sw_unknown_frame_cnt(const struct cntr_entry *entry,
1481                                        void *context, int vl, int mode,
1482                                        u64 data)
1483 {
1484         struct hfi1_pportdata *ppd = (struct hfi1_pportdata *)context;
1485
1486         if (vl != CNTR_INVALID_VL)
1487                 return 0;
1488         return read_write_sw(ppd->dd, &ppd->unknown_frame_count, mode, data);
1489 }
1490
1491 static u64 access_sw_xmit_discards(const struct cntr_entry *entry,
1492                                    void *context, int vl, int mode, u64 data)
1493 {
1494         struct hfi1_pportdata *ppd = (struct hfi1_pportdata *)context;
1495         u64 zero = 0;
1496         u64 *counter;
1497
1498         if (vl == CNTR_INVALID_VL)
1499                 counter = &ppd->port_xmit_discards;
1500         else if (vl >= 0 && vl < C_VL_COUNT)
1501                 counter = &ppd->port_xmit_discards_vl[vl];
1502         else
1503                 counter = &zero;
1504
1505         return read_write_sw(ppd->dd, counter, mode, data);
1506 }
1507
1508 static u64 access_xmit_constraint_errs(const struct cntr_entry *entry,
1509                                        void *context, int vl, int mode,
1510                                        u64 data)
1511 {
1512         struct hfi1_pportdata *ppd = context;
1513
1514         if (vl != CNTR_INVALID_VL)
1515                 return 0;
1516
1517         return read_write_sw(ppd->dd, &ppd->port_xmit_constraint_errors,
1518                              mode, data);
1519 }
1520
1521 static u64 access_rcv_constraint_errs(const struct cntr_entry *entry,
1522                                       void *context, int vl, int mode, u64 data)
1523 {
1524         struct hfi1_pportdata *ppd = context;
1525
1526         if (vl != CNTR_INVALID_VL)
1527                 return 0;
1528
1529         return read_write_sw(ppd->dd, &ppd->port_rcv_constraint_errors,
1530                              mode, data);
1531 }
1532
1533 u64 get_all_cpu_total(u64 __percpu *cntr)
1534 {
1535         int cpu;
1536         u64 counter = 0;
1537
1538         for_each_possible_cpu(cpu)
1539                 counter += *per_cpu_ptr(cntr, cpu);
1540         return counter;
1541 }
1542
1543 static u64 read_write_cpu(struct hfi1_devdata *dd, u64 *z_val,
1544                           u64 __percpu *cntr,
1545                           int vl, int mode, u64 data)
1546 {
1547         u64 ret = 0;
1548
1549         if (vl != CNTR_INVALID_VL)
1550                 return 0;
1551
1552         if (mode == CNTR_MODE_R) {
1553                 ret = get_all_cpu_total(cntr) - *z_val;
1554         } else if (mode == CNTR_MODE_W) {
1555                 /* A write can only zero the counter */
1556                 if (data == 0)
1557                         *z_val = get_all_cpu_total(cntr);
1558                 else
1559                         dd_dev_err(dd, "Per CPU cntrs can only be zeroed");
1560         } else {
1561                 dd_dev_err(dd, "Invalid cntr sw cpu access mode");
1562                 return 0;
1563         }
1564
1565         return ret;
1566 }
1567
1568 static u64 access_sw_cpu_intr(const struct cntr_entry *entry,
1569                               void *context, int vl, int mode, u64 data)
1570 {
1571         struct hfi1_devdata *dd = context;
1572
1573         return read_write_cpu(dd, &dd->z_int_counter, dd->int_counter, vl,
1574                               mode, data);
1575 }
1576
1577 static u64 access_sw_cpu_rcv_limit(const struct cntr_entry *entry,
1578                                    void *context, int vl, int mode, u64 data)
1579 {
1580         struct hfi1_devdata *dd = context;
1581
1582         return read_write_cpu(dd, &dd->z_rcv_limit, dd->rcv_limit, vl,
1583                               mode, data);
1584 }
1585
1586 static u64 access_sw_pio_wait(const struct cntr_entry *entry,
1587                               void *context, int vl, int mode, u64 data)
1588 {
1589         struct hfi1_devdata *dd = context;
1590
1591         return dd->verbs_dev.n_piowait;
1592 }
1593
1594 static u64 access_sw_pio_drain(const struct cntr_entry *entry,
1595                                void *context, int vl, int mode, u64 data)
1596 {
1597         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1598
1599         return dd->verbs_dev.n_piodrain;
1600 }
1601
1602 static u64 access_sw_vtx_wait(const struct cntr_entry *entry,
1603                               void *context, int vl, int mode, u64 data)
1604 {
1605         struct hfi1_devdata *dd = context;
1606
1607         return dd->verbs_dev.n_txwait;
1608 }
1609
1610 static u64 access_sw_kmem_wait(const struct cntr_entry *entry,
1611                                void *context, int vl, int mode, u64 data)
1612 {
1613         struct hfi1_devdata *dd = context;
1614
1615         return dd->verbs_dev.n_kmem_wait;
1616 }
1617
1618 static u64 access_sw_send_schedule(const struct cntr_entry *entry,
1619                                    void *context, int vl, int mode, u64 data)
1620 {
1621         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1622
1623         return read_write_cpu(dd, &dd->z_send_schedule, dd->send_schedule, vl,
1624                               mode, data);
1625 }
1626
1627 /* Software counters for the error status bits within MISC_ERR_STATUS */
1628 static u64 access_misc_pll_lock_fail_err_cnt(const struct cntr_entry *entry,
1629                                              void *context, int vl, int mode,
1630                                              u64 data)
1631 {
1632         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1633
1634         return dd->misc_err_status_cnt[12];
1635 }
1636
1637 static u64 access_misc_mbist_fail_err_cnt(const struct cntr_entry *entry,
1638                                           void *context, int vl, int mode,
1639                                           u64 data)
1640 {
1641         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1642
1643         return dd->misc_err_status_cnt[11];
1644 }
1645
1646 static u64 access_misc_invalid_eep_cmd_err_cnt(const struct cntr_entry *entry,
1647                                                void *context, int vl, int mode,
1648                                                u64 data)
1649 {
1650         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1651
1652         return dd->misc_err_status_cnt[10];
1653 }
1654
1655 static u64 access_misc_efuse_done_parity_err_cnt(const struct cntr_entry *entry,
1656                                                  void *context, int vl,
1657                                                  int mode, u64 data)
1658 {
1659         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1660
1661         return dd->misc_err_status_cnt[9];
1662 }
1663
1664 static u64 access_misc_efuse_write_err_cnt(const struct cntr_entry *entry,
1665                                            void *context, int vl, int mode,
1666                                            u64 data)
1667 {
1668         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1669
1670         return dd->misc_err_status_cnt[8];
1671 }
1672
1673 static u64 access_misc_efuse_read_bad_addr_err_cnt(
1674                                 const struct cntr_entry *entry,
1675                                 void *context, int vl, int mode, u64 data)
1676 {
1677         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1678
1679         return dd->misc_err_status_cnt[7];
1680 }
1681
1682 static u64 access_misc_efuse_csr_parity_err_cnt(const struct cntr_entry *entry,
1683                                                 void *context, int vl,
1684                                                 int mode, u64 data)
1685 {
1686         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1687
1688         return dd->misc_err_status_cnt[6];
1689 }
1690
1691 static u64 access_misc_fw_auth_failed_err_cnt(const struct cntr_entry *entry,
1692                                               void *context, int vl, int mode,
1693                                               u64 data)
1694 {
1695         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1696
1697         return dd->misc_err_status_cnt[5];
1698 }
1699
1700 static u64 access_misc_key_mismatch_err_cnt(const struct cntr_entry *entry,
1701                                             void *context, int vl, int mode,
1702                                             u64 data)
1703 {
1704         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1705
1706         return dd->misc_err_status_cnt[4];
1707 }
1708
1709 static u64 access_misc_sbus_write_failed_err_cnt(const struct cntr_entry *entry,
1710                                                  void *context, int vl,
1711                                                  int mode, u64 data)
1712 {
1713         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1714
1715         return dd->misc_err_status_cnt[3];
1716 }
1717
1718 static u64 access_misc_csr_write_bad_addr_err_cnt(
1719                                 const struct cntr_entry *entry,
1720                                 void *context, int vl, int mode, u64 data)
1721 {
1722         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1723
1724         return dd->misc_err_status_cnt[2];
1725 }
1726
1727 static u64 access_misc_csr_read_bad_addr_err_cnt(const struct cntr_entry *entry,
1728                                                  void *context, int vl,
1729                                                  int mode, u64 data)
1730 {
1731         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1732
1733         return dd->misc_err_status_cnt[1];
1734 }
1735
1736 static u64 access_misc_csr_parity_err_cnt(const struct cntr_entry *entry,
1737                                           void *context, int vl, int mode,
1738                                           u64 data)
1739 {
1740         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1741
1742         return dd->misc_err_status_cnt[0];
1743 }
1744
1745 /*
1746  * Software counter for the aggregate of
1747  * individual CceErrStatus counters
1748  */
1749 static u64 access_sw_cce_err_status_aggregated_cnt(
1750                                 const struct cntr_entry *entry,
1751                                 void *context, int vl, int mode, u64 data)
1752 {
1753         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1754
1755         return dd->sw_cce_err_status_aggregate;
1756 }
1757
1758 /*
1759  * Software counters corresponding to each of the
1760  * error status bits within CceErrStatus
1761  */
1762 static u64 access_cce_msix_csr_parity_err_cnt(const struct cntr_entry *entry,
1763                                               void *context, int vl, int mode,
1764                                               u64 data)
1765 {
1766         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1767
1768         return dd->cce_err_status_cnt[40];
1769 }
1770
1771 static u64 access_cce_int_map_unc_err_cnt(const struct cntr_entry *entry,
1772                                           void *context, int vl, int mode,
1773                                           u64 data)
1774 {
1775         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1776
1777         return dd->cce_err_status_cnt[39];
1778 }
1779
1780 static u64 access_cce_int_map_cor_err_cnt(const struct cntr_entry *entry,
1781                                           void *context, int vl, int mode,
1782                                           u64 data)
1783 {
1784         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1785
1786         return dd->cce_err_status_cnt[38];
1787 }
1788
1789 static u64 access_cce_msix_table_unc_err_cnt(const struct cntr_entry *entry,
1790                                              void *context, int vl, int mode,
1791                                              u64 data)
1792 {
1793         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1794
1795         return dd->cce_err_status_cnt[37];
1796 }
1797
1798 static u64 access_cce_msix_table_cor_err_cnt(const struct cntr_entry *entry,
1799                                              void *context, int vl, int mode,
1800                                              u64 data)
1801 {
1802         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1803
1804         return dd->cce_err_status_cnt[36];
1805 }
1806
1807 static u64 access_cce_rxdma_conv_fifo_parity_err_cnt(
1808                                 const struct cntr_entry *entry,
1809                                 void *context, int vl, int mode, u64 data)
1810 {
1811         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1812
1813         return dd->cce_err_status_cnt[35];
1814 }
1815
1816 static u64 access_cce_rcpl_async_fifo_parity_err_cnt(
1817                                 const struct cntr_entry *entry,
1818                                 void *context, int vl, int mode, u64 data)
1819 {
1820         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1821
1822         return dd->cce_err_status_cnt[34];
1823 }
1824
1825 static u64 access_cce_seg_write_bad_addr_err_cnt(const struct cntr_entry *entry,
1826                                                  void *context, int vl,
1827                                                  int mode, u64 data)
1828 {
1829         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1830
1831         return dd->cce_err_status_cnt[33];
1832 }
1833
1834 static u64 access_cce_seg_read_bad_addr_err_cnt(const struct cntr_entry *entry,
1835                                                 void *context, int vl, int mode,
1836                                                 u64 data)
1837 {
1838         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1839
1840         return dd->cce_err_status_cnt[32];
1841 }
1842
1843 static u64 access_la_triggered_cnt(const struct cntr_entry *entry,
1844                                    void *context, int vl, int mode, u64 data)
1845 {
1846         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1847
1848         return dd->cce_err_status_cnt[31];
1849 }
1850
1851 static u64 access_cce_trgt_cpl_timeout_err_cnt(const struct cntr_entry *entry,
1852                                                void *context, int vl, int mode,
1853                                                u64 data)
1854 {
1855         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1856
1857         return dd->cce_err_status_cnt[30];
1858 }
1859
1860 static u64 access_pcic_receive_parity_err_cnt(const struct cntr_entry *entry,
1861                                               void *context, int vl, int mode,
1862                                               u64 data)
1863 {
1864         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1865
1866         return dd->cce_err_status_cnt[29];
1867 }
1868
1869 static u64 access_pcic_transmit_back_parity_err_cnt(
1870                                 const struct cntr_entry *entry,
1871                                 void *context, int vl, int mode, u64 data)
1872 {
1873         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1874
1875         return dd->cce_err_status_cnt[28];
1876 }
1877
1878 static u64 access_pcic_transmit_front_parity_err_cnt(
1879                                 const struct cntr_entry *entry,
1880                                 void *context, int vl, int mode, u64 data)
1881 {
1882         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1883
1884         return dd->cce_err_status_cnt[27];
1885 }
1886
1887 static u64 access_pcic_cpl_dat_q_unc_err_cnt(const struct cntr_entry *entry,
1888                                              void *context, int vl, int mode,
1889                                              u64 data)
1890 {
1891         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1892
1893         return dd->cce_err_status_cnt[26];
1894 }
1895
1896 static u64 access_pcic_cpl_hd_q_unc_err_cnt(const struct cntr_entry *entry,
1897                                             void *context, int vl, int mode,
1898                                             u64 data)
1899 {
1900         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1901
1902         return dd->cce_err_status_cnt[25];
1903 }
1904
1905 static u64 access_pcic_post_dat_q_unc_err_cnt(const struct cntr_entry *entry,
1906                                               void *context, int vl, int mode,
1907                                               u64 data)
1908 {
1909         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1910
1911         return dd->cce_err_status_cnt[24];
1912 }
1913
1914 static u64 access_pcic_post_hd_q_unc_err_cnt(const struct cntr_entry *entry,
1915                                              void *context, int vl, int mode,
1916                                              u64 data)
1917 {
1918         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1919
1920         return dd->cce_err_status_cnt[23];
1921 }
1922
1923 static u64 access_pcic_retry_sot_mem_unc_err_cnt(const struct cntr_entry *entry,
1924                                                  void *context, int vl,
1925                                                  int mode, u64 data)
1926 {
1927         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1928
1929         return dd->cce_err_status_cnt[22];
1930 }
1931
1932 static u64 access_pcic_retry_mem_unc_err(const struct cntr_entry *entry,
1933                                          void *context, int vl, int mode,
1934                                          u64 data)
1935 {
1936         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1937
1938         return dd->cce_err_status_cnt[21];
1939 }
1940
1941 static u64 access_pcic_n_post_dat_q_parity_err_cnt(
1942                                 const struct cntr_entry *entry,
1943                                 void *context, int vl, int mode, u64 data)
1944 {
1945         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1946
1947         return dd->cce_err_status_cnt[20];
1948 }
1949
1950 static u64 access_pcic_n_post_h_q_parity_err_cnt(const struct cntr_entry *entry,
1951                                                  void *context, int vl,
1952                                                  int mode, u64 data)
1953 {
1954         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1955
1956         return dd->cce_err_status_cnt[19];
1957 }
1958
1959 static u64 access_pcic_cpl_dat_q_cor_err_cnt(const struct cntr_entry *entry,
1960                                              void *context, int vl, int mode,
1961                                              u64 data)
1962 {
1963         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1964
1965         return dd->cce_err_status_cnt[18];
1966 }
1967
1968 static u64 access_pcic_cpl_hd_q_cor_err_cnt(const struct cntr_entry *entry,
1969                                             void *context, int vl, int mode,
1970                                             u64 data)
1971 {
1972         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1973
1974         return dd->cce_err_status_cnt[17];
1975 }
1976
1977 static u64 access_pcic_post_dat_q_cor_err_cnt(const struct cntr_entry *entry,
1978                                               void *context, int vl, int mode,
1979                                               u64 data)
1980 {
1981         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1982
1983         return dd->cce_err_status_cnt[16];
1984 }
1985
1986 static u64 access_pcic_post_hd_q_cor_err_cnt(const struct cntr_entry *entry,
1987                                              void *context, int vl, int mode,
1988                                              u64 data)
1989 {
1990         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1991
1992         return dd->cce_err_status_cnt[15];
1993 }
1994
1995 static u64 access_pcic_retry_sot_mem_cor_err_cnt(const struct cntr_entry *entry,
1996                                                  void *context, int vl,
1997                                                  int mode, u64 data)
1998 {
1999         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2000
2001         return dd->cce_err_status_cnt[14];
2002 }
2003
2004 static u64 access_pcic_retry_mem_cor_err_cnt(const struct cntr_entry *entry,
2005                                              void *context, int vl, int mode,
2006                                              u64 data)
2007 {
2008         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2009
2010         return dd->cce_err_status_cnt[13];
2011 }
2012
2013 static u64 access_cce_cli1_async_fifo_dbg_parity_err_cnt(
2014                                 const struct cntr_entry *entry,
2015                                 void *context, int vl, int mode, u64 data)
2016 {
2017         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2018
2019         return dd->cce_err_status_cnt[12];
2020 }
2021
2022 static u64 access_cce_cli1_async_fifo_rxdma_parity_err_cnt(
2023                                 const struct cntr_entry *entry,
2024                                 void *context, int vl, int mode, u64 data)
2025 {
2026         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2027
2028         return dd->cce_err_status_cnt[11];
2029 }
2030
2031 static u64 access_cce_cli1_async_fifo_sdma_hd_parity_err_cnt(
2032                                 const struct cntr_entry *entry,
2033                                 void *context, int vl, int mode, u64 data)
2034 {
2035         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2036
2037         return dd->cce_err_status_cnt[10];
2038 }
2039
2040 static u64 access_cce_cl1_async_fifo_pio_crdt_parity_err_cnt(
2041                                 const struct cntr_entry *entry,
2042                                 void *context, int vl, int mode, u64 data)
2043 {
2044         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2045
2046         return dd->cce_err_status_cnt[9];
2047 }
2048
2049 static u64 access_cce_cli2_async_fifo_parity_err_cnt(
2050                                 const struct cntr_entry *entry,
2051                                 void *context, int vl, int mode, u64 data)
2052 {
2053         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2054
2055         return dd->cce_err_status_cnt[8];
2056 }
2057
2058 static u64 access_cce_csr_cfg_bus_parity_err_cnt(const struct cntr_entry *entry,
2059                                                  void *context, int vl,
2060                                                  int mode, u64 data)
2061 {
2062         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2063
2064         return dd->cce_err_status_cnt[7];
2065 }
2066
2067 static u64 access_cce_cli0_async_fifo_parity_err_cnt(
2068                                 const struct cntr_entry *entry,
2069                                 void *context, int vl, int mode, u64 data)
2070 {
2071         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2072
2073         return dd->cce_err_status_cnt[6];
2074 }
2075
2076 static u64 access_cce_rspd_data_parity_err_cnt(const struct cntr_entry *entry,
2077                                                void *context, int vl, int mode,
2078                                                u64 data)
2079 {
2080         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2081
2082         return dd->cce_err_status_cnt[5];
2083 }
2084
2085 static u64 access_cce_trgt_access_err_cnt(const struct cntr_entry *entry,
2086                                           void *context, int vl, int mode,
2087                                           u64 data)
2088 {
2089         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2090
2091         return dd->cce_err_status_cnt[4];
2092 }
2093
2094 static u64 access_cce_trgt_async_fifo_parity_err_cnt(
2095                                 const struct cntr_entry *entry,
2096                                 void *context, int vl, int mode, u64 data)
2097 {
2098         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2099
2100         return dd->cce_err_status_cnt[3];
2101 }
2102
2103 static u64 access_cce_csr_write_bad_addr_err_cnt(const struct cntr_entry *entry,
2104                                                  void *context, int vl,
2105                                                  int mode, u64 data)
2106 {
2107         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2108
2109         return dd->cce_err_status_cnt[2];
2110 }
2111
2112 static u64 access_cce_csr_read_bad_addr_err_cnt(const struct cntr_entry *entry,
2113                                                 void *context, int vl,
2114                                                 int mode, u64 data)
2115 {
2116         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2117
2118         return dd->cce_err_status_cnt[1];
2119 }
2120
2121 static u64 access_ccs_csr_parity_err_cnt(const struct cntr_entry *entry,
2122                                          void *context, int vl, int mode,
2123                                          u64 data)
2124 {
2125         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2126
2127         return dd->cce_err_status_cnt[0];
2128 }
2129
2130 /*
2131  * Software counters corresponding to each of the
2132  * error status bits within RcvErrStatus
2133  */
2134 static u64 access_rx_csr_parity_err_cnt(const struct cntr_entry *entry,
2135                                         void *context, int vl, int mode,
2136                                         u64 data)
2137 {
2138         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2139
2140         return dd->rcv_err_status_cnt[63];
2141 }
2142
2143 static u64 access_rx_csr_write_bad_addr_err_cnt(const struct cntr_entry *entry,
2144                                                 void *context, int vl,
2145                                                 int mode, u64 data)
2146 {
2147         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2148
2149         return dd->rcv_err_status_cnt[62];
2150 }
2151
2152 static u64 access_rx_csr_read_bad_addr_err_cnt(const struct cntr_entry *entry,
2153                                                void *context, int vl, int mode,
2154                                                u64 data)
2155 {
2156         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2157
2158         return dd->rcv_err_status_cnt[61];
2159 }
2160
2161 static u64 access_rx_dma_csr_unc_err_cnt(const struct cntr_entry *entry,
2162                                          void *context, int vl, int mode,
2163                                          u64 data)
2164 {
2165         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2166
2167         return dd->rcv_err_status_cnt[60];
2168 }
2169
2170 static u64 access_rx_dma_dq_fsm_encoding_err_cnt(const struct cntr_entry *entry,
2171                                                  void *context, int vl,
2172                                                  int mode, u64 data)
2173 {
2174         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2175
2176         return dd->rcv_err_status_cnt[59];
2177 }
2178
2179 static u64 access_rx_dma_eq_fsm_encoding_err_cnt(const struct cntr_entry *entry,
2180                                                  void *context, int vl,
2181                                                  int mode, u64 data)
2182 {
2183         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2184
2185         return dd->rcv_err_status_cnt[58];
2186 }
2187
2188 static u64 access_rx_dma_csr_parity_err_cnt(const struct cntr_entry *entry,
2189                                             void *context, int vl, int mode,
2190                                             u64 data)
2191 {
2192         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2193
2194         return dd->rcv_err_status_cnt[57];
2195 }
2196
2197 static u64 access_rx_rbuf_data_cor_err_cnt(const struct cntr_entry *entry,
2198                                            void *context, int vl, int mode,
2199                                            u64 data)
2200 {
2201         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2202
2203         return dd->rcv_err_status_cnt[56];
2204 }
2205
2206 static u64 access_rx_rbuf_data_unc_err_cnt(const struct cntr_entry *entry,
2207                                            void *context, int vl, int mode,
2208                                            u64 data)
2209 {
2210         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2211
2212         return dd->rcv_err_status_cnt[55];
2213 }
2214
2215 static u64 access_rx_dma_data_fifo_rd_cor_err_cnt(
2216                                 const struct cntr_entry *entry,
2217                                 void *context, int vl, int mode, u64 data)
2218 {
2219         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2220
2221         return dd->rcv_err_status_cnt[54];
2222 }
2223
2224 static u64 access_rx_dma_data_fifo_rd_unc_err_cnt(
2225                                 const struct cntr_entry *entry,
2226                                 void *context, int vl, int mode, u64 data)
2227 {
2228         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2229
2230         return dd->rcv_err_status_cnt[53];
2231 }
2232
2233 static u64 access_rx_dma_hdr_fifo_rd_cor_err_cnt(const struct cntr_entry *entry,
2234                                                  void *context, int vl,
2235                                                  int mode, u64 data)
2236 {
2237         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2238
2239         return dd->rcv_err_status_cnt[52];
2240 }
2241
2242 static u64 access_rx_dma_hdr_fifo_rd_unc_err_cnt(const struct cntr_entry *entry,
2243                                                  void *context, int vl,
2244                                                  int mode, u64 data)
2245 {
2246         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2247
2248         return dd->rcv_err_status_cnt[51];
2249 }
2250
2251 static u64 access_rx_rbuf_desc_part2_cor_err_cnt(const struct cntr_entry *entry,
2252                                                  void *context, int vl,
2253                                                  int mode, u64 data)
2254 {
2255         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2256
2257         return dd->rcv_err_status_cnt[50];
2258 }
2259
2260 static u64 access_rx_rbuf_desc_part2_unc_err_cnt(const struct cntr_entry *entry,
2261                                                  void *context, int vl,
2262                                                  int mode, u64 data)
2263 {
2264         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2265
2266         return dd->rcv_err_status_cnt[49];
2267 }
2268
2269 static u64 access_rx_rbuf_desc_part1_cor_err_cnt(const struct cntr_entry *entry,
2270                                                  void *context, int vl,
2271                                                  int mode, u64 data)
2272 {
2273         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2274
2275         return dd->rcv_err_status_cnt[48];
2276 }
2277
2278 static u64 access_rx_rbuf_desc_part1_unc_err_cnt(const struct cntr_entry *entry,
2279                                                  void *context, int vl,
2280                                                  int mode, u64 data)
2281 {
2282         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2283
2284         return dd->rcv_err_status_cnt[47];
2285 }
2286
2287 static u64 access_rx_hq_intr_fsm_err_cnt(const struct cntr_entry *entry,
2288                                          void *context, int vl, int mode,
2289                                          u64 data)
2290 {
2291         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2292
2293         return dd->rcv_err_status_cnt[46];
2294 }
2295
2296 static u64 access_rx_hq_intr_csr_parity_err_cnt(
2297                                 const struct cntr_entry *entry,
2298                                 void *context, int vl, int mode, u64 data)
2299 {
2300         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2301
2302         return dd->rcv_err_status_cnt[45];
2303 }
2304
2305 static u64 access_rx_lookup_csr_parity_err_cnt(
2306                                 const struct cntr_entry *entry,
2307                                 void *context, int vl, int mode, u64 data)
2308 {
2309         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2310
2311         return dd->rcv_err_status_cnt[44];
2312 }
2313
2314 static u64 access_rx_lookup_rcv_array_cor_err_cnt(
2315                                 const struct cntr_entry *entry,
2316                                 void *context, int vl, int mode, u64 data)
2317 {
2318         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2319
2320         return dd->rcv_err_status_cnt[43];
2321 }
2322
2323 static u64 access_rx_lookup_rcv_array_unc_err_cnt(
2324                                 const struct cntr_entry *entry,
2325                                 void *context, int vl, int mode, u64 data)
2326 {
2327         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2328
2329         return dd->rcv_err_status_cnt[42];
2330 }
2331
2332 static u64 access_rx_lookup_des_part2_parity_err_cnt(
2333                                 const struct cntr_entry *entry,
2334                                 void *context, int vl, int mode, u64 data)
2335 {
2336         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2337
2338         return dd->rcv_err_status_cnt[41];
2339 }
2340
2341 static u64 access_rx_lookup_des_part1_unc_cor_err_cnt(
2342                                 const struct cntr_entry *entry,
2343                                 void *context, int vl, int mode, u64 data)
2344 {
2345         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2346
2347         return dd->rcv_err_status_cnt[40];
2348 }
2349
2350 static u64 access_rx_lookup_des_part1_unc_err_cnt(
2351                                 const struct cntr_entry *entry,
2352                                 void *context, int vl, int mode, u64 data)
2353 {
2354         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2355
2356         return dd->rcv_err_status_cnt[39];
2357 }
2358
2359 static u64 access_rx_rbuf_next_free_buf_cor_err_cnt(
2360                                 const struct cntr_entry *entry,
2361                                 void *context, int vl, int mode, u64 data)
2362 {
2363         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2364
2365         return dd->rcv_err_status_cnt[38];
2366 }
2367
2368 static u64 access_rx_rbuf_next_free_buf_unc_err_cnt(
2369                                 const struct cntr_entry *entry,
2370                                 void *context, int vl, int mode, u64 data)
2371 {
2372         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2373
2374         return dd->rcv_err_status_cnt[37];
2375 }
2376
2377 static u64 access_rbuf_fl_init_wr_addr_parity_err_cnt(
2378                                 const struct cntr_entry *entry,
2379                                 void *context, int vl, int mode, u64 data)
2380 {
2381         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2382
2383         return dd->rcv_err_status_cnt[36];
2384 }
2385
2386 static u64 access_rx_rbuf_fl_initdone_parity_err_cnt(
2387                                 const struct cntr_entry *entry,
2388                                 void *context, int vl, int mode, u64 data)
2389 {
2390         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2391
2392         return dd->rcv_err_status_cnt[35];
2393 }
2394
2395 static u64 access_rx_rbuf_fl_write_addr_parity_err_cnt(
2396                                 const struct cntr_entry *entry,
2397                                 void *context, int vl, int mode, u64 data)
2398 {
2399         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2400
2401         return dd->rcv_err_status_cnt[34];
2402 }
2403
2404 static u64 access_rx_rbuf_fl_rd_addr_parity_err_cnt(
2405                                 const struct cntr_entry *entry,
2406                                 void *context, int vl, int mode, u64 data)
2407 {
2408         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2409
2410         return dd->rcv_err_status_cnt[33];
2411 }
2412
2413 static u64 access_rx_rbuf_empty_err_cnt(const struct cntr_entry *entry,
2414                                         void *context, int vl, int mode,
2415                                         u64 data)
2416 {
2417         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2418
2419         return dd->rcv_err_status_cnt[32];
2420 }
2421
2422 static u64 access_rx_rbuf_full_err_cnt(const struct cntr_entry *entry,
2423                                        void *context, int vl, int mode,
2424                                        u64 data)
2425 {
2426         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2427
2428         return dd->rcv_err_status_cnt[31];
2429 }
2430
2431 static u64 access_rbuf_bad_lookup_err_cnt(const struct cntr_entry *entry,
2432                                           void *context, int vl, int mode,
2433                                           u64 data)
2434 {
2435         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2436
2437         return dd->rcv_err_status_cnt[30];
2438 }
2439
2440 static u64 access_rbuf_ctx_id_parity_err_cnt(const struct cntr_entry *entry,
2441                                              void *context, int vl, int mode,
2442                                              u64 data)
2443 {
2444         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2445
2446         return dd->rcv_err_status_cnt[29];
2447 }
2448
2449 static u64 access_rbuf_csr_qeopdw_parity_err_cnt(const struct cntr_entry *entry,
2450                                                  void *context, int vl,
2451                                                  int mode, u64 data)
2452 {
2453         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2454
2455         return dd->rcv_err_status_cnt[28];
2456 }
2457
2458 static u64 access_rx_rbuf_csr_q_num_of_pkt_parity_err_cnt(
2459                                 const struct cntr_entry *entry,
2460                                 void *context, int vl, int mode, u64 data)
2461 {
2462         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2463
2464         return dd->rcv_err_status_cnt[27];
2465 }
2466
2467 static u64 access_rx_rbuf_csr_q_t1_ptr_parity_err_cnt(
2468                                 const struct cntr_entry *entry,
2469                                 void *context, int vl, int mode, u64 data)
2470 {
2471         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2472
2473         return dd->rcv_err_status_cnt[26];
2474 }
2475
2476 static u64 access_rx_rbuf_csr_q_hd_ptr_parity_err_cnt(
2477                                 const struct cntr_entry *entry,
2478                                 void *context, int vl, int mode, u64 data)
2479 {
2480         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2481
2482         return dd->rcv_err_status_cnt[25];
2483 }
2484
2485 static u64 access_rx_rbuf_csr_q_vld_bit_parity_err_cnt(
2486                                 const struct cntr_entry *entry,
2487                                 void *context, int vl, int mode, u64 data)
2488 {
2489         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2490
2491         return dd->rcv_err_status_cnt[24];
2492 }
2493
2494 static u64 access_rx_rbuf_csr_q_next_buf_parity_err_cnt(
2495                                 const struct cntr_entry *entry,
2496                                 void *context, int vl, int mode, u64 data)
2497 {
2498         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2499
2500         return dd->rcv_err_status_cnt[23];
2501 }
2502
2503 static u64 access_rx_rbuf_csr_q_ent_cnt_parity_err_cnt(
2504                                 const struct cntr_entry *entry,
2505                                 void *context, int vl, int mode, u64 data)
2506 {
2507         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2508
2509         return dd->rcv_err_status_cnt[22];
2510 }
2511
2512 static u64 access_rx_rbuf_csr_q_head_buf_num_parity_err_cnt(
2513                                 const struct cntr_entry *entry,
2514                                 void *context, int vl, int mode, u64 data)
2515 {
2516         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2517
2518         return dd->rcv_err_status_cnt[21];
2519 }
2520
2521 static u64 access_rx_rbuf_block_list_read_cor_err_cnt(
2522                                 const struct cntr_entry *entry,
2523                                 void *context, int vl, int mode, u64 data)
2524 {
2525         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2526
2527         return dd->rcv_err_status_cnt[20];
2528 }
2529
2530 static u64 access_rx_rbuf_block_list_read_unc_err_cnt(
2531                                 const struct cntr_entry *entry,
2532                                 void *context, int vl, int mode, u64 data)
2533 {
2534         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2535
2536         return dd->rcv_err_status_cnt[19];
2537 }
2538
2539 static u64 access_rx_rbuf_lookup_des_cor_err_cnt(const struct cntr_entry *entry,
2540                                                  void *context, int vl,
2541                                                  int mode, u64 data)
2542 {
2543         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2544
2545         return dd->rcv_err_status_cnt[18];
2546 }
2547
2548 static u64 access_rx_rbuf_lookup_des_unc_err_cnt(const struct cntr_entry *entry,
2549                                                  void *context, int vl,
2550                                                  int mode, u64 data)
2551 {
2552         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2553
2554         return dd->rcv_err_status_cnt[17];
2555 }
2556
2557 static u64 access_rx_rbuf_lookup_des_reg_unc_cor_err_cnt(
2558                                 const struct cntr_entry *entry,
2559                                 void *context, int vl, int mode, u64 data)
2560 {
2561         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2562
2563         return dd->rcv_err_status_cnt[16];
2564 }
2565
2566 static u64 access_rx_rbuf_lookup_des_reg_unc_err_cnt(
2567                                 const struct cntr_entry *entry,
2568                                 void *context, int vl, int mode, u64 data)
2569 {
2570         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2571
2572         return dd->rcv_err_status_cnt[15];
2573 }
2574
2575 static u64 access_rx_rbuf_free_list_cor_err_cnt(const struct cntr_entry *entry,
2576                                                 void *context, int vl,
2577                                                 int mode, u64 data)
2578 {
2579         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2580
2581         return dd->rcv_err_status_cnt[14];
2582 }
2583
2584 static u64 access_rx_rbuf_free_list_unc_err_cnt(const struct cntr_entry *entry,
2585                                                 void *context, int vl,
2586                                                 int mode, u64 data)
2587 {
2588         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2589
2590         return dd->rcv_err_status_cnt[13];
2591 }
2592
2593 static u64 access_rx_rcv_fsm_encoding_err_cnt(const struct cntr_entry *entry,
2594                                               void *context, int vl, int mode,
2595                                               u64 data)
2596 {
2597         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2598
2599         return dd->rcv_err_status_cnt[12];
2600 }
2601
2602 static u64 access_rx_dma_flag_cor_err_cnt(const struct cntr_entry *entry,
2603                                           void *context, int vl, int mode,
2604                                           u64 data)
2605 {
2606         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2607
2608         return dd->rcv_err_status_cnt[11];
2609 }
2610
2611 static u64 access_rx_dma_flag_unc_err_cnt(const struct cntr_entry *entry,
2612                                           void *context, int vl, int mode,
2613                                           u64 data)
2614 {
2615         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2616
2617         return dd->rcv_err_status_cnt[10];
2618 }
2619
2620 static u64 access_rx_dc_sop_eop_parity_err_cnt(const struct cntr_entry *entry,
2621                                                void *context, int vl, int mode,
2622                                                u64 data)
2623 {
2624         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2625
2626         return dd->rcv_err_status_cnt[9];
2627 }
2628
2629 static u64 access_rx_rcv_csr_parity_err_cnt(const struct cntr_entry *entry,
2630                                             void *context, int vl, int mode,
2631                                             u64 data)
2632 {
2633         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2634
2635         return dd->rcv_err_status_cnt[8];
2636 }
2637
2638 static u64 access_rx_rcv_qp_map_table_cor_err_cnt(
2639                                 const struct cntr_entry *entry,
2640                                 void *context, int vl, int mode, u64 data)
2641 {
2642         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2643
2644         return dd->rcv_err_status_cnt[7];
2645 }
2646
2647 static u64 access_rx_rcv_qp_map_table_unc_err_cnt(
2648                                 const struct cntr_entry *entry,
2649                                 void *context, int vl, int mode, u64 data)
2650 {
2651         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2652
2653         return dd->rcv_err_status_cnt[6];
2654 }
2655
2656 static u64 access_rx_rcv_data_cor_err_cnt(const struct cntr_entry *entry,
2657                                           void *context, int vl, int mode,
2658                                           u64 data)
2659 {
2660         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2661
2662         return dd->rcv_err_status_cnt[5];
2663 }
2664
2665 static u64 access_rx_rcv_data_unc_err_cnt(const struct cntr_entry *entry,
2666                                           void *context, int vl, int mode,
2667                                           u64 data)
2668 {
2669         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2670
2671         return dd->rcv_err_status_cnt[4];
2672 }
2673
2674 static u64 access_rx_rcv_hdr_cor_err_cnt(const struct cntr_entry *entry,
2675                                          void *context, int vl, int mode,
2676                                          u64 data)
2677 {
2678         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2679
2680         return dd->rcv_err_status_cnt[3];
2681 }
2682
2683 static u64 access_rx_rcv_hdr_unc_err_cnt(const struct cntr_entry *entry,
2684                                          void *context, int vl, int mode,
2685                                          u64 data)
2686 {
2687         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2688
2689         return dd->rcv_err_status_cnt[2];
2690 }
2691
2692 static u64 access_rx_dc_intf_parity_err_cnt(const struct cntr_entry *entry,
2693                                             void *context, int vl, int mode,
2694                                             u64 data)
2695 {
2696         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2697
2698         return dd->rcv_err_status_cnt[1];
2699 }
2700
2701 static u64 access_rx_dma_csr_cor_err_cnt(const struct cntr_entry *entry,
2702                                          void *context, int vl, int mode,
2703                                          u64 data)
2704 {
2705         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2706
2707         return dd->rcv_err_status_cnt[0];
2708 }
2709
2710 /*
2711  * Software counters corresponding to each of the
2712  * error status bits within SendPioErrStatus
2713  */
2714 static u64 access_pio_pec_sop_head_parity_err_cnt(
2715                                 const struct cntr_entry *entry,
2716                                 void *context, int vl, int mode, u64 data)
2717 {
2718         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2719
2720         return dd->send_pio_err_status_cnt[35];
2721 }
2722
2723 static u64 access_pio_pcc_sop_head_parity_err_cnt(
2724                                 const struct cntr_entry *entry,
2725                                 void *context, int vl, int mode, u64 data)
2726 {
2727         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2728
2729         return dd->send_pio_err_status_cnt[34];
2730 }
2731
2732 static u64 access_pio_last_returned_cnt_parity_err_cnt(
2733                                 const struct cntr_entry *entry,
2734                                 void *context, int vl, int mode, u64 data)
2735 {
2736         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2737
2738         return dd->send_pio_err_status_cnt[33];
2739 }
2740
2741 static u64 access_pio_current_free_cnt_parity_err_cnt(
2742                                 const struct cntr_entry *entry,
2743                                 void *context, int vl, int mode, u64 data)
2744 {
2745         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2746
2747         return dd->send_pio_err_status_cnt[32];
2748 }
2749
2750 static u64 access_pio_reserved_31_err_cnt(const struct cntr_entry *entry,
2751                                           void *context, int vl, int mode,
2752                                           u64 data)
2753 {
2754         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2755
2756         return dd->send_pio_err_status_cnt[31];
2757 }
2758
2759 static u64 access_pio_reserved_30_err_cnt(const struct cntr_entry *entry,
2760                                           void *context, int vl, int mode,
2761                                           u64 data)
2762 {
2763         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2764
2765         return dd->send_pio_err_status_cnt[30];
2766 }
2767
2768 static u64 access_pio_ppmc_sop_len_err_cnt(const struct cntr_entry *entry,
2769                                            void *context, int vl, int mode,
2770                                            u64 data)
2771 {
2772         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2773
2774         return dd->send_pio_err_status_cnt[29];
2775 }
2776
2777 static u64 access_pio_ppmc_bqc_mem_parity_err_cnt(
2778                                 const struct cntr_entry *entry,
2779                                 void *context, int vl, int mode, u64 data)
2780 {
2781         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2782
2783         return dd->send_pio_err_status_cnt[28];
2784 }
2785
2786 static u64 access_pio_vl_fifo_parity_err_cnt(const struct cntr_entry *entry,
2787                                              void *context, int vl, int mode,
2788                                              u64 data)
2789 {
2790         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2791
2792         return dd->send_pio_err_status_cnt[27];
2793 }
2794
2795 static u64 access_pio_vlf_sop_parity_err_cnt(const struct cntr_entry *entry,
2796                                              void *context, int vl, int mode,
2797                                              u64 data)
2798 {
2799         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2800
2801         return dd->send_pio_err_status_cnt[26];
2802 }
2803
2804 static u64 access_pio_vlf_v1_len_parity_err_cnt(const struct cntr_entry *entry,
2805                                                 void *context, int vl,
2806                                                 int mode, u64 data)
2807 {
2808         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2809
2810         return dd->send_pio_err_status_cnt[25];
2811 }
2812
2813 static u64 access_pio_block_qw_count_parity_err_cnt(
2814                                 const struct cntr_entry *entry,
2815                                 void *context, int vl, int mode, u64 data)
2816 {
2817         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2818
2819         return dd->send_pio_err_status_cnt[24];
2820 }
2821
2822 static u64 access_pio_write_qw_valid_parity_err_cnt(
2823                                 const struct cntr_entry *entry,
2824                                 void *context, int vl, int mode, u64 data)
2825 {
2826         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2827
2828         return dd->send_pio_err_status_cnt[23];
2829 }
2830
2831 static u64 access_pio_state_machine_err_cnt(const struct cntr_entry *entry,
2832                                             void *context, int vl, int mode,
2833                                             u64 data)
2834 {
2835         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2836
2837         return dd->send_pio_err_status_cnt[22];
2838 }
2839
2840 static u64 access_pio_write_data_parity_err_cnt(const struct cntr_entry *entry,
2841                                                 void *context, int vl,
2842                                                 int mode, u64 data)
2843 {
2844         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2845
2846         return dd->send_pio_err_status_cnt[21];
2847 }
2848
2849 static u64 access_pio_host_addr_mem_cor_err_cnt(const struct cntr_entry *entry,
2850                                                 void *context, int vl,
2851                                                 int mode, u64 data)
2852 {
2853         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2854
2855         return dd->send_pio_err_status_cnt[20];
2856 }
2857
2858 static u64 access_pio_host_addr_mem_unc_err_cnt(const struct cntr_entry *entry,
2859                                                 void *context, int vl,
2860                                                 int mode, u64 data)
2861 {
2862         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2863
2864         return dd->send_pio_err_status_cnt[19];
2865 }
2866
2867 static u64 access_pio_pkt_evict_sm_or_arb_sm_err_cnt(
2868                                 const struct cntr_entry *entry,
2869                                 void *context, int vl, int mode, u64 data)
2870 {
2871         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2872
2873         return dd->send_pio_err_status_cnt[18];
2874 }
2875
2876 static u64 access_pio_init_sm_in_err_cnt(const struct cntr_entry *entry,
2877                                          void *context, int vl, int mode,
2878                                          u64 data)
2879 {
2880         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2881
2882         return dd->send_pio_err_status_cnt[17];
2883 }
2884
2885 static u64 access_pio_ppmc_pbl_fifo_err_cnt(const struct cntr_entry *entry,
2886                                             void *context, int vl, int mode,
2887                                             u64 data)
2888 {
2889         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2890
2891         return dd->send_pio_err_status_cnt[16];
2892 }
2893
2894 static u64 access_pio_credit_ret_fifo_parity_err_cnt(
2895                                 const struct cntr_entry *entry,
2896                                 void *context, int vl, int mode, u64 data)
2897 {
2898         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2899
2900         return dd->send_pio_err_status_cnt[15];
2901 }
2902
2903 static u64 access_pio_v1_len_mem_bank1_cor_err_cnt(
2904                                 const struct cntr_entry *entry,
2905                                 void *context, int vl, int mode, u64 data)
2906 {
2907         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2908
2909         return dd->send_pio_err_status_cnt[14];
2910 }
2911
2912 static u64 access_pio_v1_len_mem_bank0_cor_err_cnt(
2913                                 const struct cntr_entry *entry,
2914                                 void *context, int vl, int mode, u64 data)
2915 {
2916         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2917
2918         return dd->send_pio_err_status_cnt[13];
2919 }
2920
2921 static u64 access_pio_v1_len_mem_bank1_unc_err_cnt(
2922                                 const struct cntr_entry *entry,
2923                                 void *context, int vl, int mode, u64 data)
2924 {
2925         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2926
2927         return dd->send_pio_err_status_cnt[12];
2928 }
2929
2930 static u64 access_pio_v1_len_mem_bank0_unc_err_cnt(
2931                                 const struct cntr_entry *entry,
2932                                 void *context, int vl, int mode, u64 data)
2933 {
2934         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2935
2936         return dd->send_pio_err_status_cnt[11];
2937 }
2938
2939 static u64 access_pio_sm_pkt_reset_parity_err_cnt(
2940                                 const struct cntr_entry *entry,
2941                                 void *context, int vl, int mode, u64 data)
2942 {
2943         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2944
2945         return dd->send_pio_err_status_cnt[10];
2946 }
2947
2948 static u64 access_pio_pkt_evict_fifo_parity_err_cnt(
2949                                 const struct cntr_entry *entry,
2950                                 void *context, int vl, int mode, u64 data)
2951 {
2952         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2953
2954         return dd->send_pio_err_status_cnt[9];
2955 }
2956
2957 static u64 access_pio_sbrdctrl_crrel_fifo_parity_err_cnt(
2958                                 const struct cntr_entry *entry,
2959                                 void *context, int vl, int mode, u64 data)
2960 {
2961         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2962
2963         return dd->send_pio_err_status_cnt[8];
2964 }
2965
2966 static u64 access_pio_sbrdctl_crrel_parity_err_cnt(
2967                                 const struct cntr_entry *entry,
2968                                 void *context, int vl, int mode, u64 data)
2969 {
2970         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2971
2972         return dd->send_pio_err_status_cnt[7];
2973 }
2974
2975 static u64 access_pio_pec_fifo_parity_err_cnt(const struct cntr_entry *entry,
2976                                               void *context, int vl, int mode,
2977                                               u64 data)
2978 {
2979         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2980
2981         return dd->send_pio_err_status_cnt[6];
2982 }
2983
2984 static u64 access_pio_pcc_fifo_parity_err_cnt(const struct cntr_entry *entry,
2985                                               void *context, int vl, int mode,
2986                                               u64 data)
2987 {
2988         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2989
2990         return dd->send_pio_err_status_cnt[5];
2991 }
2992
2993 static u64 access_pio_sb_mem_fifo1_err_cnt(const struct cntr_entry *entry,
2994                                            void *context, int vl, int mode,
2995                                            u64 data)
2996 {
2997         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2998
2999         return dd->send_pio_err_status_cnt[4];
3000 }
3001
3002 static u64 access_pio_sb_mem_fifo0_err_cnt(const struct cntr_entry *entry,
3003                                            void *context, int vl, int mode,
3004                                            u64 data)
3005 {
3006         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3007
3008         return dd->send_pio_err_status_cnt[3];
3009 }
3010
3011 static u64 access_pio_csr_parity_err_cnt(const struct cntr_entry *entry,
3012                                          void *context, int vl, int mode,
3013                                          u64 data)
3014 {
3015         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3016
3017         return dd->send_pio_err_status_cnt[2];
3018 }
3019
3020 static u64 access_pio_write_addr_parity_err_cnt(const struct cntr_entry *entry,
3021                                                 void *context, int vl,
3022                                                 int mode, u64 data)
3023 {
3024         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3025
3026         return dd->send_pio_err_status_cnt[1];
3027 }
3028
3029 static u64 access_pio_write_bad_ctxt_err_cnt(const struct cntr_entry *entry,
3030                                              void *context, int vl, int mode,
3031                                              u64 data)
3032 {
3033         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3034
3035         return dd->send_pio_err_status_cnt[0];
3036 }
3037
3038 /*
3039  * Software counters corresponding to each of the
3040  * error status bits within SendDmaErrStatus
3041  */
3042 static u64 access_sdma_pcie_req_tracking_cor_err_cnt(
3043                                 const struct cntr_entry *entry,
3044                                 void *context, int vl, int mode, u64 data)
3045 {
3046         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3047
3048         return dd->send_dma_err_status_cnt[3];
3049 }
3050
3051 static u64 access_sdma_pcie_req_tracking_unc_err_cnt(
3052                                 const struct cntr_entry *entry,
3053                                 void *context, int vl, int mode, u64 data)
3054 {
3055         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3056
3057         return dd->send_dma_err_status_cnt[2];
3058 }
3059
3060 static u64 access_sdma_csr_parity_err_cnt(const struct cntr_entry *entry,
3061                                           void *context, int vl, int mode,
3062                                           u64 data)
3063 {
3064         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3065
3066         return dd->send_dma_err_status_cnt[1];
3067 }
3068
3069 static u64 access_sdma_rpy_tag_err_cnt(const struct cntr_entry *entry,
3070                                        void *context, int vl, int mode,
3071                                        u64 data)
3072 {
3073         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3074
3075         return dd->send_dma_err_status_cnt[0];
3076 }
3077
3078 /*
3079  * Software counters corresponding to each of the
3080  * error status bits within SendEgressErrStatus
3081  */
3082 static u64 access_tx_read_pio_memory_csr_unc_err_cnt(
3083                                 const struct cntr_entry *entry,
3084                                 void *context, int vl, int mode, u64 data)
3085 {
3086         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3087
3088         return dd->send_egress_err_status_cnt[63];
3089 }
3090
3091 static u64 access_tx_read_sdma_memory_csr_err_cnt(
3092                                 const struct cntr_entry *entry,
3093                                 void *context, int vl, int mode, u64 data)
3094 {
3095         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3096
3097         return dd->send_egress_err_status_cnt[62];
3098 }
3099
3100 static u64 access_tx_egress_fifo_cor_err_cnt(const struct cntr_entry *entry,
3101                                              void *context, int vl, int mode,
3102                                              u64 data)
3103 {
3104         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3105
3106         return dd->send_egress_err_status_cnt[61];
3107 }
3108
3109 static u64 access_tx_read_pio_memory_cor_err_cnt(const struct cntr_entry *entry,
3110                                                  void *context, int vl,
3111                                                  int mode, u64 data)
3112 {
3113         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3114
3115         return dd->send_egress_err_status_cnt[60];
3116 }
3117
3118 static u64 access_tx_read_sdma_memory_cor_err_cnt(
3119                                 const struct cntr_entry *entry,
3120                                 void *context, int vl, int mode, u64 data)
3121 {
3122         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3123
3124         return dd->send_egress_err_status_cnt[59];
3125 }
3126
3127 static u64 access_tx_sb_hdr_cor_err_cnt(const struct cntr_entry *entry,
3128                                         void *context, int vl, int mode,
3129                                         u64 data)
3130 {
3131         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3132
3133         return dd->send_egress_err_status_cnt[58];
3134 }
3135
3136 static u64 access_tx_credit_overrun_err_cnt(const struct cntr_entry *entry,
3137                                             void *context, int vl, int mode,
3138                                             u64 data)
3139 {
3140         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3141
3142         return dd->send_egress_err_status_cnt[57];
3143 }
3144
3145 static u64 access_tx_launch_fifo8_cor_err_cnt(const struct cntr_entry *entry,
3146                                               void *context, int vl, int mode,
3147                                               u64 data)
3148 {
3149         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3150
3151         return dd->send_egress_err_status_cnt[56];
3152 }
3153
3154 static u64 access_tx_launch_fifo7_cor_err_cnt(const struct cntr_entry *entry,
3155                                               void *context, int vl, int mode,
3156                                               u64 data)
3157 {
3158         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3159
3160         return dd->send_egress_err_status_cnt[55];
3161 }
3162
3163 static u64 access_tx_launch_fifo6_cor_err_cnt(const struct cntr_entry *entry,
3164                                               void *context, int vl, int mode,
3165                                               u64 data)
3166 {
3167         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3168
3169         return dd->send_egress_err_status_cnt[54];
3170 }
3171
3172 static u64 access_tx_launch_fifo5_cor_err_cnt(const struct cntr_entry *entry,
3173                                               void *context, int vl, int mode,
3174                                               u64 data)
3175 {
3176         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3177
3178         return dd->send_egress_err_status_cnt[53];
3179 }
3180
3181 static u64 access_tx_launch_fifo4_cor_err_cnt(const struct cntr_entry *entry,
3182                                               void *context, int vl, int mode,
3183                                               u64 data)
3184 {
3185         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3186
3187         return dd->send_egress_err_status_cnt[52];
3188 }
3189
3190 static u64 access_tx_launch_fifo3_cor_err_cnt(const struct cntr_entry *entry,
3191                                               void *context, int vl, int mode,
3192                                               u64 data)
3193 {
3194         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3195
3196         return dd->send_egress_err_status_cnt[51];
3197 }
3198
3199 static u64 access_tx_launch_fifo2_cor_err_cnt(const struct cntr_entry *entry,
3200                                               void *context, int vl, int mode,
3201                                               u64 data)
3202 {
3203         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3204
3205         return dd->send_egress_err_status_cnt[50];
3206 }
3207
3208 static u64 access_tx_launch_fifo1_cor_err_cnt(const struct cntr_entry *entry,
3209                                               void *context, int vl, int mode,
3210                                               u64 data)
3211 {
3212         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3213
3214         return dd->send_egress_err_status_cnt[49];
3215 }
3216
3217 static u64 access_tx_launch_fifo0_cor_err_cnt(const struct cntr_entry *entry,
3218                                               void *context, int vl, int mode,
3219                                               u64 data)
3220 {
3221         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3222
3223         return dd->send_egress_err_status_cnt[48];
3224 }
3225
3226 static u64 access_tx_credit_return_vl_err_cnt(const struct cntr_entry *entry,
3227                                               void *context, int vl, int mode,
3228                                               u64 data)
3229 {
3230         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3231
3232         return dd->send_egress_err_status_cnt[47];
3233 }
3234
3235 static u64 access_tx_hcrc_insertion_err_cnt(const struct cntr_entry *entry,
3236                                             void *context, int vl, int mode,
3237                                             u64 data)
3238 {
3239         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3240
3241         return dd->send_egress_err_status_cnt[46];
3242 }
3243
3244 static u64 access_tx_egress_fifo_unc_err_cnt(const struct cntr_entry *entry,
3245                                              void *context, int vl, int mode,
3246                                              u64 data)
3247 {
3248         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3249
3250         return dd->send_egress_err_status_cnt[45];
3251 }
3252
3253 static u64 access_tx_read_pio_memory_unc_err_cnt(const struct cntr_entry *entry,
3254                                                  void *context, int vl,
3255                                                  int mode, u64 data)
3256 {
3257         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3258
3259         return dd->send_egress_err_status_cnt[44];
3260 }
3261
3262 static u64 access_tx_read_sdma_memory_unc_err_cnt(
3263                                 const struct cntr_entry *entry,
3264                                 void *context, int vl, int mode, u64 data)
3265 {
3266         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3267
3268         return dd->send_egress_err_status_cnt[43];
3269 }
3270
3271 static u64 access_tx_sb_hdr_unc_err_cnt(const struct cntr_entry *entry,
3272                                         void *context, int vl, int mode,
3273                                         u64 data)
3274 {
3275         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3276
3277         return dd->send_egress_err_status_cnt[42];
3278 }
3279
3280 static u64 access_tx_credit_return_partiy_err_cnt(
3281                                 const struct cntr_entry *entry,
3282                                 void *context, int vl, int mode, u64 data)
3283 {
3284         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3285
3286         return dd->send_egress_err_status_cnt[41];
3287 }
3288
3289 static u64 access_tx_launch_fifo8_unc_or_parity_err_cnt(
3290                                 const struct cntr_entry *entry,
3291                                 void *context, int vl, int mode, u64 data)
3292 {
3293         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3294
3295         return dd->send_egress_err_status_cnt[40];
3296 }
3297
3298 static u64 access_tx_launch_fifo7_unc_or_parity_err_cnt(
3299                                 const struct cntr_entry *entry,
3300                                 void *context, int vl, int mode, u64 data)
3301 {
3302         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3303
3304         return dd->send_egress_err_status_cnt[39];
3305 }
3306
3307 static u64 access_tx_launch_fifo6_unc_or_parity_err_cnt(
3308                                 const struct cntr_entry *entry,
3309                                 void *context, int vl, int mode, u64 data)
3310 {
3311         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3312
3313         return dd->send_egress_err_status_cnt[38];
3314 }
3315
3316 static u64 access_tx_launch_fifo5_unc_or_parity_err_cnt(
3317                                 const struct cntr_entry *entry,
3318                                 void *context, int vl, int mode, u64 data)
3319 {
3320         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3321
3322         return dd->send_egress_err_status_cnt[37];
3323 }
3324
3325 static u64 access_tx_launch_fifo4_unc_or_parity_err_cnt(
3326                                 const struct cntr_entry *entry,
3327                                 void *context, int vl, int mode, u64 data)
3328 {
3329         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3330
3331         return dd->send_egress_err_status_cnt[36];
3332 }
3333
3334 static u64 access_tx_launch_fifo3_unc_or_parity_err_cnt(
3335                                 const struct cntr_entry *entry,
3336                                 void *context, int vl, int mode, u64 data)
3337 {
3338         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3339
3340         return dd->send_egress_err_status_cnt[35];
3341 }
3342
3343 static u64 access_tx_launch_fifo2_unc_or_parity_err_cnt(
3344                                 const struct cntr_entry *entry,
3345                                 void *context, int vl, int mode, u64 data)
3346 {
3347         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3348
3349         return dd->send_egress_err_status_cnt[34];
3350 }
3351
3352 static u64 access_tx_launch_fifo1_unc_or_parity_err_cnt(
3353                                 const struct cntr_entry *entry,
3354                                 void *context, int vl, int mode, u64 data)
3355 {
3356         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3357
3358         return dd->send_egress_err_status_cnt[33];
3359 }
3360
3361 static u64 access_tx_launch_fifo0_unc_or_parity_err_cnt(
3362                                 const struct cntr_entry *entry,
3363                                 void *context, int vl, int mode, u64 data)
3364 {
3365         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3366
3367         return dd->send_egress_err_status_cnt[32];
3368 }
3369
3370 static u64 access_tx_sdma15_disallowed_packet_err_cnt(
3371                                 const struct cntr_entry *entry,
3372                                 void *context, int vl, int mode, u64 data)
3373 {
3374         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3375
3376         return dd->send_egress_err_status_cnt[31];
3377 }
3378
3379 static u64 access_tx_sdma14_disallowed_packet_err_cnt(
3380                                 const struct cntr_entry *entry,
3381                                 void *context, int vl, int mode, u64 data)
3382 {
3383         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3384
3385         return dd->send_egress_err_status_cnt[30];
3386 }
3387
3388 static u64 access_tx_sdma13_disallowed_packet_err_cnt(
3389                                 const struct cntr_entry *entry,
3390                                 void *context, int vl, int mode, u64 data)
3391 {
3392         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3393
3394         return dd->send_egress_err_status_cnt[29];
3395 }
3396
3397 static u64 access_tx_sdma12_disallowed_packet_err_cnt(
3398                                 const struct cntr_entry *entry,
3399                                 void *context, int vl, int mode, u64 data)
3400 {
3401         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3402
3403         return dd->send_egress_err_status_cnt[28];
3404 }
3405
3406 static u64 access_tx_sdma11_disallowed_packet_err_cnt(
3407                                 const struct cntr_entry *entry,
3408                                 void *context, int vl, int mode, u64 data)
3409 {
3410         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3411
3412         return dd->send_egress_err_status_cnt[27];
3413 }
3414
3415 static u64 access_tx_sdma10_disallowed_packet_err_cnt(
3416                                 const struct cntr_entry *entry,
3417                                 void *context, int vl, int mode, u64 data)
3418 {
3419         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3420
3421         return dd->send_egress_err_status_cnt[26];
3422 }
3423
3424 static u64 access_tx_sdma9_disallowed_packet_err_cnt(
3425                                 const struct cntr_entry *entry,
3426                                 void *context, int vl, int mode, u64 data)
3427 {
3428         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3429
3430         return dd->send_egress_err_status_cnt[25];
3431 }
3432
3433 static u64 access_tx_sdma8_disallowed_packet_err_cnt(
3434                                 const struct cntr_entry *entry,
3435                                 void *context, int vl, int mode, u64 data)
3436 {
3437         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3438
3439         return dd->send_egress_err_status_cnt[24];
3440 }
3441
3442 static u64 access_tx_sdma7_disallowed_packet_err_cnt(
3443                                 const struct cntr_entry *entry,
3444                                 void *context, int vl, int mode, u64 data)
3445 {
3446         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3447
3448         return dd->send_egress_err_status_cnt[23];
3449 }
3450
3451 static u64 access_tx_sdma6_disallowed_packet_err_cnt(
3452                                 const struct cntr_entry *entry,
3453                                 void *context, int vl, int mode, u64 data)
3454 {
3455         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3456
3457         return dd->send_egress_err_status_cnt[22];
3458 }
3459
3460 static u64 access_tx_sdma5_disallowed_packet_err_cnt(
3461                                 const struct cntr_entry *entry,
3462                                 void *context, int vl, int mode, u64 data)
3463 {
3464         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3465
3466         return dd->send_egress_err_status_cnt[21];
3467 }
3468
3469 static u64 access_tx_sdma4_disallowed_packet_err_cnt(
3470                                 const struct cntr_entry *entry,
3471                                 void *context, int vl, int mode, u64 data)
3472 {
3473         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3474
3475         return dd->send_egress_err_status_cnt[20];
3476 }
3477
3478 static u64 access_tx_sdma3_disallowed_packet_err_cnt(
3479                                 const struct cntr_entry *entry,
3480                                 void *context, int vl, int mode, u64 data)
3481 {
3482         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3483
3484         return dd->send_egress_err_status_cnt[19];
3485 }
3486
3487 static u64 access_tx_sdma2_disallowed_packet_err_cnt(
3488                                 const struct cntr_entry *entry,
3489                                 void *context, int vl, int mode, u64 data)
3490 {
3491         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3492
3493         return dd->send_egress_err_status_cnt[18];
3494 }
3495
3496 static u64 access_tx_sdma1_disallowed_packet_err_cnt(
3497                                 const struct cntr_entry *entry,
3498                                 void *context, int vl, int mode, u64 data)
3499 {
3500         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3501
3502         return dd->send_egress_err_status_cnt[17];
3503 }
3504
3505 static u64 access_tx_sdma0_disallowed_packet_err_cnt(
3506                                 const struct cntr_entry *entry,
3507                                 void *context, int vl, int mode, u64 data)
3508 {
3509         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3510
3511         return dd->send_egress_err_status_cnt[16];
3512 }
3513
3514 static u64 access_tx_config_parity_err_cnt(const struct cntr_entry *entry,
3515                                            void *context, int vl, int mode,
3516                                            u64 data)
3517 {
3518         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3519
3520         return dd->send_egress_err_status_cnt[15];
3521 }
3522
3523 static u64 access_tx_sbrd_ctl_csr_parity_err_cnt(const struct cntr_entry *entry,
3524                                                  void *context, int vl,
3525                                                  int mode, u64 data)
3526 {
3527         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3528
3529         return dd->send_egress_err_status_cnt[14];
3530 }
3531
3532 static u64 access_tx_launch_csr_parity_err_cnt(const struct cntr_entry *entry,
3533                                                void *context, int vl, int mode,
3534                                                u64 data)
3535 {
3536         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3537
3538         return dd->send_egress_err_status_cnt[13];
3539 }
3540
3541 static u64 access_tx_illegal_vl_err_cnt(const struct cntr_entry *entry,
3542                                         void *context, int vl, int mode,
3543                                         u64 data)
3544 {
3545         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3546
3547         return dd->send_egress_err_status_cnt[12];
3548 }
3549
3550 static u64 access_tx_sbrd_ctl_state_machine_parity_err_cnt(
3551                                 const struct cntr_entry *entry,
3552                                 void *context, int vl, int mode, u64 data)
3553 {
3554         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3555
3556         return dd->send_egress_err_status_cnt[11];
3557 }
3558
3559 static u64 access_egress_reserved_10_err_cnt(const struct cntr_entry *entry,
3560                                              void *context, int vl, int mode,
3561                                              u64 data)
3562 {
3563         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3564
3565         return dd->send_egress_err_status_cnt[10];
3566 }
3567
3568 static u64 access_egress_reserved_9_err_cnt(const struct cntr_entry *entry,
3569                                             void *context, int vl, int mode,
3570                                             u64 data)
3571 {
3572         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3573
3574         return dd->send_egress_err_status_cnt[9];
3575 }
3576
3577 static u64 access_tx_sdma_launch_intf_parity_err_cnt(
3578                                 const struct cntr_entry *entry,
3579                                 void *context, int vl, int mode, u64 data)
3580 {
3581         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3582
3583         return dd->send_egress_err_status_cnt[8];
3584 }
3585
3586 static u64 access_tx_pio_launch_intf_parity_err_cnt(
3587                                 const struct cntr_entry *entry,
3588                                 void *context, int vl, int mode, u64 data)
3589 {
3590         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3591
3592         return dd->send_egress_err_status_cnt[7];
3593 }
3594
3595 static u64 access_egress_reserved_6_err_cnt(const struct cntr_entry *entry,
3596                                             void *context, int vl, int mode,
3597                                             u64 data)
3598 {
3599         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3600
3601         return dd->send_egress_err_status_cnt[6];
3602 }
3603
3604 static u64 access_tx_incorrect_link_state_err_cnt(
3605                                 const struct cntr_entry *entry,
3606                                 void *context, int vl, int mode, u64 data)
3607 {
3608         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3609
3610         return dd->send_egress_err_status_cnt[5];
3611 }
3612
3613 static u64 access_tx_linkdown_err_cnt(const struct cntr_entry *entry,
3614                                       void *context, int vl, int mode,
3615                                       u64 data)
3616 {
3617         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3618
3619         return dd->send_egress_err_status_cnt[4];
3620 }
3621
3622 static u64 access_tx_egress_fifi_underrun_or_parity_err_cnt(
3623                                 const struct cntr_entry *entry,
3624                                 void *context, int vl, int mode, u64 data)
3625 {
3626         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3627
3628         return dd->send_egress_err_status_cnt[3];
3629 }
3630
3631 static u64 access_egress_reserved_2_err_cnt(const struct cntr_entry *entry,
3632                                             void *context, int vl, int mode,
3633                                             u64 data)
3634 {
3635         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3636
3637         return dd->send_egress_err_status_cnt[2];
3638 }
3639
3640 static u64 access_tx_pkt_integrity_mem_unc_err_cnt(
3641                                 const struct cntr_entry *entry,
3642                                 void *context, int vl, int mode, u64 data)
3643 {
3644         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3645
3646         return dd->send_egress_err_status_cnt[1];
3647 }
3648
3649 static u64 access_tx_pkt_integrity_mem_cor_err_cnt(
3650                                 const struct cntr_entry *entry,
3651                                 void *context, int vl, int mode, u64 data)
3652 {
3653         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3654
3655         return dd->send_egress_err_status_cnt[0];
3656 }
3657
3658 /*
3659  * Software counters corresponding to each of the
3660  * error status bits within SendErrStatus
3661  */
3662 static u64 access_send_csr_write_bad_addr_err_cnt(
3663                                 const struct cntr_entry *entry,
3664                                 void *context, int vl, int mode, u64 data)
3665 {
3666         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3667
3668         return dd->send_err_status_cnt[2];
3669 }
3670
3671 static u64 access_send_csr_read_bad_addr_err_cnt(const struct cntr_entry *entry,
3672                                                  void *context, int vl,
3673                                                  int mode, u64 data)
3674 {
3675         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3676
3677         return dd->send_err_status_cnt[1];
3678 }
3679
3680 static u64 access_send_csr_parity_cnt(const struct cntr_entry *entry,
3681                                       void *context, int vl, int mode,
3682                                       u64 data)
3683 {
3684         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3685
3686         return dd->send_err_status_cnt[0];
3687 }
3688
3689 /*
3690  * Software counters corresponding to each of the
3691  * error status bits within SendCtxtErrStatus
3692  */
3693 static u64 access_pio_write_out_of_bounds_err_cnt(
3694                                 const struct cntr_entry *entry,
3695                                 void *context, int vl, int mode, u64 data)
3696 {
3697         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3698
3699         return dd->sw_ctxt_err_status_cnt[4];
3700 }
3701
3702 static u64 access_pio_write_overflow_err_cnt(const struct cntr_entry *entry,
3703                                              void *context, int vl, int mode,
3704                                              u64 data)
3705 {
3706         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3707
3708         return dd->sw_ctxt_err_status_cnt[3];
3709 }
3710
3711 static u64 access_pio_write_crosses_boundary_err_cnt(
3712                                 const struct cntr_entry *entry,
3713                                 void *context, int vl, int mode, u64 data)
3714 {
3715         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3716
3717         return dd->sw_ctxt_err_status_cnt[2];
3718 }
3719
3720 static u64 access_pio_disallowed_packet_err_cnt(const struct cntr_entry *entry,
3721                                                 void *context, int vl,
3722                                                 int mode, u64 data)
3723 {
3724         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3725
3726         return dd->sw_ctxt_err_status_cnt[1];
3727 }
3728
3729 static u64 access_pio_inconsistent_sop_err_cnt(const struct cntr_entry *entry,
3730                                                void *context, int vl, int mode,
3731                                                u64 data)
3732 {
3733         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3734
3735         return dd->sw_ctxt_err_status_cnt[0];
3736 }
3737
3738 /*
3739  * Software counters corresponding to each of the
3740  * error status bits within SendDmaEngErrStatus
3741  */
3742 static u64 access_sdma_header_request_fifo_cor_err_cnt(
3743                                 const struct cntr_entry *entry,
3744                                 void *context, int vl, int mode, u64 data)
3745 {
3746         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3747
3748         return dd->sw_send_dma_eng_err_status_cnt[23];
3749 }
3750
3751 static u64 access_sdma_header_storage_cor_err_cnt(
3752                                 const struct cntr_entry *entry,
3753                                 void *context, int vl, int mode, u64 data)
3754 {
3755         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3756
3757         return dd->sw_send_dma_eng_err_status_cnt[22];
3758 }
3759
3760 static u64 access_sdma_packet_tracking_cor_err_cnt(
3761                                 const struct cntr_entry *entry,
3762                                 void *context, int vl, int mode, u64 data)
3763 {
3764         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3765
3766         return dd->sw_send_dma_eng_err_status_cnt[21];
3767 }
3768
3769 static u64 access_sdma_assembly_cor_err_cnt(const struct cntr_entry *entry,
3770                                             void *context, int vl, int mode,
3771                                             u64 data)
3772 {
3773         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3774
3775         return dd->sw_send_dma_eng_err_status_cnt[20];
3776 }
3777
3778 static u64 access_sdma_desc_table_cor_err_cnt(const struct cntr_entry *entry,
3779                                               void *context, int vl, int mode,
3780                                               u64 data)
3781 {
3782         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3783
3784         return dd->sw_send_dma_eng_err_status_cnt[19];
3785 }
3786
3787 static u64 access_sdma_header_request_fifo_unc_err_cnt(
3788                                 const struct cntr_entry *entry,
3789                                 void *context, int vl, int mode, u64 data)
3790 {
3791         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3792
3793         return dd->sw_send_dma_eng_err_status_cnt[18];
3794 }
3795
3796 static u64 access_sdma_header_storage_unc_err_cnt(
3797                                 const struct cntr_entry *entry,
3798                                 void *context, int vl, int mode, u64 data)
3799 {
3800         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3801
3802         return dd->sw_send_dma_eng_err_status_cnt[17];
3803 }
3804
3805 static u64 access_sdma_packet_tracking_unc_err_cnt(
3806                                 const struct cntr_entry *entry,
3807                                 void *context, int vl, int mode, u64 data)
3808 {
3809         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3810
3811         return dd->sw_send_dma_eng_err_status_cnt[16];
3812 }
3813
3814 static u64 access_sdma_assembly_unc_err_cnt(const struct cntr_entry *entry,
3815                                             void *context, int vl, int mode,
3816                                             u64 data)
3817 {
3818         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3819
3820         return dd->sw_send_dma_eng_err_status_cnt[15];
3821 }
3822
3823 static u64 access_sdma_desc_table_unc_err_cnt(const struct cntr_entry *entry,
3824                                               void *context, int vl, int mode,
3825                                               u64 data)
3826 {
3827         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3828
3829         return dd->sw_send_dma_eng_err_status_cnt[14];
3830 }
3831
3832 static u64 access_sdma_timeout_err_cnt(const struct cntr_entry *entry,
3833                                        void *context, int vl, int mode,
3834                                        u64 data)
3835 {
3836         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3837
3838         return dd->sw_send_dma_eng_err_status_cnt[13];
3839 }
3840
3841 static u64 access_sdma_header_length_err_cnt(const struct cntr_entry *entry,
3842                                              void *context, int vl, int mode,
3843                                              u64 data)
3844 {
3845         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3846
3847         return dd->sw_send_dma_eng_err_status_cnt[12];
3848 }
3849
3850 static u64 access_sdma_header_address_err_cnt(const struct cntr_entry *entry,
3851                                               void *context, int vl, int mode,
3852                                               u64 data)
3853 {
3854         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3855
3856         return dd->sw_send_dma_eng_err_status_cnt[11];
3857 }
3858
3859 static u64 access_sdma_header_select_err_cnt(const struct cntr_entry *entry,
3860                                              void *context, int vl, int mode,
3861                                              u64 data)
3862 {
3863         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3864
3865         return dd->sw_send_dma_eng_err_status_cnt[10];
3866 }
3867
3868 static u64 access_sdma_reserved_9_err_cnt(const struct cntr_entry *entry,
3869                                           void *context, int vl, int mode,
3870                                           u64 data)
3871 {
3872         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3873
3874         return dd->sw_send_dma_eng_err_status_cnt[9];
3875 }
3876
3877 static u64 access_sdma_packet_desc_overflow_err_cnt(
3878                                 const struct cntr_entry *entry,
3879                                 void *context, int vl, int mode, u64 data)
3880 {
3881         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3882
3883         return dd->sw_send_dma_eng_err_status_cnt[8];
3884 }
3885
3886 static u64 access_sdma_length_mismatch_err_cnt(const struct cntr_entry *entry,
3887                                                void *context, int vl,
3888                                                int mode, u64 data)
3889 {
3890         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3891
3892         return dd->sw_send_dma_eng_err_status_cnt[7];
3893 }
3894
3895 static u64 access_sdma_halt_err_cnt(const struct cntr_entry *entry,
3896                                     void *context, int vl, int mode, u64 data)
3897 {
3898         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3899
3900         return dd->sw_send_dma_eng_err_status_cnt[6];
3901 }
3902
3903 static u64 access_sdma_mem_read_err_cnt(const struct cntr_entry *entry,
3904                                         void *context, int vl, int mode,
3905                                         u64 data)
3906 {
3907         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3908
3909         return dd->sw_send_dma_eng_err_status_cnt[5];
3910 }
3911
3912 static u64 access_sdma_first_desc_err_cnt(const struct cntr_entry *entry,
3913                                           void *context, int vl, int mode,
3914                                           u64 data)
3915 {
3916         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3917
3918         return dd->sw_send_dma_eng_err_status_cnt[4];
3919 }
3920
3921 static u64 access_sdma_tail_out_of_bounds_err_cnt(
3922                                 const struct cntr_entry *entry,
3923                                 void *context, int vl, int mode, u64 data)
3924 {
3925         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3926
3927         return dd->sw_send_dma_eng_err_status_cnt[3];
3928 }
3929
3930 static u64 access_sdma_too_long_err_cnt(const struct cntr_entry *entry,
3931                                         void *context, int vl, int mode,
3932                                         u64 data)
3933 {
3934         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3935
3936         return dd->sw_send_dma_eng_err_status_cnt[2];
3937 }
3938
3939 static u64 access_sdma_gen_mismatch_err_cnt(const struct cntr_entry *entry,
3940                                             void *context, int vl, int mode,
3941                                             u64 data)
3942 {
3943         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3944
3945         return dd->sw_send_dma_eng_err_status_cnt[1];
3946 }
3947
3948 static u64 access_sdma_wrong_dw_err_cnt(const struct cntr_entry *entry,
3949                                         void *context, int vl, int mode,
3950                                         u64 data)
3951 {
3952         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3953
3954         return dd->sw_send_dma_eng_err_status_cnt[0];
3955 }
3956
3957 static u64 access_dc_rcv_err_cnt(const struct cntr_entry *entry,
3958                                  void *context, int vl, int mode,
3959                                  u64 data)
3960 {
3961         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3962
3963         u64 val = 0;
3964         u64 csr = entry->csr;
3965
3966         val = read_write_csr(dd, csr, mode, data);
3967         if (mode == CNTR_MODE_R) {
3968                 val = val > CNTR_MAX - dd->sw_rcv_bypass_packet_errors ?
3969                         CNTR_MAX : val + dd->sw_rcv_bypass_packet_errors;
3970         } else if (mode == CNTR_MODE_W) {
3971                 dd->sw_rcv_bypass_packet_errors = 0;
3972         } else {
3973                 dd_dev_err(dd, "Invalid cntr register access mode");
3974                 return 0;
3975         }
3976         return val;
3977 }
3978
3979 #define def_access_sw_cpu(cntr) \
3980 static u64 access_sw_cpu_##cntr(const struct cntr_entry *entry,               \
3981                               void *context, int vl, int mode, u64 data)      \
3982 {                                                                             \
3983         struct hfi1_pportdata *ppd = (struct hfi1_pportdata *)context;        \
3984         return read_write_cpu(ppd->dd, &ppd->ibport_data.rvp.z_ ##cntr,       \
3985                               ppd->ibport_data.rvp.cntr, vl,                  \
3986                               mode, data);                                    \
3987 }
3988
3989 def_access_sw_cpu(rc_acks);
3990 def_access_sw_cpu(rc_qacks);
3991 def_access_sw_cpu(rc_delayed_comp);
3992
3993 #define def_access_ibp_counter(cntr) \
3994 static u64 access_ibp_##cntr(const struct cntr_entry *entry,                  \
3995                                 void *context, int vl, int mode, u64 data)    \
3996 {                                                                             \
3997         struct hfi1_pportdata *ppd = (struct hfi1_pportdata *)context;        \
3998                                                                               \
3999         if (vl != CNTR_INVALID_VL)                                            \
4000                 return 0;                                                     \
4001                                                                               \
4002         return read_write_sw(ppd->dd, &ppd->ibport_data.rvp.n_ ##cntr,        \
4003                              mode, data);                                     \
4004 }
4005
4006 def_access_ibp_counter(loop_pkts);
4007 def_access_ibp_counter(rc_resends);
4008 def_access_ibp_counter(rnr_naks);
4009 def_access_ibp_counter(other_naks);
4010 def_access_ibp_counter(rc_timeouts);
4011 def_access_ibp_counter(pkt_drops);
4012 def_access_ibp_counter(dmawait);
4013 def_access_ibp_counter(rc_seqnak);
4014 def_access_ibp_counter(rc_dupreq);
4015 def_access_ibp_counter(rdma_seq);
4016 def_access_ibp_counter(unaligned);
4017 def_access_ibp_counter(seq_naks);
4018
4019 static struct cntr_entry dev_cntrs[DEV_CNTR_LAST] = {
4020 [C_RCV_OVF] = RXE32_DEV_CNTR_ELEM(RcvOverflow, RCV_BUF_OVFL_CNT, CNTR_SYNTH),
4021 [C_RX_TID_FULL] = RXE32_DEV_CNTR_ELEM(RxTIDFullEr, RCV_TID_FULL_ERR_CNT,
4022                         CNTR_NORMAL),
4023 [C_RX_TID_INVALID] = RXE32_DEV_CNTR_ELEM(RxTIDInvalid, RCV_TID_VALID_ERR_CNT,
4024                         CNTR_NORMAL),
4025 [C_RX_TID_FLGMS] = RXE32_DEV_CNTR_ELEM(RxTidFLGMs,
4026                         RCV_TID_FLOW_GEN_MISMATCH_CNT,
4027                         CNTR_NORMAL),
4028 [C_RX_CTX_EGRS] = RXE32_DEV_CNTR_ELEM(RxCtxEgrS, RCV_CONTEXT_EGR_STALL,
4029                         CNTR_NORMAL),
4030 [C_RCV_TID_FLSMS] = RXE32_DEV_CNTR_ELEM(RxTidFLSMs,
4031                         RCV_TID_FLOW_SEQ_MISMATCH_CNT, CNTR_NORMAL),
4032 [C_CCE_PCI_CR_ST] = CCE_PERF_DEV_CNTR_ELEM(CcePciCrSt,
4033                         CCE_PCIE_POSTED_CRDT_STALL_CNT, CNTR_NORMAL),
4034 [C_CCE_PCI_TR_ST] = CCE_PERF_DEV_CNTR_ELEM(CcePciTrSt, CCE_PCIE_TRGT_STALL_CNT,
4035                         CNTR_NORMAL),
4036 [C_CCE_PIO_WR_ST] = CCE_PERF_DEV_CNTR_ELEM(CcePioWrSt, CCE_PIO_WR_STALL_CNT,
4037                         CNTR_NORMAL),
4038 [C_CCE_ERR_INT] = CCE_INT_DEV_CNTR_ELEM(CceErrInt, CCE_ERR_INT_CNT,
4039                         CNTR_NORMAL),
4040 [C_CCE_SDMA_INT] = CCE_INT_DEV_CNTR_ELEM(CceSdmaInt, CCE_SDMA_INT_CNT,
4041                         CNTR_NORMAL),
4042 [C_CCE_MISC_INT] = CCE_INT_DEV_CNTR_ELEM(CceMiscInt, CCE_MISC_INT_CNT,
4043                         CNTR_NORMAL),
4044 [C_CCE_RCV_AV_INT] = CCE_INT_DEV_CNTR_ELEM(CceRcvAvInt, CCE_RCV_AVAIL_INT_CNT,
4045                         CNTR_NORMAL),
4046 [C_CCE_RCV_URG_INT] = CCE_INT_DEV_CNTR_ELEM(CceRcvUrgInt,
4047                         CCE_RCV_URGENT_INT_CNT, CNTR_NORMAL),
4048 [C_CCE_SEND_CR_INT] = CCE_INT_DEV_CNTR_ELEM(CceSndCrInt,
4049                         CCE_SEND_CREDIT_INT_CNT, CNTR_NORMAL),
4050 [C_DC_UNC_ERR] = DC_PERF_CNTR(DcUnctblErr, DCC_ERR_UNCORRECTABLE_CNT,
4051                               CNTR_SYNTH),
4052 [C_DC_RCV_ERR] = CNTR_ELEM("DcRecvErr", DCC_ERR_PORTRCV_ERR_CNT, 0, CNTR_SYNTH,
4053                             access_dc_rcv_err_cnt),
4054 [C_DC_FM_CFG_ERR] = DC_PERF_CNTR(DcFmCfgErr, DCC_ERR_FMCONFIG_ERR_CNT,
4055                                  CNTR_SYNTH),
4056 [C_DC_RMT_PHY_ERR] = DC_PERF_CNTR(DcRmtPhyErr, DCC_ERR_RCVREMOTE_PHY_ERR_CNT,
4057                                   CNTR_SYNTH),
4058 [C_DC_DROPPED_PKT] = DC_PERF_CNTR(DcDroppedPkt, DCC_ERR_DROPPED_PKT_CNT,
4059                                   CNTR_SYNTH),
4060 [C_DC_MC_XMIT_PKTS] = DC_PERF_CNTR(DcMcXmitPkts,
4061                                    DCC_PRF_PORT_XMIT_MULTICAST_CNT, CNTR_SYNTH),
4062 [C_DC_MC_RCV_PKTS] = DC_PERF_CNTR(DcMcRcvPkts,
4063                                   DCC_PRF_PORT_RCV_MULTICAST_PKT_CNT,
4064                                   CNTR_SYNTH),
4065 [C_DC_XMIT_CERR] = DC_PERF_CNTR(DcXmitCorr,
4066                                 DCC_PRF_PORT_XMIT_CORRECTABLE_CNT, CNTR_SYNTH),
4067 [C_DC_RCV_CERR] = DC_PERF_CNTR(DcRcvCorrCnt, DCC_PRF_PORT_RCV_CORRECTABLE_CNT,
4068                                CNTR_SYNTH),
4069 [C_DC_RCV_FCC] = DC_PERF_CNTR(DcRxFCntl, DCC_PRF_RX_FLOW_CRTL_CNT,
4070                               CNTR_SYNTH),
4071 [C_DC_XMIT_FCC] = DC_PERF_CNTR(DcXmitFCntl, DCC_PRF_TX_FLOW_CRTL_CNT,
4072                                CNTR_SYNTH),
4073 [C_DC_XMIT_FLITS] = DC_PERF_CNTR(DcXmitFlits, DCC_PRF_PORT_XMIT_DATA_CNT,
4074                                  CNTR_SYNTH),
4075 [C_DC_RCV_FLITS] = DC_PERF_CNTR(DcRcvFlits, DCC_PRF_PORT_RCV_DATA_CNT,
4076                                 CNTR_SYNTH),
4077 [C_DC_XMIT_PKTS] = DC_PERF_CNTR(DcXmitPkts, DCC_PRF_PORT_XMIT_PKTS_CNT,
4078                                 CNTR_SYNTH),
4079 [C_DC_RCV_PKTS] = DC_PERF_CNTR(DcRcvPkts, DCC_PRF_PORT_RCV_PKTS_CNT,
4080                                CNTR_SYNTH),
4081 [C_DC_RX_FLIT_VL] = DC_PERF_CNTR(DcRxFlitVl, DCC_PRF_PORT_VL_RCV_DATA_CNT,
4082                                  CNTR_SYNTH | CNTR_VL),
4083 [C_DC_RX_PKT_VL] = DC_PERF_CNTR(DcRxPktVl, DCC_PRF_PORT_VL_RCV_PKTS_CNT,
4084                                 CNTR_SYNTH | CNTR_VL),
4085 [C_DC_RCV_FCN] = DC_PERF_CNTR(DcRcvFcn, DCC_PRF_PORT_RCV_FECN_CNT, CNTR_SYNTH),
4086 [C_DC_RCV_FCN_VL] = DC_PERF_CNTR(DcRcvFcnVl, DCC_PRF_PORT_VL_RCV_FECN_CNT,
4087                                  CNTR_SYNTH | CNTR_VL),
4088 [C_DC_RCV_BCN] = DC_PERF_CNTR(DcRcvBcn, DCC_PRF_PORT_RCV_BECN_CNT, CNTR_SYNTH),
4089 [C_DC_RCV_BCN_VL] = DC_PERF_CNTR(DcRcvBcnVl, DCC_PRF_PORT_VL_RCV_BECN_CNT,
4090                                  CNTR_SYNTH | CNTR_VL),
4091 [C_DC_RCV_BBL] = DC_PERF_CNTR(DcRcvBbl, DCC_PRF_PORT_RCV_BUBBLE_CNT,
4092                               CNTR_SYNTH),
4093 [C_DC_RCV_BBL_VL] = DC_PERF_CNTR(DcRcvBblVl, DCC_PRF_PORT_VL_RCV_BUBBLE_CNT,
4094                                  CNTR_SYNTH | CNTR_VL),
4095 [C_DC_MARK_FECN] = DC_PERF_CNTR(DcMarkFcn, DCC_PRF_PORT_MARK_FECN_CNT,
4096                                 CNTR_SYNTH),
4097 [C_DC_MARK_FECN_VL] = DC_PERF_CNTR(DcMarkFcnVl, DCC_PRF_PORT_VL_MARK_FECN_CNT,
4098                                    CNTR_SYNTH | CNTR_VL),
4099 [C_DC_TOTAL_CRC] =
4100         DC_PERF_CNTR_LCB(DcTotCrc, DC_LCB_ERR_INFO_TOTAL_CRC_ERR,
4101                          CNTR_SYNTH),
4102 [C_DC_CRC_LN0] = DC_PERF_CNTR_LCB(DcCrcLn0, DC_LCB_ERR_INFO_CRC_ERR_LN0,
4103                                   CNTR_SYNTH),
4104 [C_DC_CRC_LN1] = DC_PERF_CNTR_LCB(DcCrcLn1, DC_LCB_ERR_INFO_CRC_ERR_LN1,
4105                                   CNTR_SYNTH),
4106 [C_DC_CRC_LN2] = DC_PERF_CNTR_LCB(DcCrcLn2, DC_LCB_ERR_INFO_CRC_ERR_LN2,
4107                                   CNTR_SYNTH),
4108 [C_DC_CRC_LN3] = DC_PERF_CNTR_LCB(DcCrcLn3, DC_LCB_ERR_INFO_CRC_ERR_LN3,
4109                                   CNTR_SYNTH),
4110 [C_DC_CRC_MULT_LN] =
4111         DC_PERF_CNTR_LCB(DcMultLn, DC_LCB_ERR_INFO_CRC_ERR_MULTI_LN,
4112                          CNTR_SYNTH),
4113 [C_DC_TX_REPLAY] = DC_PERF_CNTR_LCB(DcTxReplay, DC_LCB_ERR_INFO_TX_REPLAY_CNT,
4114                                     CNTR_SYNTH),
4115 [C_DC_RX_REPLAY] = DC_PERF_CNTR_LCB(DcRxReplay, DC_LCB_ERR_INFO_RX_REPLAY_CNT,
4116                                     CNTR_SYNTH),
4117 [C_DC_SEQ_CRC_CNT] =
4118         DC_PERF_CNTR_LCB(DcLinkSeqCrc, DC_LCB_ERR_INFO_SEQ_CRC_CNT,
4119                          CNTR_SYNTH),
4120 [C_DC_ESC0_ONLY_CNT] =
4121         DC_PERF_CNTR_LCB(DcEsc0, DC_LCB_ERR_INFO_ESCAPE_0_ONLY_CNT,
4122                          CNTR_SYNTH),
4123 [C_DC_ESC0_PLUS1_CNT] =
4124         DC_PERF_CNTR_LCB(DcEsc1, DC_LCB_ERR_INFO_ESCAPE_0_PLUS1_CNT,
4125                          CNTR_SYNTH),
4126 [C_DC_ESC0_PLUS2_CNT] =
4127         DC_PERF_CNTR_LCB(DcEsc0Plus2, DC_LCB_ERR_INFO_ESCAPE_0_PLUS2_CNT,
4128                          CNTR_SYNTH),
4129 [C_DC_REINIT_FROM_PEER_CNT] =
4130         DC_PERF_CNTR_LCB(DcReinitPeer, DC_LCB_ERR_INFO_REINIT_FROM_PEER_CNT,
4131                          CNTR_SYNTH),
4132 [C_DC_SBE_CNT] = DC_PERF_CNTR_LCB(DcSbe, DC_LCB_ERR_INFO_SBE_CNT,
4133                                   CNTR_SYNTH),
4134 [C_DC_MISC_FLG_CNT] =
4135         DC_PERF_CNTR_LCB(DcMiscFlg, DC_LCB_ERR_INFO_MISC_FLG_CNT,
4136                          CNTR_SYNTH),
4137 [C_DC_PRF_GOOD_LTP_CNT] =
4138         DC_PERF_CNTR_LCB(DcGoodLTP, DC_LCB_PRF_GOOD_LTP_CNT, CNTR_SYNTH),
4139 [C_DC_PRF_ACCEPTED_LTP_CNT] =
4140         DC_PERF_CNTR_LCB(DcAccLTP, DC_LCB_PRF_ACCEPTED_LTP_CNT,
4141                          CNTR_SYNTH),
4142 [C_DC_PRF_RX_FLIT_CNT] =
4143         DC_PERF_CNTR_LCB(DcPrfRxFlit, DC_LCB_PRF_RX_FLIT_CNT, CNTR_SYNTH),
4144 [C_DC_PRF_TX_FLIT_CNT] =
4145         DC_PERF_CNTR_LCB(DcPrfTxFlit, DC_LCB_PRF_TX_FLIT_CNT, CNTR_SYNTH),
4146 [C_DC_PRF_CLK_CNTR] =
4147         DC_PERF_CNTR_LCB(DcPrfClk, DC_LCB_PRF_CLK_CNTR, CNTR_SYNTH),
4148 [C_DC_PG_DBG_FLIT_CRDTS_CNT] =
4149         DC_PERF_CNTR_LCB(DcFltCrdts, DC_LCB_PG_DBG_FLIT_CRDTS_CNT, CNTR_SYNTH),
4150 [C_DC_PG_STS_PAUSE_COMPLETE_CNT] =
4151         DC_PERF_CNTR_LCB(DcPauseComp, DC_LCB_PG_STS_PAUSE_COMPLETE_CNT,
4152                          CNTR_SYNTH),
4153 [C_DC_PG_STS_TX_SBE_CNT] =
4154         DC_PERF_CNTR_LCB(DcStsTxSbe, DC_LCB_PG_STS_TX_SBE_CNT, CNTR_SYNTH),
4155 [C_DC_PG_STS_TX_MBE_CNT] =
4156         DC_PERF_CNTR_LCB(DcStsTxMbe, DC_LCB_PG_STS_TX_MBE_CNT,
4157                          CNTR_SYNTH),
4158 [C_SW_CPU_INTR] = CNTR_ELEM("Intr", 0, 0, CNTR_NORMAL,
4159                             access_sw_cpu_intr),
4160 [C_SW_CPU_RCV_LIM] = CNTR_ELEM("RcvLimit", 0, 0, CNTR_NORMAL,
4161                             access_sw_cpu_rcv_limit),
4162 [C_SW_VTX_WAIT] = CNTR_ELEM("vTxWait", 0, 0, CNTR_NORMAL,
4163                             access_sw_vtx_wait),
4164 [C_SW_PIO_WAIT] = CNTR_ELEM("PioWait", 0, 0, CNTR_NORMAL,
4165                             access_sw_pio_wait),
4166 [C_SW_PIO_DRAIN] = CNTR_ELEM("PioDrain", 0, 0, CNTR_NORMAL,
4167                             access_sw_pio_drain),
4168 [C_SW_KMEM_WAIT] = CNTR_ELEM("KmemWait", 0, 0, CNTR_NORMAL,
4169                             access_sw_kmem_wait),
4170 [C_SW_SEND_SCHED] = CNTR_ELEM("SendSched", 0, 0, CNTR_NORMAL,
4171                             access_sw_send_schedule),
4172 [C_SDMA_DESC_FETCHED_CNT] = CNTR_ELEM("SDEDscFdCn",
4173                                       SEND_DMA_DESC_FETCHED_CNT, 0,
4174                                       CNTR_NORMAL | CNTR_32BIT | CNTR_SDMA,
4175                                       dev_access_u32_csr),
4176 [C_SDMA_INT_CNT] = CNTR_ELEM("SDMAInt", 0, 0,
4177                              CNTR_NORMAL | CNTR_32BIT | CNTR_SDMA,
4178                              access_sde_int_cnt),
4179 [C_SDMA_ERR_CNT] = CNTR_ELEM("SDMAErrCt", 0, 0,
4180                              CNTR_NORMAL | CNTR_32BIT | CNTR_SDMA,
4181                              access_sde_err_cnt),
4182 [C_SDMA_IDLE_INT_CNT] = CNTR_ELEM("SDMAIdInt", 0, 0,
4183                                   CNTR_NORMAL | CNTR_32BIT | CNTR_SDMA,
4184                                   access_sde_idle_int_cnt),
4185 [C_SDMA_PROGRESS_INT_CNT] = CNTR_ELEM("SDMAPrIntCn", 0, 0,
4186                                       CNTR_NORMAL | CNTR_32BIT | CNTR_SDMA,
4187                                       access_sde_progress_int_cnt),
4188 /* MISC_ERR_STATUS */
4189 [C_MISC_PLL_LOCK_FAIL_ERR] = CNTR_ELEM("MISC_PLL_LOCK_FAIL_ERR", 0, 0,
4190                                 CNTR_NORMAL,
4191                                 access_misc_pll_lock_fail_err_cnt),
4192 [C_MISC_MBIST_FAIL_ERR] = CNTR_ELEM("MISC_MBIST_FAIL_ERR", 0, 0,
4193                                 CNTR_NORMAL,
4194                                 access_misc_mbist_fail_err_cnt),
4195 [C_MISC_INVALID_EEP_CMD_ERR] = CNTR_ELEM("MISC_INVALID_EEP_CMD_ERR", 0, 0,
4196                                 CNTR_NORMAL,
4197                                 access_misc_invalid_eep_cmd_err_cnt),
4198 [C_MISC_EFUSE_DONE_PARITY_ERR] = CNTR_ELEM("MISC_EFUSE_DONE_PARITY_ERR", 0, 0,
4199                                 CNTR_NORMAL,
4200                                 access_misc_efuse_done_parity_err_cnt),
4201 [C_MISC_EFUSE_WRITE_ERR] = CNTR_ELEM("MISC_EFUSE_WRITE_ERR", 0, 0,
4202                                 CNTR_NORMAL,
4203                                 access_misc_efuse_write_err_cnt),
4204 [C_MISC_EFUSE_READ_BAD_ADDR_ERR] = CNTR_ELEM("MISC_EFUSE_READ_BAD_ADDR_ERR", 0,
4205                                 0, CNTR_NORMAL,
4206                                 access_misc_efuse_read_bad_addr_err_cnt),
4207 [C_MISC_EFUSE_CSR_PARITY_ERR] = CNTR_ELEM("MISC_EFUSE_CSR_PARITY_ERR", 0, 0,
4208                                 CNTR_NORMAL,
4209                                 access_misc_efuse_csr_parity_err_cnt),
4210 [C_MISC_FW_AUTH_FAILED_ERR] = CNTR_ELEM("MISC_FW_AUTH_FAILED_ERR", 0, 0,
4211                                 CNTR_NORMAL,
4212                                 access_misc_fw_auth_failed_err_cnt),
4213 [C_MISC_KEY_MISMATCH_ERR] = CNTR_ELEM("MISC_KEY_MISMATCH_ERR", 0, 0,
4214                                 CNTR_NORMAL,
4215                                 access_misc_key_mismatch_err_cnt),
4216 [C_MISC_SBUS_WRITE_FAILED_ERR] = CNTR_ELEM("MISC_SBUS_WRITE_FAILED_ERR", 0, 0,
4217                                 CNTR_NORMAL,
4218                                 access_misc_sbus_write_failed_err_cnt),
4219 [C_MISC_CSR_WRITE_BAD_ADDR_ERR] = CNTR_ELEM("MISC_CSR_WRITE_BAD_ADDR_ERR", 0, 0,
4220                                 CNTR_NORMAL,
4221                                 access_misc_csr_write_bad_addr_err_cnt),
4222 [C_MISC_CSR_READ_BAD_ADDR_ERR] = CNTR_ELEM("MISC_CSR_READ_BAD_ADDR_ERR", 0, 0,
4223                                 CNTR_NORMAL,
4224                                 access_misc_csr_read_bad_addr_err_cnt),
4225 [C_MISC_CSR_PARITY_ERR] = CNTR_ELEM("MISC_CSR_PARITY_ERR", 0, 0,
4226                                 CNTR_NORMAL,
4227                                 access_misc_csr_parity_err_cnt),
4228 /* CceErrStatus */
4229 [C_CCE_ERR_STATUS_AGGREGATED_CNT] = CNTR_ELEM("CceErrStatusAggregatedCnt", 0, 0,
4230                                 CNTR_NORMAL,
4231                                 access_sw_cce_err_status_aggregated_cnt),
4232 [C_CCE_MSIX_CSR_PARITY_ERR] = CNTR_ELEM("CceMsixCsrParityErr", 0, 0,
4233                                 CNTR_NORMAL,
4234                                 access_cce_msix_csr_parity_err_cnt),
4235 [C_CCE_INT_MAP_UNC_ERR] = CNTR_ELEM("CceIntMapUncErr", 0, 0,
4236                                 CNTR_NORMAL,
4237                                 access_cce_int_map_unc_err_cnt),
4238 [C_CCE_INT_MAP_COR_ERR] = CNTR_ELEM("CceIntMapCorErr", 0, 0,
4239                                 CNTR_NORMAL,
4240                                 access_cce_int_map_cor_err_cnt),
4241 [C_CCE_MSIX_TABLE_UNC_ERR] = CNTR_ELEM("CceMsixTableUncErr", 0, 0,
4242                                 CNTR_NORMAL,
4243                                 access_cce_msix_table_unc_err_cnt),
4244 [C_CCE_MSIX_TABLE_COR_ERR] = CNTR_ELEM("CceMsixTableCorErr", 0, 0,
4245                                 CNTR_NORMAL,
4246                                 access_cce_msix_table_cor_err_cnt),
4247 [C_CCE_RXDMA_CONV_FIFO_PARITY_ERR] = CNTR_ELEM("CceRxdmaConvFifoParityErr", 0,
4248                                 0, CNTR_NORMAL,
4249                                 access_cce_rxdma_conv_fifo_parity_err_cnt),
4250 [C_CCE_RCPL_ASYNC_FIFO_PARITY_ERR] = CNTR_ELEM("CceRcplAsyncFifoParityErr", 0,
4251                                 0, CNTR_NORMAL,
4252                                 access_cce_rcpl_async_fifo_parity_err_cnt),
4253 [C_CCE_SEG_WRITE_BAD_ADDR_ERR] = CNTR_ELEM("CceSegWriteBadAddrErr", 0, 0,
4254                                 CNTR_NORMAL,
4255                                 access_cce_seg_write_bad_addr_err_cnt),
4256 [C_CCE_SEG_READ_BAD_ADDR_ERR] = CNTR_ELEM("CceSegReadBadAddrErr", 0, 0,
4257                                 CNTR_NORMAL,
4258                                 access_cce_seg_read_bad_addr_err_cnt),
4259 [C_LA_TRIGGERED] = CNTR_ELEM("Cce LATriggered", 0, 0,
4260                                 CNTR_NORMAL,
4261                                 access_la_triggered_cnt),
4262 [C_CCE_TRGT_CPL_TIMEOUT_ERR] = CNTR_ELEM("CceTrgtCplTimeoutErr", 0, 0,
4263                                 CNTR_NORMAL,
4264                                 access_cce_trgt_cpl_timeout_err_cnt),
4265 [C_PCIC_RECEIVE_PARITY_ERR] = CNTR_ELEM("PcicReceiveParityErr", 0, 0,
4266                                 CNTR_NORMAL,
4267                                 access_pcic_receive_parity_err_cnt),
4268 [C_PCIC_TRANSMIT_BACK_PARITY_ERR] = CNTR_ELEM("PcicTransmitBackParityErr", 0, 0,
4269                                 CNTR_NORMAL,
4270                                 access_pcic_transmit_back_parity_err_cnt),
4271 [C_PCIC_TRANSMIT_FRONT_PARITY_ERR] = CNTR_ELEM("PcicTransmitFrontParityErr", 0,
4272                                 0, CNTR_NORMAL,
4273                                 access_pcic_transmit_front_parity_err_cnt),
4274 [C_PCIC_CPL_DAT_Q_UNC_ERR] = CNTR_ELEM("PcicCplDatQUncErr", 0, 0,
4275                                 CNTR_NORMAL,
4276                                 access_pcic_cpl_dat_q_unc_err_cnt),
4277 [C_PCIC_CPL_HD_Q_UNC_ERR] = CNTR_ELEM("PcicCplHdQUncErr", 0, 0,
4278                                 CNTR_NORMAL,
4279                                 access_pcic_cpl_hd_q_unc_err_cnt),
4280 [C_PCIC_POST_DAT_Q_UNC_ERR] = CNTR_ELEM("PcicPostDatQUncErr", 0, 0,
4281                                 CNTR_NORMAL,
4282                                 access_pcic_post_dat_q_unc_err_cnt),
4283 [C_PCIC_POST_HD_Q_UNC_ERR] = CNTR_ELEM("PcicPostHdQUncErr", 0, 0,
4284                                 CNTR_NORMAL,
4285                                 access_pcic_post_hd_q_unc_err_cnt),
4286 [C_PCIC_RETRY_SOT_MEM_UNC_ERR] = CNTR_ELEM("PcicRetrySotMemUncErr", 0, 0,
4287                                 CNTR_NORMAL,
4288                                 access_pcic_retry_sot_mem_unc_err_cnt),
4289 [C_PCIC_RETRY_MEM_UNC_ERR] = CNTR_ELEM("PcicRetryMemUncErr", 0, 0,
4290                                 CNTR_NORMAL,
4291                                 access_pcic_retry_mem_unc_err),
4292 [C_PCIC_N_POST_DAT_Q_PARITY_ERR] = CNTR_ELEM("PcicNPostDatQParityErr", 0, 0,
4293                                 CNTR_NORMAL,
4294                                 access_pcic_n_post_dat_q_parity_err_cnt),
4295 [C_PCIC_N_POST_H_Q_PARITY_ERR] = CNTR_ELEM("PcicNPostHQParityErr", 0, 0,
4296                                 CNTR_NORMAL,
4297                                 access_pcic_n_post_h_q_parity_err_cnt),
4298 [C_PCIC_CPL_DAT_Q_COR_ERR] = CNTR_ELEM("PcicCplDatQCorErr", 0, 0,
4299                                 CNTR_NORMAL,
4300                                 access_pcic_cpl_dat_q_cor_err_cnt),
4301 [C_PCIC_CPL_HD_Q_COR_ERR] = CNTR_ELEM("PcicCplHdQCorErr", 0, 0,
4302                                 CNTR_NORMAL,
4303                                 access_pcic_cpl_hd_q_cor_err_cnt),
4304 [C_PCIC_POST_DAT_Q_COR_ERR] = CNTR_ELEM("PcicPostDatQCorErr", 0, 0,
4305                                 CNTR_NORMAL,
4306                                 access_pcic_post_dat_q_cor_err_cnt),
4307 [C_PCIC_POST_HD_Q_COR_ERR] = CNTR_ELEM("PcicPostHdQCorErr", 0, 0,
4308                                 CNTR_NORMAL,
4309                                 access_pcic_post_hd_q_cor_err_cnt),
4310 [C_PCIC_RETRY_SOT_MEM_COR_ERR] = CNTR_ELEM("PcicRetrySotMemCorErr", 0, 0,
4311                                 CNTR_NORMAL,
4312                                 access_pcic_retry_sot_mem_cor_err_cnt),
4313 [C_PCIC_RETRY_MEM_COR_ERR] = CNTR_ELEM("PcicRetryMemCorErr", 0, 0,
4314                                 CNTR_NORMAL,
4315                                 access_pcic_retry_mem_cor_err_cnt),
4316 [C_CCE_CLI1_ASYNC_FIFO_DBG_PARITY_ERR] = CNTR_ELEM(
4317                                 "CceCli1AsyncFifoDbgParityError", 0, 0,
4318                                 CNTR_NORMAL,
4319                                 access_cce_cli1_async_fifo_dbg_parity_err_cnt),
4320 [C_CCE_CLI1_ASYNC_FIFO_RXDMA_PARITY_ERR] = CNTR_ELEM(
4321                                 "CceCli1AsyncFifoRxdmaParityError", 0, 0,
4322                                 CNTR_NORMAL,
4323                                 access_cce_cli1_async_fifo_rxdma_parity_err_cnt
4324                                 ),
4325 [C_CCE_CLI1_ASYNC_FIFO_SDMA_HD_PARITY_ERR] = CNTR_ELEM(
4326                         "CceCli1AsyncFifoSdmaHdParityErr", 0, 0,
4327                         CNTR_NORMAL,
4328                         access_cce_cli1_async_fifo_sdma_hd_parity_err_cnt),
4329 [C_CCE_CLI1_ASYNC_FIFO_PIO_CRDT_PARITY_ERR] = CNTR_ELEM(
4330                         "CceCli1AsyncFifoPioCrdtParityErr", 0, 0,
4331                         CNTR_NORMAL,
4332                         access_cce_cl1_async_fifo_pio_crdt_parity_err_cnt),
4333 [C_CCE_CLI2_ASYNC_FIFO_PARITY_ERR] = CNTR_ELEM("CceCli2AsyncFifoParityErr", 0,
4334                         0, CNTR_NORMAL,
4335                         access_cce_cli2_async_fifo_parity_err_cnt),
4336 [C_CCE_CSR_CFG_BUS_PARITY_ERR] = CNTR_ELEM("CceCsrCfgBusParityErr", 0, 0,
4337                         CNTR_NORMAL,
4338                         access_cce_csr_cfg_bus_parity_err_cnt),
4339 [C_CCE_CLI0_ASYNC_FIFO_PARTIY_ERR] = CNTR_ELEM("CceCli0AsyncFifoParityErr", 0,
4340                         0, CNTR_NORMAL,
4341                         access_cce_cli0_async_fifo_parity_err_cnt),
4342 [C_CCE_RSPD_DATA_PARITY_ERR] = CNTR_ELEM("CceRspdDataParityErr", 0, 0,
4343                         CNTR_NORMAL,
4344                         access_cce_rspd_data_parity_err_cnt),
4345 [C_CCE_TRGT_ACCESS_ERR] = CNTR_ELEM("CceTrgtAccessErr", 0, 0,
4346                         CNTR_NORMAL,
4347                         access_cce_trgt_access_err_cnt),
4348 [C_CCE_TRGT_ASYNC_FIFO_PARITY_ERR] = CNTR_ELEM("CceTrgtAsyncFifoParityErr", 0,
4349                         0, CNTR_NORMAL,
4350                         access_cce_trgt_async_fifo_parity_err_cnt),
4351 [C_CCE_CSR_WRITE_BAD_ADDR_ERR] = CNTR_ELEM("CceCsrWriteBadAddrErr", 0, 0,
4352                         CNTR_NORMAL,
4353                         access_cce_csr_write_bad_addr_err_cnt),
4354 [C_CCE_CSR_READ_BAD_ADDR_ERR] = CNTR_ELEM("CceCsrReadBadAddrErr", 0, 0,
4355                         CNTR_NORMAL,
4356                         access_cce_csr_read_bad_addr_err_cnt),
4357 [C_CCE_CSR_PARITY_ERR] = CNTR_ELEM("CceCsrParityErr", 0, 0,
4358                         CNTR_NORMAL,
4359                         access_ccs_csr_parity_err_cnt),
4360
4361 /* RcvErrStatus */
4362 [C_RX_CSR_PARITY_ERR] = CNTR_ELEM("RxCsrParityErr", 0, 0,
4363                         CNTR_NORMAL,
4364                         access_rx_csr_parity_err_cnt),
4365 [C_RX_CSR_WRITE_BAD_ADDR_ERR] = CNTR_ELEM("RxCsrWriteBadAddrErr", 0, 0,
4366                         CNTR_NORMAL,
4367                         access_rx_csr_write_bad_addr_err_cnt),
4368 [C_RX_CSR_READ_BAD_ADDR_ERR] = CNTR_ELEM("RxCsrReadBadAddrErr", 0, 0,
4369                         CNTR_NORMAL,
4370                         access_rx_csr_read_bad_addr_err_cnt),
4371 [C_RX_DMA_CSR_UNC_ERR] = CNTR_ELEM("RxDmaCsrUncErr", 0, 0,
4372                         CNTR_NORMAL,
4373                         access_rx_dma_csr_unc_err_cnt),
4374 [C_RX_DMA_DQ_FSM_ENCODING_ERR] = CNTR_ELEM("RxDmaDqFsmEncodingErr", 0, 0,
4375                         CNTR_NORMAL,
4376                         access_rx_dma_dq_fsm_encoding_err_cnt),
4377 [C_RX_DMA_EQ_FSM_ENCODING_ERR] = CNTR_ELEM("RxDmaEqFsmEncodingErr", 0, 0,
4378                         CNTR_NORMAL,
4379                         access_rx_dma_eq_fsm_encoding_err_cnt),
4380 [C_RX_DMA_CSR_PARITY_ERR] = CNTR_ELEM("RxDmaCsrParityErr", 0, 0,
4381                         CNTR_NORMAL,
4382                         access_rx_dma_csr_parity_err_cnt),
4383 [C_RX_RBUF_DATA_COR_ERR] = CNTR_ELEM("RxRbufDataCorErr", 0, 0,
4384                         CNTR_NORMAL,
4385                         access_rx_rbuf_data_cor_err_cnt),
4386 [C_RX_RBUF_DATA_UNC_ERR] = CNTR_ELEM("RxRbufDataUncErr", 0, 0,
4387                         CNTR_NORMAL,
4388                         access_rx_rbuf_data_unc_err_cnt),
4389 [C_RX_DMA_DATA_FIFO_RD_COR_ERR] = CNTR_ELEM("RxDmaDataFifoRdCorErr", 0, 0,
4390                         CNTR_NORMAL,
4391                         access_rx_dma_data_fifo_rd_cor_err_cnt),
4392 [C_RX_DMA_DATA_FIFO_RD_UNC_ERR] = CNTR_ELEM("RxDmaDataFifoRdUncErr", 0, 0,
4393                         CNTR_NORMAL,
4394                         access_rx_dma_data_fifo_rd_unc_err_cnt),
4395 [C_RX_DMA_HDR_FIFO_RD_COR_ERR] = CNTR_ELEM("RxDmaHdrFifoRdCorErr", 0, 0,
4396                         CNTR_NORMAL,
4397                         access_rx_dma_hdr_fifo_rd_cor_err_cnt),
4398 [C_RX_DMA_HDR_FIFO_RD_UNC_ERR] = CNTR_ELEM("RxDmaHdrFifoRdUncErr", 0, 0,
4399                         CNTR_NORMAL,
4400                         access_rx_dma_hdr_fifo_rd_unc_err_cnt),
4401 [C_RX_RBUF_DESC_PART2_COR_ERR] = CNTR_ELEM("RxRbufDescPart2CorErr", 0, 0,
4402                         CNTR_NORMAL,
4403                         access_rx_rbuf_desc_part2_cor_err_cnt),
4404 [C_RX_RBUF_DESC_PART2_UNC_ERR] = CNTR_ELEM("RxRbufDescPart2UncErr", 0, 0,
4405                         CNTR_NORMAL,
4406                         access_rx_rbuf_desc_part2_unc_err_cnt),
4407 [C_RX_RBUF_DESC_PART1_COR_ERR] = CNTR_ELEM("RxRbufDescPart1CorErr", 0, 0,
4408                         CNTR_NORMAL,
4409                         access_rx_rbuf_desc_part1_cor_err_cnt),
4410 [C_RX_RBUF_DESC_PART1_UNC_ERR] = CNTR_ELEM("RxRbufDescPart1UncErr", 0, 0,
4411                         CNTR_NORMAL,
4412                         access_rx_rbuf_desc_part1_unc_err_cnt),
4413 [C_RX_HQ_INTR_FSM_ERR] = CNTR_ELEM("RxHqIntrFsmErr", 0, 0,
4414                         CNTR_NORMAL,
4415                         access_rx_hq_intr_fsm_err_cnt),
4416 [C_RX_HQ_INTR_CSR_PARITY_ERR] = CNTR_ELEM("RxHqIntrCsrParityErr", 0, 0,
4417                         CNTR_NORMAL,
4418                         access_rx_hq_intr_csr_parity_err_cnt),
4419 [C_RX_LOOKUP_CSR_PARITY_ERR] = CNTR_ELEM("RxLookupCsrParityErr", 0, 0,
4420                         CNTR_NORMAL,
4421                         access_rx_lookup_csr_parity_err_cnt),
4422 [C_RX_LOOKUP_RCV_ARRAY_COR_ERR] = CNTR_ELEM("RxLookupRcvArrayCorErr", 0, 0,
4423                         CNTR_NORMAL,
4424                         access_rx_lookup_rcv_array_cor_err_cnt),
4425 [C_RX_LOOKUP_RCV_ARRAY_UNC_ERR] = CNTR_ELEM("RxLookupRcvArrayUncErr", 0, 0,
4426                         CNTR_NORMAL,
4427                         access_rx_lookup_rcv_array_unc_err_cnt),
4428 [C_RX_LOOKUP_DES_PART2_PARITY_ERR] = CNTR_ELEM("RxLookupDesPart2ParityErr", 0,
4429                         0, CNTR_NORMAL,
4430                         access_rx_lookup_des_part2_parity_err_cnt),
4431 [C_RX_LOOKUP_DES_PART1_UNC_COR_ERR] = CNTR_ELEM("RxLookupDesPart1UncCorErr", 0,
4432                         0, CNTR_NORMAL,
4433                         access_rx_lookup_des_part1_unc_cor_err_cnt),
4434 [C_RX_LOOKUP_DES_PART1_UNC_ERR] = CNTR_ELEM("RxLookupDesPart1UncErr", 0, 0,
4435                         CNTR_NORMAL,
4436                         access_rx_lookup_des_part1_unc_err_cnt),
4437 [C_RX_RBUF_NEXT_FREE_BUF_COR_ERR] = CNTR_ELEM("RxRbufNextFreeBufCorErr", 0, 0,
4438                         CNTR_NORMAL,
4439                         access_rx_rbuf_next_free_buf_cor_err_cnt),
4440 [C_RX_RBUF_NEXT_FREE_BUF_UNC_ERR] = CNTR_ELEM("RxRbufNextFreeBufUncErr", 0, 0,
4441                         CNTR_NORMAL,
4442                         access_rx_rbuf_next_free_buf_unc_err_cnt),
4443 [C_RX_RBUF_FL_INIT_WR_ADDR_PARITY_ERR] = CNTR_ELEM(
4444                         "RxRbufFlInitWrAddrParityErr", 0, 0,
4445                         CNTR_NORMAL,
4446                         access_rbuf_fl_init_wr_addr_parity_err_cnt),
4447 [C_RX_RBUF_FL_INITDONE_PARITY_ERR] = CNTR_ELEM("RxRbufFlInitdoneParityErr", 0,
4448                         0, CNTR_NORMAL,
4449                         access_rx_rbuf_fl_initdone_parity_err_cnt),
4450 [C_RX_RBUF_FL_WRITE_ADDR_PARITY_ERR] = CNTR_ELEM("RxRbufFlWrAddrParityErr", 0,
4451                         0, CNTR_NORMAL,
4452                         access_rx_rbuf_fl_write_addr_parity_err_cnt),
4453 [C_RX_RBUF_FL_RD_ADDR_PARITY_ERR] = CNTR_ELEM("RxRbufFlRdAddrParityErr", 0, 0,
4454                         CNTR_NORMAL,
4455                         access_rx_rbuf_fl_rd_addr_parity_err_cnt),
4456 [C_RX_RBUF_EMPTY_ERR] = CNTR_ELEM("RxRbufEmptyErr", 0, 0,
4457                         CNTR_NORMAL,
4458                         access_rx_rbuf_empty_err_cnt),
4459 [C_RX_RBUF_FULL_ERR] = CNTR_ELEM("RxRbufFullErr", 0, 0,
4460                         CNTR_NORMAL,
4461                         access_rx_rbuf_full_err_cnt),
4462 [C_RX_RBUF_BAD_LOOKUP_ERR] = CNTR_ELEM("RxRBufBadLookupErr", 0, 0,
4463                         CNTR_NORMAL,
4464                         access_rbuf_bad_lookup_err_cnt),
4465 [C_RX_RBUF_CTX_ID_PARITY_ERR] = CNTR_ELEM("RxRbufCtxIdParityErr", 0, 0,
4466                         CNTR_NORMAL,
4467                         access_rbuf_ctx_id_parity_err_cnt),
4468 [C_RX_RBUF_CSR_QEOPDW_PARITY_ERR] = CNTR_ELEM("RxRbufCsrQEOPDWParityErr", 0, 0,
4469                         CNTR_NORMAL,
4470                         access_rbuf_csr_qeopdw_parity_err_cnt),
4471 [C_RX_RBUF_CSR_Q_NUM_OF_PKT_PARITY_ERR] = CNTR_ELEM(
4472                         "RxRbufCsrQNumOfPktParityErr", 0, 0,
4473                         CNTR_NORMAL,
4474                         access_rx_rbuf_csr_q_num_of_pkt_parity_err_cnt),
4475 [C_RX_RBUF_CSR_Q_T1_PTR_PARITY_ERR] = CNTR_ELEM(
4476                         "RxRbufCsrQTlPtrParityErr", 0, 0,
4477                         CNTR_NORMAL,
4478                         access_rx_rbuf_csr_q_t1_ptr_parity_err_cnt),
4479 [C_RX_RBUF_CSR_Q_HD_PTR_PARITY_ERR] = CNTR_ELEM("RxRbufCsrQHdPtrParityErr", 0,
4480                         0, CNTR_NORMAL,
4481                         access_rx_rbuf_csr_q_hd_ptr_parity_err_cnt),
4482 [C_RX_RBUF_CSR_Q_VLD_BIT_PARITY_ERR] = CNTR_ELEM("RxRbufCsrQVldBitParityErr", 0,
4483                         0, CNTR_NORMAL,
4484                         access_rx_rbuf_csr_q_vld_bit_parity_err_cnt),
4485 [C_RX_RBUF_CSR_Q_NEXT_BUF_PARITY_ERR] = CNTR_ELEM("RxRbufCsrQNextBufParityErr",
4486                         0, 0, CNTR_NORMAL,
4487                         access_rx_rbuf_csr_q_next_buf_parity_err_cnt),
4488 [C_RX_RBUF_CSR_Q_ENT_CNT_PARITY_ERR] = CNTR_ELEM("RxRbufCsrQEntCntParityErr", 0,
4489                         0, CNTR_NORMAL,
4490                         access_rx_rbuf_csr_q_ent_cnt_parity_err_cnt),
4491 [C_RX_RBUF_CSR_Q_HEAD_BUF_NUM_PARITY_ERR] = CNTR_ELEM(
4492                         "RxRbufCsrQHeadBufNumParityErr", 0, 0,
4493                         CNTR_NORMAL,
4494                         access_rx_rbuf_csr_q_head_buf_num_parity_err_cnt),
4495 [C_RX_RBUF_BLOCK_LIST_READ_COR_ERR] = CNTR_ELEM("RxRbufBlockListReadCorErr", 0,
4496                         0, CNTR_NORMAL,
4497                         access_rx_rbuf_block_list_read_cor_err_cnt),
4498 [C_RX_RBUF_BLOCK_LIST_READ_UNC_ERR] = CNTR_ELEM("RxRbufBlockListReadUncErr", 0,
4499                         0, CNTR_NORMAL,
4500                         access_rx_rbuf_block_list_read_unc_err_cnt),
4501 [C_RX_RBUF_LOOKUP_DES_COR_ERR] = CNTR_ELEM("RxRbufLookupDesCorErr", 0, 0,
4502                         CNTR_NORMAL,
4503                         access_rx_rbuf_lookup_des_cor_err_cnt),
4504 [C_RX_RBUF_LOOKUP_DES_UNC_ERR] = CNTR_ELEM("RxRbufLookupDesUncErr", 0, 0,
4505                         CNTR_NORMAL,
4506                         access_rx_rbuf_lookup_des_unc_err_cnt),
4507 [C_RX_RBUF_LOOKUP_DES_REG_UNC_COR_ERR] = CNTR_ELEM(
4508                         "RxRbufLookupDesRegUncCorErr", 0, 0,
4509                         CNTR_NORMAL,
4510                         access_rx_rbuf_lookup_des_reg_unc_cor_err_cnt),
4511 [C_RX_RBUF_LOOKUP_DES_REG_UNC_ERR] = CNTR_ELEM("RxRbufLookupDesRegUncErr", 0, 0,
4512                         CNTR_NORMAL,
4513                         access_rx_rbuf_lookup_des_reg_unc_err_cnt),
4514 [C_RX_RBUF_FREE_LIST_COR_ERR] = CNTR_ELEM("RxRbufFreeListCorErr", 0, 0,
4515                         CNTR_NORMAL,
4516                         access_rx_rbuf_free_list_cor_err_cnt),
4517 [C_RX_RBUF_FREE_LIST_UNC_ERR] = CNTR_ELEM("RxRbufFreeListUncErr", 0, 0,
4518                         CNTR_NORMAL,
4519                         access_rx_rbuf_free_list_unc_err_cnt),
4520 [C_RX_RCV_FSM_ENCODING_ERR] = CNTR_ELEM("RxRcvFsmEncodingErr", 0, 0,
4521                         CNTR_NORMAL,
4522                         access_rx_rcv_fsm_encoding_err_cnt),
4523 [C_RX_DMA_FLAG_COR_ERR] = CNTR_ELEM("RxDmaFlagCorErr", 0, 0,
4524                         CNTR_NORMAL,
4525                         access_rx_dma_flag_cor_err_cnt),
4526 [C_RX_DMA_FLAG_UNC_ERR] = CNTR_ELEM("RxDmaFlagUncErr", 0, 0,
4527                         CNTR_NORMAL,
4528                         access_rx_dma_flag_unc_err_cnt),
4529 [C_RX_DC_SOP_EOP_PARITY_ERR] = CNTR_ELEM("RxDcSopEopParityErr", 0, 0,
4530                         CNTR_NORMAL,
4531                         access_rx_dc_sop_eop_parity_err_cnt),
4532 [C_RX_RCV_CSR_PARITY_ERR] = CNTR_ELEM("RxRcvCsrParityErr", 0, 0,
4533                         CNTR_NORMAL,
4534                         access_rx_rcv_csr_parity_err_cnt),
4535 [C_RX_RCV_QP_MAP_TABLE_COR_ERR] = CNTR_ELEM("RxRcvQpMapTableCorErr", 0, 0,
4536                         CNTR_NORMAL,
4537                         access_rx_rcv_qp_map_table_cor_err_cnt),
4538 [C_RX_RCV_QP_MAP_TABLE_UNC_ERR] = CNTR_ELEM("RxRcvQpMapTableUncErr", 0, 0,
4539                         CNTR_NORMAL,
4540                         access_rx_rcv_qp_map_table_unc_err_cnt),
4541 [C_RX_RCV_DATA_COR_ERR] = CNTR_ELEM("RxRcvDataCorErr", 0, 0,
4542                         CNTR_NORMAL,
4543                         access_rx_rcv_data_cor_err_cnt),
4544 [C_RX_RCV_DATA_UNC_ERR] = CNTR_ELEM("RxRcvDataUncErr", 0, 0,
4545                         CNTR_NORMAL,
4546                         access_rx_rcv_data_unc_err_cnt),
4547 [C_RX_RCV_HDR_COR_ERR] = CNTR_ELEM("RxRcvHdrCorErr", 0, 0,
4548                         CNTR_NORMAL,
4549                         access_rx_rcv_hdr_cor_err_cnt),
4550 [C_RX_RCV_HDR_UNC_ERR] = CNTR_ELEM("RxRcvHdrUncErr", 0, 0,
4551                         CNTR_NORMAL,
4552                         access_rx_rcv_hdr_unc_err_cnt),
4553 [C_RX_DC_INTF_PARITY_ERR] = CNTR_ELEM("RxDcIntfParityErr", 0, 0,
4554                         CNTR_NORMAL,
4555                         access_rx_dc_intf_parity_err_cnt),
4556 [C_RX_DMA_CSR_COR_ERR] = CNTR_ELEM("RxDmaCsrCorErr", 0, 0,
4557                         CNTR_NORMAL,
4558                         access_rx_dma_csr_cor_err_cnt),
4559 /* SendPioErrStatus */
4560 [C_PIO_PEC_SOP_HEAD_PARITY_ERR] = CNTR_ELEM("PioPecSopHeadParityErr", 0, 0,
4561                         CNTR_NORMAL,
4562                         access_pio_pec_sop_head_parity_err_cnt),
4563 [C_PIO_PCC_SOP_HEAD_PARITY_ERR] = CNTR_ELEM("PioPccSopHeadParityErr", 0, 0,
4564                         CNTR_NORMAL,
4565                         access_pio_pcc_sop_head_parity_err_cnt),
4566 [C_PIO_LAST_RETURNED_CNT_PARITY_ERR] = CNTR_ELEM("PioLastReturnedCntParityErr",
4567                         0, 0, CNTR_NORMAL,
4568                         access_pio_last_returned_cnt_parity_err_cnt),
4569 [C_PIO_CURRENT_FREE_CNT_PARITY_ERR] = CNTR_ELEM("PioCurrentFreeCntParityErr", 0,
4570                         0, CNTR_NORMAL,
4571                         access_pio_current_free_cnt_parity_err_cnt),
4572 [C_PIO_RSVD_31_ERR] = CNTR_ELEM("Pio Reserved 31", 0, 0,
4573                         CNTR_NORMAL,
4574                         access_pio_reserved_31_err_cnt),
4575 [C_PIO_RSVD_30_ERR] = CNTR_ELEM("Pio Reserved 30", 0, 0,
4576                         CNTR_NORMAL,
4577                         access_pio_reserved_30_err_cnt),
4578 [C_PIO_PPMC_SOP_LEN_ERR] = CNTR_ELEM("PioPpmcSopLenErr", 0, 0,
4579                         CNTR_NORMAL,
4580                         access_pio_ppmc_sop_len_err_cnt),
4581 [C_PIO_PPMC_BQC_MEM_PARITY_ERR] = CNTR_ELEM("PioPpmcBqcMemParityErr", 0, 0,
4582                         CNTR_NORMAL,
4583                         access_pio_ppmc_bqc_mem_parity_err_cnt),
4584 [C_PIO_VL_FIFO_PARITY_ERR] = CNTR_ELEM("PioVlFifoParityErr", 0, 0,
4585                         CNTR_NORMAL,
4586                         access_pio_vl_fifo_parity_err_cnt),
4587 [C_PIO_VLF_SOP_PARITY_ERR] = CNTR_ELEM("PioVlfSopParityErr", 0, 0,
4588                         CNTR_NORMAL,
4589                         access_pio_vlf_sop_parity_err_cnt),
4590 [C_PIO_VLF_V1_LEN_PARITY_ERR] = CNTR_ELEM("PioVlfVlLenParityErr", 0, 0,
4591                         CNTR_NORMAL,
4592                         access_pio_vlf_v1_len_parity_err_cnt),
4593 [C_PIO_BLOCK_QW_COUNT_PARITY_ERR] = CNTR_ELEM("PioBlockQwCountParityErr", 0, 0,
4594                         CNTR_NORMAL,
4595                         access_pio_block_qw_count_parity_err_cnt),
4596 [C_PIO_WRITE_QW_VALID_PARITY_ERR] = CNTR_ELEM("PioWriteQwValidParityErr", 0, 0,
4597                         CNTR_NORMAL,
4598                         access_pio_write_qw_valid_parity_err_cnt),
4599 [C_PIO_STATE_MACHINE_ERR] = CNTR_ELEM("PioStateMachineErr", 0, 0,
4600                         CNTR_NORMAL,
4601                         access_pio_state_machine_err_cnt),
4602 [C_PIO_WRITE_DATA_PARITY_ERR] = CNTR_ELEM("PioWriteDataParityErr", 0, 0,
4603                         CNTR_NORMAL,
4604                         access_pio_write_data_parity_err_cnt),
4605 [C_PIO_HOST_ADDR_MEM_COR_ERR] = CNTR_ELEM("PioHostAddrMemCorErr", 0, 0,
4606                         CNTR_NORMAL,
4607                         access_pio_host_addr_mem_cor_err_cnt),
4608 [C_PIO_HOST_ADDR_MEM_UNC_ERR] = CNTR_ELEM("PioHostAddrMemUncErr", 0, 0,
4609                         CNTR_NORMAL,
4610                         access_pio_host_addr_mem_unc_err_cnt),
4611 [C_PIO_PKT_EVICT_SM_OR_ARM_SM_ERR] = CNTR_ELEM("PioPktEvictSmOrArbSmErr", 0, 0,
4612                         CNTR_NORMAL,
4613                         access_pio_pkt_evict_sm_or_arb_sm_err_cnt),
4614 [C_PIO_INIT_SM_IN_ERR] = CNTR_ELEM("PioInitSmInErr", 0, 0,
4615                         CNTR_NORMAL,
4616                         access_pio_init_sm_in_err_cnt),
4617 [C_PIO_PPMC_PBL_FIFO_ERR] = CNTR_ELEM("PioPpmcPblFifoErr", 0, 0,
4618                         CNTR_NORMAL,
4619                         access_pio_ppmc_pbl_fifo_err_cnt),
4620 [C_PIO_CREDIT_RET_FIFO_PARITY_ERR] = CNTR_ELEM("PioCreditRetFifoParityErr", 0,
4621                         0, CNTR_NORMAL,
4622                         access_pio_credit_ret_fifo_parity_err_cnt),
4623 [C_PIO_V1_LEN_MEM_BANK1_COR_ERR] = CNTR_ELEM("PioVlLenMemBank1CorErr", 0, 0,
4624                         CNTR_NORMAL,
4625                         access_pio_v1_len_mem_bank1_cor_err_cnt),
4626 [C_PIO_V1_LEN_MEM_BANK0_COR_ERR] = CNTR_ELEM("PioVlLenMemBank0CorErr", 0, 0,
4627                         CNTR_NORMAL,
4628                         access_pio_v1_len_mem_bank0_cor_err_cnt),
4629 [C_PIO_V1_LEN_MEM_BANK1_UNC_ERR] = CNTR_ELEM("PioVlLenMemBank1UncErr", 0, 0,
4630                         CNTR_NORMAL,
4631                         access_pio_v1_len_mem_bank1_unc_err_cnt),
4632 [C_PIO_V1_LEN_MEM_BANK0_UNC_ERR] = CNTR_ELEM("PioVlLenMemBank0UncErr", 0, 0,
4633                         CNTR_NORMAL,
4634                         access_pio_v1_len_mem_bank0_unc_err_cnt),
4635 [C_PIO_SM_PKT_RESET_PARITY_ERR] = CNTR_ELEM("PioSmPktResetParityErr", 0, 0,
4636                         CNTR_NORMAL,
4637                         access_pio_sm_pkt_reset_parity_err_cnt),
4638 [C_PIO_PKT_EVICT_FIFO_PARITY_ERR] = CNTR_ELEM("PioPktEvictFifoParityErr", 0, 0,
4639                         CNTR_NORMAL,
4640                         access_pio_pkt_evict_fifo_parity_err_cnt),
4641 [C_PIO_SBRDCTRL_CRREL_FIFO_PARITY_ERR] = CNTR_ELEM(
4642                         "PioSbrdctrlCrrelFifoParityErr", 0, 0,
4643                         CNTR_NORMAL,
4644                         access_pio_sbrdctrl_crrel_fifo_parity_err_cnt),
4645 [C_PIO_SBRDCTL_CRREL_PARITY_ERR] = CNTR_ELEM("PioSbrdctlCrrelParityErr", 0, 0,
4646                         CNTR_NORMAL,
4647                         access_pio_sbrdctl_crrel_parity_err_cnt),
4648 [C_PIO_PEC_FIFO_PARITY_ERR] = CNTR_ELEM("PioPecFifoParityErr", 0, 0,
4649                         CNTR_NORMAL,
4650                         access_pio_pec_fifo_parity_err_cnt),
4651 [C_PIO_PCC_FIFO_PARITY_ERR] = CNTR_ELEM("PioPccFifoParityErr", 0, 0,
4652                         CNTR_NORMAL,
4653                         access_pio_pcc_fifo_parity_err_cnt),
4654 [C_PIO_SB_MEM_FIFO1_ERR] = CNTR_ELEM("PioSbMemFifo1Err", 0, 0,
4655                         CNTR_NORMAL,
4656                         access_pio_sb_mem_fifo1_err_cnt),
4657 [C_PIO_SB_MEM_FIFO0_ERR] = CNTR_ELEM("PioSbMemFifo0Err", 0, 0,
4658                         CNTR_NORMAL,
4659                         access_pio_sb_mem_fifo0_err_cnt),
4660 [C_PIO_CSR_PARITY_ERR] = CNTR_ELEM("PioCsrParityErr", 0, 0,
4661                         CNTR_NORMAL,
4662                         access_pio_csr_parity_err_cnt),
4663 [C_PIO_WRITE_ADDR_PARITY_ERR] = CNTR_ELEM("PioWriteAddrParityErr", 0, 0,
4664                         CNTR_NORMAL,
4665                         access_pio_write_addr_parity_err_cnt),
4666 [C_PIO_WRITE_BAD_CTXT_ERR] = CNTR_ELEM("PioWriteBadCtxtErr", 0, 0,
4667                         CNTR_NORMAL,
4668                         access_pio_write_bad_ctxt_err_cnt),
4669 /* SendDmaErrStatus */
4670 [C_SDMA_PCIE_REQ_TRACKING_COR_ERR] = CNTR_ELEM("SDmaPcieReqTrackingCorErr", 0,
4671                         0, CNTR_NORMAL,
4672                         access_sdma_pcie_req_tracking_cor_err_cnt),
4673 [C_SDMA_PCIE_REQ_TRACKING_UNC_ERR] = CNTR_ELEM("SDmaPcieReqTrackingUncErr", 0,
4674                         0, CNTR_NORMAL,
4675                         access_sdma_pcie_req_tracking_unc_err_cnt),
4676 [C_SDMA_CSR_PARITY_ERR] = CNTR_ELEM("SDmaCsrParityErr", 0, 0,
4677                         CNTR_NORMAL,
4678                         access_sdma_csr_parity_err_cnt),
4679 [C_SDMA_RPY_TAG_ERR] = CNTR_ELEM("SDmaRpyTagErr", 0, 0,
4680                         CNTR_NORMAL,
4681                         access_sdma_rpy_tag_err_cnt),
4682 /* SendEgressErrStatus */
4683 [C_TX_READ_PIO_MEMORY_CSR_UNC_ERR] = CNTR_ELEM("TxReadPioMemoryCsrUncErr", 0, 0,
4684                         CNTR_NORMAL,
4685                         access_tx_read_pio_memory_csr_unc_err_cnt),
4686 [C_TX_READ_SDMA_MEMORY_CSR_UNC_ERR] = CNTR_ELEM("TxReadSdmaMemoryCsrUncErr", 0,
4687                         0, CNTR_NORMAL,
4688                         access_tx_read_sdma_memory_csr_err_cnt),
4689 [C_TX_EGRESS_FIFO_COR_ERR] = CNTR_ELEM("TxEgressFifoCorErr", 0, 0,
4690                         CNTR_NORMAL,
4691                         access_tx_egress_fifo_cor_err_cnt),
4692 [C_TX_READ_PIO_MEMORY_COR_ERR] = CNTR_ELEM("TxReadPioMemoryCorErr", 0, 0,
4693                         CNTR_NORMAL,
4694                         access_tx_read_pio_memory_cor_err_cnt),
4695 [C_TX_READ_SDMA_MEMORY_COR_ERR] = CNTR_ELEM("TxReadSdmaMemoryCorErr", 0, 0,
4696                         CNTR_NORMAL,
4697                         access_tx_read_sdma_memory_cor_err_cnt),
4698 [C_TX_SB_HDR_COR_ERR] = CNTR_ELEM("TxSbHdrCorErr", 0, 0,
4699                         CNTR_NORMAL,
4700                         access_tx_sb_hdr_cor_err_cnt),
4701 [C_TX_CREDIT_OVERRUN_ERR] = CNTR_ELEM("TxCreditOverrunErr", 0, 0,
4702                         CNTR_NORMAL,
4703                         access_tx_credit_overrun_err_cnt),
4704 [C_TX_LAUNCH_FIFO8_COR_ERR] = CNTR_ELEM("TxLaunchFifo8CorErr", 0, 0,
4705                         CNTR_NORMAL,
4706                         access_tx_launch_fifo8_cor_err_cnt),
4707 [C_TX_LAUNCH_FIFO7_COR_ERR] = CNTR_ELEM("TxLaunchFifo7CorErr", 0, 0,
4708                         CNTR_NORMAL,
4709                         access_tx_launch_fifo7_cor_err_cnt),
4710 [C_TX_LAUNCH_FIFO6_COR_ERR] = CNTR_ELEM("TxLaunchFifo6CorErr", 0, 0,
4711                         CNTR_NORMAL,
4712                         access_tx_launch_fifo6_cor_err_cnt),
4713 [C_TX_LAUNCH_FIFO5_COR_ERR] = CNTR_ELEM("TxLaunchFifo5CorErr", 0, 0,
4714                         CNTR_NORMAL,
4715                         access_tx_launch_fifo5_cor_err_cnt),
4716 [C_TX_LAUNCH_FIFO4_COR_ERR] = CNTR_ELEM("TxLaunchFifo4CorErr", 0, 0,
4717                         CNTR_NORMAL,
4718                         access_tx_launch_fifo4_cor_err_cnt),
4719 [C_TX_LAUNCH_FIFO3_COR_ERR] = CNTR_ELEM("TxLaunchFifo3CorErr", 0, 0,
4720                         CNTR_NORMAL,
4721                         access_tx_launch_fifo3_cor_err_cnt),
4722 [C_TX_LAUNCH_FIFO2_COR_ERR] = CNTR_ELEM("TxLaunchFifo2CorErr", 0, 0,
4723                         CNTR_NORMAL,
4724                         access_tx_launch_fifo2_cor_err_cnt),
4725 [C_TX_LAUNCH_FIFO1_COR_ERR] = CNTR_ELEM("TxLaunchFifo1CorErr", 0, 0,
4726                         CNTR_NORMAL,
4727                         access_tx_launch_fifo1_cor_err_cnt),
4728 [C_TX_LAUNCH_FIFO0_COR_ERR] = CNTR_ELEM("TxLaunchFifo0CorErr", 0, 0,
4729                         CNTR_NORMAL,
4730                         access_tx_launch_fifo0_cor_err_cnt),
4731 [C_TX_CREDIT_RETURN_VL_ERR] = CNTR_ELEM("TxCreditReturnVLErr", 0, 0,
4732                         CNTR_NORMAL,
4733                         access_tx_credit_return_vl_err_cnt),
4734 [C_TX_HCRC_INSERTION_ERR] = CNTR_ELEM("TxHcrcInsertionErr", 0, 0,
4735                         CNTR_NORMAL,
4736                         access_tx_hcrc_insertion_err_cnt),
4737 [C_TX_EGRESS_FIFI_UNC_ERR] = CNTR_ELEM("TxEgressFifoUncErr", 0, 0,
4738                         CNTR_NORMAL,
4739                         access_tx_egress_fifo_unc_err_cnt),
4740 [C_TX_READ_PIO_MEMORY_UNC_ERR] = CNTR_ELEM("TxReadPioMemoryUncErr", 0, 0,
4741                         CNTR_NORMAL,
4742                         access_tx_read_pio_memory_unc_err_cnt),
4743 [C_TX_READ_SDMA_MEMORY_UNC_ERR] = CNTR_ELEM("TxReadSdmaMemoryUncErr", 0, 0,
4744                         CNTR_NORMAL,
4745                         access_tx_read_sdma_memory_unc_err_cnt),
4746 [C_TX_SB_HDR_UNC_ERR] = CNTR_ELEM("TxSbHdrUncErr", 0, 0,
4747                         CNTR_NORMAL,
4748                         access_tx_sb_hdr_unc_err_cnt),
4749 [C_TX_CREDIT_RETURN_PARITY_ERR] = CNTR_ELEM("TxCreditReturnParityErr", 0, 0,
4750                         CNTR_NORMAL,
4751                         access_tx_credit_return_partiy_err_cnt),
4752 [C_TX_LAUNCH_FIFO8_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo8UncOrParityErr",
4753                         0, 0, CNTR_NORMAL,
4754                         access_tx_launch_fifo8_unc_or_parity_err_cnt),
4755 [C_TX_LAUNCH_FIFO7_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo7UncOrParityErr",
4756                         0, 0, CNTR_NORMAL,
4757                         access_tx_launch_fifo7_unc_or_parity_err_cnt),
4758 [C_TX_LAUNCH_FIFO6_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo6UncOrParityErr",
4759                         0, 0, CNTR_NORMAL,
4760                         access_tx_launch_fifo6_unc_or_parity_err_cnt),
4761 [C_TX_LAUNCH_FIFO5_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo5UncOrParityErr",
4762                         0, 0, CNTR_NORMAL,
4763                         access_tx_launch_fifo5_unc_or_parity_err_cnt),
4764 [C_TX_LAUNCH_FIFO4_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo4UncOrParityErr",
4765                         0, 0, CNTR_NORMAL,
4766                         access_tx_launch_fifo4_unc_or_parity_err_cnt),
4767 [C_TX_LAUNCH_FIFO3_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo3UncOrParityErr",
4768                         0, 0, CNTR_NORMAL,
4769                         access_tx_launch_fifo3_unc_or_parity_err_cnt),
4770 [C_TX_LAUNCH_FIFO2_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo2UncOrParityErr",
4771                         0, 0, CNTR_NORMAL,
4772                         access_tx_launch_fifo2_unc_or_parity_err_cnt),
4773 [C_TX_LAUNCH_FIFO1_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo1UncOrParityErr",
4774                         0, 0, CNTR_NORMAL,
4775                         access_tx_launch_fifo1_unc_or_parity_err_cnt),
4776 [C_TX_LAUNCH_FIFO0_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo0UncOrParityErr",
4777                         0, 0, CNTR_NORMAL,
4778                         access_tx_launch_fifo0_unc_or_parity_err_cnt),
4779 [C_TX_SDMA15_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma15DisallowedPacketErr",
4780                         0, 0, CNTR_NORMAL,
4781                         access_tx_sdma15_disallowed_packet_err_cnt),
4782 [C_TX_SDMA14_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma14DisallowedPacketErr",
4783                         0, 0, CNTR_NORMAL,
4784                         access_tx_sdma14_disallowed_packet_err_cnt),
4785 [C_TX_SDMA13_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma13DisallowedPacketErr",
4786                         0, 0, CNTR_NORMAL,
4787                         access_tx_sdma13_disallowed_packet_err_cnt),
4788 [C_TX_SDMA12_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma12DisallowedPacketErr",
4789                         0, 0, CNTR_NORMAL,
4790                         access_tx_sdma12_disallowed_packet_err_cnt),
4791 [C_TX_SDMA11_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma11DisallowedPacketErr",
4792                         0, 0, CNTR_NORMAL,
4793                         access_tx_sdma11_disallowed_packet_err_cnt),
4794 [C_TX_SDMA10_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma10DisallowedPacketErr",
4795                         0, 0, CNTR_NORMAL,
4796                         access_tx_sdma10_disallowed_packet_err_cnt),
4797 [C_TX_SDMA9_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma9DisallowedPacketErr",
4798                         0, 0, CNTR_NORMAL,
4799                         access_tx_sdma9_disallowed_packet_err_cnt),
4800 [C_TX_SDMA8_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma8DisallowedPacketErr",
4801                         0, 0, CNTR_NORMAL,
4802                         access_tx_sdma8_disallowed_packet_err_cnt),
4803 [C_TX_SDMA7_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma7DisallowedPacketErr",
4804                         0, 0, CNTR_NORMAL,
4805                         access_tx_sdma7_disallowed_packet_err_cnt),
4806 [C_TX_SDMA6_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma6DisallowedPacketErr",
4807                         0, 0, CNTR_NORMAL,
4808                         access_tx_sdma6_disallowed_packet_err_cnt),
4809 [C_TX_SDMA5_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma5DisallowedPacketErr",
4810                         0, 0, CNTR_NORMAL,
4811                         access_tx_sdma5_disallowed_packet_err_cnt),
4812 [C_TX_SDMA4_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma4DisallowedPacketErr",
4813                         0, 0, CNTR_NORMAL,
4814                         access_tx_sdma4_disallowed_packet_err_cnt),
4815 [C_TX_SDMA3_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma3DisallowedPacketErr",
4816                         0, 0, CNTR_NORMAL,
4817                         access_tx_sdma3_disallowed_packet_err_cnt),
4818 [C_TX_SDMA2_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma2DisallowedPacketErr",
4819                         0, 0, CNTR_NORMAL,
4820                         access_tx_sdma2_disallowed_packet_err_cnt),
4821 [C_TX_SDMA1_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma1DisallowedPacketErr",
4822                         0, 0, CNTR_NORMAL,
4823                         access_tx_sdma1_disallowed_packet_err_cnt),
4824 [C_TX_SDMA0_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma0DisallowedPacketErr",
4825                         0, 0, CNTR_NORMAL,
4826                         access_tx_sdma0_disallowed_packet_err_cnt),
4827 [C_TX_CONFIG_PARITY_ERR] = CNTR_ELEM("TxConfigParityErr", 0, 0,
4828                         CNTR_NORMAL,
4829                         access_tx_config_parity_err_cnt),
4830 [C_TX_SBRD_CTL_CSR_PARITY_ERR] = CNTR_ELEM("TxSbrdCtlCsrParityErr", 0, 0,
4831                         CNTR_NORMAL,
4832                         access_tx_sbrd_ctl_csr_parity_err_cnt),
4833 [C_TX_LAUNCH_CSR_PARITY_ERR] = CNTR_ELEM("TxLaunchCsrParityErr", 0, 0,
4834                         CNTR_NORMAL,
4835                         access_tx_launch_csr_parity_err_cnt),
4836 [C_TX_ILLEGAL_CL_ERR] = CNTR_ELEM("TxIllegalVLErr", 0, 0,
4837                         CNTR_NORMAL,
4838                         access_tx_illegal_vl_err_cnt),
4839 [C_TX_SBRD_CTL_STATE_MACHINE_PARITY_ERR] = CNTR_ELEM(
4840                         "TxSbrdCtlStateMachineParityErr", 0, 0,
4841                         CNTR_NORMAL,
4842                         access_tx_sbrd_ctl_state_machine_parity_err_cnt),
4843 [C_TX_RESERVED_10] = CNTR_ELEM("Tx Egress Reserved 10", 0, 0,
4844                         CNTR_NORMAL,
4845                         access_egress_reserved_10_err_cnt),
4846 [C_TX_RESERVED_9] = CNTR_ELEM("Tx Egress Reserved 9", 0, 0,
4847                         CNTR_NORMAL,
4848                         access_egress_reserved_9_err_cnt),
4849 [C_TX_SDMA_LAUNCH_INTF_PARITY_ERR] = CNTR_ELEM("TxSdmaLaunchIntfParityErr",
4850                         0, 0, CNTR_NORMAL,
4851                         access_tx_sdma_launch_intf_parity_err_cnt),
4852 [C_TX_PIO_LAUNCH_INTF_PARITY_ERR] = CNTR_ELEM("TxPioLaunchIntfParityErr", 0, 0,
4853                         CNTR_NORMAL,
4854                         access_tx_pio_launch_intf_parity_err_cnt),
4855 [C_TX_RESERVED_6] = CNTR_ELEM("Tx Egress Reserved 6", 0, 0,
4856                         CNTR_NORMAL,
4857                         access_egress_reserved_6_err_cnt),
4858 [C_TX_INCORRECT_LINK_STATE_ERR] = CNTR_ELEM("TxIncorrectLinkStateErr", 0, 0,
4859                         CNTR_NORMAL,
4860                         access_tx_incorrect_link_state_err_cnt),
4861 [C_TX_LINK_DOWN_ERR] = CNTR_ELEM("TxLinkdownErr", 0, 0,
4862                         CNTR_NORMAL,
4863                         access_tx_linkdown_err_cnt),
4864 [C_TX_EGRESS_FIFO_UNDERRUN_OR_PARITY_ERR] = CNTR_ELEM(
4865                         "EgressFifoUnderrunOrParityErr", 0, 0,
4866                         CNTR_NORMAL,
4867                         access_tx_egress_fifi_underrun_or_parity_err_cnt),
4868 [C_TX_RESERVED_2] = CNTR_ELEM("Tx Egress Reserved 2", 0, 0,
4869                         CNTR_NORMAL,
4870                         access_egress_reserved_2_err_cnt),
4871 [C_TX_PKT_INTEGRITY_MEM_UNC_ERR] = CNTR_ELEM("TxPktIntegrityMemUncErr", 0, 0,
4872                         CNTR_NORMAL,
4873                         access_tx_pkt_integrity_mem_unc_err_cnt),
4874 [C_TX_PKT_INTEGRITY_MEM_COR_ERR] = CNTR_ELEM("TxPktIntegrityMemCorErr", 0, 0,
4875                         CNTR_NORMAL,
4876                         access_tx_pkt_integrity_mem_cor_err_cnt),
4877 /* SendErrStatus */
4878 [C_SEND_CSR_WRITE_BAD_ADDR_ERR] = CNTR_ELEM("SendCsrWriteBadAddrErr", 0, 0,
4879                         CNTR_NORMAL,
4880                         access_send_csr_write_bad_addr_err_cnt),
4881 [C_SEND_CSR_READ_BAD_ADD_ERR] = CNTR_ELEM("SendCsrReadBadAddrErr", 0, 0,
4882                         CNTR_NORMAL,
4883                         access_send_csr_read_bad_addr_err_cnt),
4884 [C_SEND_CSR_PARITY_ERR] = CNTR_ELEM("SendCsrParityErr", 0, 0,
4885                         CNTR_NORMAL,
4886                         access_send_csr_parity_cnt),
4887 /* SendCtxtErrStatus */
4888 [C_PIO_WRITE_OUT_OF_BOUNDS_ERR] = CNTR_ELEM("PioWriteOutOfBoundsErr", 0, 0,
4889                         CNTR_NORMAL,
4890                         access_pio_write_out_of_bounds_err_cnt),
4891 [C_PIO_WRITE_OVERFLOW_ERR] = CNTR_ELEM("PioWriteOverflowErr", 0, 0,
4892                         CNTR_NORMAL,
4893                         access_pio_write_overflow_err_cnt),
4894 [C_PIO_WRITE_CROSSES_BOUNDARY_ERR] = CNTR_ELEM("PioWriteCrossesBoundaryErr",
4895                         0, 0, CNTR_NORMAL,
4896                         access_pio_write_crosses_boundary_err_cnt),
4897 [C_PIO_DISALLOWED_PACKET_ERR] = CNTR_ELEM("PioDisallowedPacketErr", 0, 0,
4898                         CNTR_NORMAL,
4899                         access_pio_disallowed_packet_err_cnt),
4900 [C_PIO_INCONSISTENT_SOP_ERR] = CNTR_ELEM("PioInconsistentSopErr", 0, 0,
4901                         CNTR_NORMAL,
4902                         access_pio_inconsistent_sop_err_cnt),
4903 /* SendDmaEngErrStatus */
4904 [C_SDMA_HEADER_REQUEST_FIFO_COR_ERR] = CNTR_ELEM("SDmaHeaderRequestFifoCorErr",
4905                         0, 0, CNTR_NORMAL,
4906                         access_sdma_header_request_fifo_cor_err_cnt),
4907 [C_SDMA_HEADER_STORAGE_COR_ERR] = CNTR_ELEM("SDmaHeaderStorageCorErr", 0, 0,
4908                         CNTR_NORMAL,
4909                         access_sdma_header_storage_cor_err_cnt),
4910 [C_SDMA_PACKET_TRACKING_COR_ERR] = CNTR_ELEM("SDmaPacketTrackingCorErr", 0, 0,
4911                         CNTR_NORMAL,
4912                         access_sdma_packet_tracking_cor_err_cnt),
4913 [C_SDMA_ASSEMBLY_COR_ERR] = CNTR_ELEM("SDmaAssemblyCorErr", 0, 0,
4914                         CNTR_NORMAL,
4915                         access_sdma_assembly_cor_err_cnt),
4916 [C_SDMA_DESC_TABLE_COR_ERR] = CNTR_ELEM("SDmaDescTableCorErr", 0, 0,
4917                         CNTR_NORMAL,
4918                         access_sdma_desc_table_cor_err_cnt),
4919 [C_SDMA_HEADER_REQUEST_FIFO_UNC_ERR] = CNTR_ELEM("SDmaHeaderRequestFifoUncErr",
4920                         0, 0, CNTR_NORMAL,
4921                         access_sdma_header_request_fifo_unc_err_cnt),
4922 [C_SDMA_HEADER_STORAGE_UNC_ERR] = CNTR_ELEM("SDmaHeaderStorageUncErr", 0, 0,
4923                         CNTR_NORMAL,
4924                         access_sdma_header_storage_unc_err_cnt),
4925 [C_SDMA_PACKET_TRACKING_UNC_ERR] = CNTR_ELEM("SDmaPacketTrackingUncErr", 0, 0,
4926                         CNTR_NORMAL,
4927                         access_sdma_packet_tracking_unc_err_cnt),
4928 [C_SDMA_ASSEMBLY_UNC_ERR] = CNTR_ELEM("SDmaAssemblyUncErr", 0, 0,
4929                         CNTR_NORMAL,
4930                         access_sdma_assembly_unc_err_cnt),
4931 [C_SDMA_DESC_TABLE_UNC_ERR] = CNTR_ELEM("SDmaDescTableUncErr", 0, 0,
4932                         CNTR_NORMAL,
4933                         access_sdma_desc_table_unc_err_cnt),
4934 [C_SDMA_TIMEOUT_ERR] = CNTR_ELEM("SDmaTimeoutErr", 0, 0,
4935                         CNTR_NORMAL,
4936                         access_sdma_timeout_err_cnt),
4937 [C_SDMA_HEADER_LENGTH_ERR] = CNTR_ELEM("SDmaHeaderLengthErr", 0, 0,
4938                         CNTR_NORMAL,
4939                         access_sdma_header_length_err_cnt),
4940 [C_SDMA_HEADER_ADDRESS_ERR] = CNTR_ELEM("SDmaHeaderAddressErr", 0, 0,
4941                         CNTR_NORMAL,
4942                         access_sdma_header_address_err_cnt),
4943 [C_SDMA_HEADER_SELECT_ERR] = CNTR_ELEM("SDmaHeaderSelectErr", 0, 0,
4944                         CNTR_NORMAL,
4945                         access_sdma_header_select_err_cnt),
4946 [C_SMDA_RESERVED_9] = CNTR_ELEM("SDma Reserved 9", 0, 0,
4947                         CNTR_NORMAL,
4948                         access_sdma_reserved_9_err_cnt),
4949 [C_SDMA_PACKET_DESC_OVERFLOW_ERR] = CNTR_ELEM("SDmaPacketDescOverflowErr", 0, 0,
4950                         CNTR_NORMAL,
4951                         access_sdma_packet_desc_overflow_err_cnt),
4952 [C_SDMA_LENGTH_MISMATCH_ERR] = CNTR_ELEM("SDmaLengthMismatchErr", 0, 0,
4953                         CNTR_NORMAL,
4954                         access_sdma_length_mismatch_err_cnt),
4955 [C_SDMA_HALT_ERR] = CNTR_ELEM("SDmaHaltErr", 0, 0,
4956                         CNTR_NORMAL,
4957                         access_sdma_halt_err_cnt),
4958 [C_SDMA_MEM_READ_ERR] = CNTR_ELEM("SDmaMemReadErr", 0, 0,
4959                         CNTR_NORMAL,
4960                         access_sdma_mem_read_err_cnt),
4961 [C_SDMA_FIRST_DESC_ERR] = CNTR_ELEM("SDmaFirstDescErr", 0, 0,
4962                         CNTR_NORMAL,
4963                         access_sdma_first_desc_err_cnt),
4964 [C_SDMA_TAIL_OUT_OF_BOUNDS_ERR] = CNTR_ELEM("SDmaTailOutOfBoundsErr", 0, 0,
4965                         CNTR_NORMAL,
4966                         access_sdma_tail_out_of_bounds_err_cnt),
4967 [C_SDMA_TOO_LONG_ERR] = CNTR_ELEM("SDmaTooLongErr", 0, 0,
4968                         CNTR_NORMAL,
4969                         access_sdma_too_long_err_cnt),
4970 [C_SDMA_GEN_MISMATCH_ERR] = CNTR_ELEM("SDmaGenMismatchErr", 0, 0,
4971                         CNTR_NORMAL,
4972                         access_sdma_gen_mismatch_err_cnt),
4973 [C_SDMA_WRONG_DW_ERR] = CNTR_ELEM("SDmaWrongDwErr", 0, 0,
4974                         CNTR_NORMAL,
4975                         access_sdma_wrong_dw_err_cnt),
4976 };
4977
4978 static struct cntr_entry port_cntrs[PORT_CNTR_LAST] = {
4979 [C_TX_UNSUP_VL] = TXE32_PORT_CNTR_ELEM(TxUnVLErr, SEND_UNSUP_VL_ERR_CNT,
4980                         CNTR_NORMAL),
4981 [C_TX_INVAL_LEN] = TXE32_PORT_CNTR_ELEM(TxInvalLen, SEND_LEN_ERR_CNT,
4982                         CNTR_NORMAL),
4983 [C_TX_MM_LEN_ERR] = TXE32_PORT_CNTR_ELEM(TxMMLenErr, SEND_MAX_MIN_LEN_ERR_CNT,
4984                         CNTR_NORMAL),
4985 [C_TX_UNDERRUN] = TXE32_PORT_CNTR_ELEM(TxUnderrun, SEND_UNDERRUN_CNT,
4986                         CNTR_NORMAL),
4987 [C_TX_FLOW_STALL] = TXE32_PORT_CNTR_ELEM(TxFlowStall, SEND_FLOW_STALL_CNT,
4988                         CNTR_NORMAL),
4989 [C_TX_DROPPED] = TXE32_PORT_CNTR_ELEM(TxDropped, SEND_DROPPED_PKT_CNT,
4990                         CNTR_NORMAL),
4991 [C_TX_HDR_ERR] = TXE32_PORT_CNTR_ELEM(TxHdrErr, SEND_HEADERS_ERR_CNT,
4992                         CNTR_NORMAL),
4993 [C_TX_PKT] = TXE64_PORT_CNTR_ELEM(TxPkt, SEND_DATA_PKT_CNT, CNTR_NORMAL),
4994 [C_TX_WORDS] = TXE64_PORT_CNTR_ELEM(TxWords, SEND_DWORD_CNT, CNTR_NORMAL),
4995 [C_TX_WAIT] = TXE64_PORT_CNTR_ELEM(TxWait, SEND_WAIT_CNT, CNTR_SYNTH),
4996 [C_TX_FLIT_VL] = TXE64_PORT_CNTR_ELEM(TxFlitVL, SEND_DATA_VL0_CNT,
4997                                       CNTR_SYNTH | CNTR_VL),
4998 [C_TX_PKT_VL] = TXE64_PORT_CNTR_ELEM(TxPktVL, SEND_DATA_PKT_VL0_CNT,
4999                                      CNTR_SYNTH | CNTR_VL),
5000 [C_TX_WAIT_VL] = TXE64_PORT_CNTR_ELEM(TxWaitVL, SEND_WAIT_VL0_CNT,
5001                                       CNTR_SYNTH | CNTR_VL),
5002 [C_RX_PKT] = RXE64_PORT_CNTR_ELEM(RxPkt, RCV_DATA_PKT_CNT, CNTR_NORMAL),
5003 [C_RX_WORDS] = RXE64_PORT_CNTR_ELEM(RxWords, RCV_DWORD_CNT, CNTR_NORMAL),
5004 [C_SW_LINK_DOWN] = CNTR_ELEM("SwLinkDown", 0, 0, CNTR_SYNTH | CNTR_32BIT,
5005                              access_sw_link_dn_cnt),
5006 [C_SW_LINK_UP] = CNTR_ELEM("SwLinkUp", 0, 0, CNTR_SYNTH | CNTR_32BIT,
5007                            access_sw_link_up_cnt),
5008 [C_SW_UNKNOWN_FRAME] = CNTR_ELEM("UnknownFrame", 0, 0, CNTR_NORMAL,
5009                                  access_sw_unknown_frame_cnt),
5010 [C_SW_XMIT_DSCD] = CNTR_ELEM("XmitDscd", 0, 0, CNTR_SYNTH | CNTR_32BIT,
5011                              access_sw_xmit_discards),
5012 [C_SW_XMIT_DSCD_VL] = CNTR_ELEM("XmitDscdVl", 0, 0,
5013                                 CNTR_SYNTH | CNTR_32BIT | CNTR_VL,
5014                                 access_sw_xmit_discards),
5015 [C_SW_XMIT_CSTR_ERR] = CNTR_ELEM("XmitCstrErr", 0, 0, CNTR_SYNTH,
5016                                  access_xmit_constraint_errs),
5017 [C_SW_RCV_CSTR_ERR] = CNTR_ELEM("RcvCstrErr", 0, 0, CNTR_SYNTH,
5018                                 access_rcv_constraint_errs),
5019 [C_SW_IBP_LOOP_PKTS] = SW_IBP_CNTR(LoopPkts, loop_pkts),
5020 [C_SW_IBP_RC_RESENDS] = SW_IBP_CNTR(RcResend, rc_resends),
5021 [C_SW_IBP_RNR_NAKS] = SW_IBP_CNTR(RnrNak, rnr_naks),
5022 [C_SW_IBP_OTHER_NAKS] = SW_IBP_CNTR(OtherNak, other_naks),
5023 [C_SW_IBP_RC_TIMEOUTS] = SW_IBP_CNTR(RcTimeOut, rc_timeouts),
5024 [C_SW_IBP_PKT_DROPS] = SW_IBP_CNTR(PktDrop, pkt_drops),
5025 [C_SW_IBP_DMA_WAIT] = SW_IBP_CNTR(DmaWait, dmawait),
5026 [C_SW_IBP_RC_SEQNAK] = SW_IBP_CNTR(RcSeqNak, rc_seqnak),
5027 [C_SW_IBP_RC_DUPREQ] = SW_IBP_CNTR(RcDupRew, rc_dupreq),
5028 [C_SW_IBP_RDMA_SEQ] = SW_IBP_CNTR(RdmaSeq, rdma_seq),
5029 [C_SW_IBP_UNALIGNED] = SW_IBP_CNTR(Unaligned, unaligned),
5030 [C_SW_IBP_SEQ_NAK] = SW_IBP_CNTR(SeqNak, seq_naks),
5031 [C_SW_CPU_RC_ACKS] = CNTR_ELEM("RcAcks", 0, 0, CNTR_NORMAL,
5032                                access_sw_cpu_rc_acks),
5033 [C_SW_CPU_RC_QACKS] = CNTR_ELEM("RcQacks", 0, 0, CNTR_NORMAL,
5034                                 access_sw_cpu_rc_qacks),
5035 [C_SW_CPU_RC_DELAYED_COMP] = CNTR_ELEM("RcDelayComp", 0, 0, CNTR_NORMAL,
5036                                        access_sw_cpu_rc_delayed_comp),
5037 [OVR_LBL(0)] = OVR_ELM(0), [OVR_LBL(1)] = OVR_ELM(1),
5038 [OVR_LBL(2)] = OVR_ELM(2), [OVR_LBL(3)] = OVR_ELM(3),
5039 [OVR_LBL(4)] = OVR_ELM(4), [OVR_LBL(5)] = OVR_ELM(5),
5040 [OVR_LBL(6)] = OVR_ELM(6), [OVR_LBL(7)] = OVR_ELM(7),
5041 [OVR_LBL(8)] = OVR_ELM(8), [OVR_LBL(9)] = OVR_ELM(9),
5042 [OVR_LBL(10)] = OVR_ELM(10), [OVR_LBL(11)] = OVR_ELM(11),
5043 [OVR_LBL(12)] = OVR_ELM(12), [OVR_LBL(13)] = OVR_ELM(13),
5044 [OVR_LBL(14)] = OVR_ELM(14), [OVR_LBL(15)] = OVR_ELM(15),
5045 [OVR_LBL(16)] = OVR_ELM(16), [OVR_LBL(17)] = OVR_ELM(17),
5046 [OVR_LBL(18)] = OVR_ELM(18), [OVR_LBL(19)] = OVR_ELM(19),
5047 [OVR_LBL(20)] = OVR_ELM(20), [OVR_LBL(21)] = OVR_ELM(21),
5048 [OVR_LBL(22)] = OVR_ELM(22), [OVR_LBL(23)] = OVR_ELM(23),
5049 [OVR_LBL(24)] = OVR_ELM(24), [OVR_LBL(25)] = OVR_ELM(25),
5050 [OVR_LBL(26)] = OVR_ELM(26), [OVR_LBL(27)] = OVR_ELM(27),
5051 [OVR_LBL(28)] = OVR_ELM(28), [OVR_LBL(29)] = OVR_ELM(29),
5052 [OVR_LBL(30)] = OVR_ELM(30), [OVR_LBL(31)] = OVR_ELM(31),
5053 [OVR_LBL(32)] = OVR_ELM(32), [OVR_LBL(33)] = OVR_ELM(33),
5054 [OVR_LBL(34)] = OVR_ELM(34), [OVR_LBL(35)] = OVR_ELM(35),
5055 [OVR_LBL(36)] = OVR_ELM(36), [OVR_LBL(37)] = OVR_ELM(37),
5056 [OVR_LBL(38)] = OVR_ELM(38), [OVR_LBL(39)] = OVR_ELM(39),
5057 [OVR_LBL(40)] = OVR_ELM(40), [OVR_LBL(41)] = OVR_ELM(41),
5058 [OVR_LBL(42)] = OVR_ELM(42), [OVR_LBL(43)] = OVR_ELM(43),
5059 [OVR_LBL(44)] = OVR_ELM(44), [OVR_LBL(45)] = OVR_ELM(45),
5060 [OVR_LBL(46)] = OVR_ELM(46), [OVR_LBL(47)] = OVR_ELM(47),
5061 [OVR_LBL(48)] = OVR_ELM(48), [OVR_LBL(49)] = OVR_ELM(49),
5062 [OVR_LBL(50)] = OVR_ELM(50), [OVR_LBL(51)] = OVR_ELM(51),
5063 [OVR_LBL(52)] = OVR_ELM(52), [OVR_LBL(53)] = OVR_ELM(53),
5064 [OVR_LBL(54)] = OVR_ELM(54), [OVR_LBL(55)] = OVR_ELM(55),
5065 [OVR_LBL(56)] = OVR_ELM(56), [OVR_LBL(57)] = OVR_ELM(57),
5066 [OVR_LBL(58)] = OVR_ELM(58), [OVR_LBL(59)] = OVR_ELM(59),
5067 [OVR_LBL(60)] = OVR_ELM(60), [OVR_LBL(61)] = OVR_ELM(61),
5068 [OVR_LBL(62)] = OVR_ELM(62), [OVR_LBL(63)] = OVR_ELM(63),
5069 [OVR_LBL(64)] = OVR_ELM(64), [OVR_LBL(65)] = OVR_ELM(65),
5070 [OVR_LBL(66)] = OVR_ELM(66), [OVR_LBL(67)] = OVR_ELM(67),
5071 [OVR_LBL(68)] = OVR_ELM(68), [OVR_LBL(69)] = OVR_ELM(69),
5072 [OVR_LBL(70)] = OVR_ELM(70), [OVR_LBL(71)] = OVR_ELM(71),
5073 [OVR_LBL(72)] = OVR_ELM(72), [OVR_LBL(73)] = OVR_ELM(73),
5074 [OVR_LBL(74)] = OVR_ELM(74), [OVR_LBL(75)] = OVR_ELM(75),
5075 [OVR_LBL(76)] = OVR_ELM(76), [OVR_LBL(77)] = OVR_ELM(77),
5076 [OVR_LBL(78)] = OVR_ELM(78), [OVR_LBL(79)] = OVR_ELM(79),
5077 [OVR_LBL(80)] = OVR_ELM(80), [OVR_LBL(81)] = OVR_ELM(81),
5078 [OVR_LBL(82)] = OVR_ELM(82), [OVR_LBL(83)] = OVR_ELM(83),
5079 [OVR_LBL(84)] = OVR_ELM(84), [OVR_LBL(85)] = OVR_ELM(85),
5080 [OVR_LBL(86)] = OVR_ELM(86), [OVR_LBL(87)] = OVR_ELM(87),
5081 [OVR_LBL(88)] = OVR_ELM(88), [OVR_LBL(89)] = OVR_ELM(89),
5082 [OVR_LBL(90)] = OVR_ELM(90), [OVR_LBL(91)] = OVR_ELM(91),
5083 [OVR_LBL(92)] = OVR_ELM(92), [OVR_LBL(93)] = OVR_ELM(93),
5084 [OVR_LBL(94)] = OVR_ELM(94), [OVR_LBL(95)] = OVR_ELM(95),
5085 [OVR_LBL(96)] = OVR_ELM(96), [OVR_LBL(97)] = OVR_ELM(97),
5086 [OVR_LBL(98)] = OVR_ELM(98), [OVR_LBL(99)] = OVR_ELM(99),
5087 [OVR_LBL(100)] = OVR_ELM(100), [OVR_LBL(101)] = OVR_ELM(101),
5088 [OVR_LBL(102)] = OVR_ELM(102), [OVR_LBL(103)] = OVR_ELM(103),
5089 [OVR_LBL(104)] = OVR_ELM(104), [OVR_LBL(105)] = OVR_ELM(105),
5090 [OVR_LBL(106)] = OVR_ELM(106), [OVR_LBL(107)] = OVR_ELM(107),
5091 [OVR_LBL(108)] = OVR_ELM(108), [OVR_LBL(109)] = OVR_ELM(109),
5092 [OVR_LBL(110)] = OVR_ELM(110), [OVR_LBL(111)] = OVR_ELM(111),
5093 [OVR_LBL(112)] = OVR_ELM(112), [OVR_LBL(113)] = OVR_ELM(113),
5094 [OVR_LBL(114)] = OVR_ELM(114), [OVR_LBL(115)] = OVR_ELM(115),
5095 [OVR_LBL(116)] = OVR_ELM(116), [OVR_LBL(117)] = OVR_ELM(117),
5096 [OVR_LBL(118)] = OVR_ELM(118), [OVR_LBL(119)] = OVR_ELM(119),
5097 [OVR_LBL(120)] = OVR_ELM(120), [OVR_LBL(121)] = OVR_ELM(121),
5098 [OVR_LBL(122)] = OVR_ELM(122), [OVR_LBL(123)] = OVR_ELM(123),
5099 [OVR_LBL(124)] = OVR_ELM(124), [OVR_LBL(125)] = OVR_ELM(125),
5100 [OVR_LBL(126)] = OVR_ELM(126), [OVR_LBL(127)] = OVR_ELM(127),
5101 [OVR_LBL(128)] = OVR_ELM(128), [OVR_LBL(129)] = OVR_ELM(129),
5102 [OVR_LBL(130)] = OVR_ELM(130), [OVR_LBL(131)] = OVR_ELM(131),
5103 [OVR_LBL(132)] = OVR_ELM(132), [OVR_LBL(133)] = OVR_ELM(133),
5104 [OVR_LBL(134)] = OVR_ELM(134), [OVR_LBL(135)] = OVR_ELM(135),
5105 [OVR_LBL(136)] = OVR_ELM(136), [OVR_LBL(137)] = OVR_ELM(137),
5106 [OVR_LBL(138)] = OVR_ELM(138), [OVR_LBL(139)] = OVR_ELM(139),
5107 [OVR_LBL(140)] = OVR_ELM(140), [OVR_LBL(141)] = OVR_ELM(141),
5108 [OVR_LBL(142)] = OVR_ELM(142), [OVR_LBL(143)] = OVR_ELM(143),
5109 [OVR_LBL(144)] = OVR_ELM(144), [OVR_LBL(145)] = OVR_ELM(145),
5110 [OVR_LBL(146)] = OVR_ELM(146), [OVR_LBL(147)] = OVR_ELM(147),
5111 [OVR_LBL(148)] = OVR_ELM(148), [OVR_LBL(149)] = OVR_ELM(149),
5112 [OVR_LBL(150)] = OVR_ELM(150), [OVR_LBL(151)] = OVR_ELM(151),
5113 [OVR_LBL(152)] = OVR_ELM(152), [OVR_LBL(153)] = OVR_ELM(153),
5114 [OVR_LBL(154)] = OVR_ELM(154), [OVR_LBL(155)] = OVR_ELM(155),
5115 [OVR_LBL(156)] = OVR_ELM(156), [OVR_LBL(157)] = OVR_ELM(157),
5116 [OVR_LBL(158)] = OVR_ELM(158), [OVR_LBL(159)] = OVR_ELM(159),
5117 };
5118
5119 /* ======================================================================== */
5120
5121 /* return true if this is chip revision revision a */
5122 int is_ax(struct hfi1_devdata *dd)
5123 {
5124         u8 chip_rev_minor =
5125                 dd->revision >> CCE_REVISION_CHIP_REV_MINOR_SHIFT
5126                         & CCE_REVISION_CHIP_REV_MINOR_MASK;
5127         return (chip_rev_minor & 0xf0) == 0;
5128 }
5129
5130 /* return true if this is chip revision revision b */
5131 int is_bx(struct hfi1_devdata *dd)
5132 {
5133         u8 chip_rev_minor =
5134                 dd->revision >> CCE_REVISION_CHIP_REV_MINOR_SHIFT
5135                         & CCE_REVISION_CHIP_REV_MINOR_MASK;
5136         return (chip_rev_minor & 0xF0) == 0x10;
5137 }
5138
5139 /*
5140  * Append string s to buffer buf.  Arguments curp and len are the current
5141  * position and remaining length, respectively.
5142  *
5143  * return 0 on success, 1 on out of room
5144  */
5145 static int append_str(char *buf, char **curp, int *lenp, const char *s)
5146 {
5147         char *p = *curp;
5148         int len = *lenp;
5149         int result = 0; /* success */
5150         char c;
5151
5152         /* add a comma, if first in the buffer */
5153         if (p != buf) {
5154                 if (len == 0) {
5155                         result = 1; /* out of room */
5156                         goto done;
5157                 }
5158                 *p++ = ',';
5159                 len--;
5160         }
5161
5162         /* copy the string */
5163         while ((c = *s++) != 0) {
5164                 if (len == 0) {
5165                         result = 1; /* out of room */
5166                         goto done;
5167                 }
5168                 *p++ = c;
5169                 len--;
5170         }
5171
5172 done:
5173         /* write return values */
5174         *curp = p;
5175         *lenp = len;
5176
5177         return result;
5178 }
5179
5180 /*
5181  * Using the given flag table, print a comma separated string into
5182  * the buffer.  End in '*' if the buffer is too short.
5183  */
5184 static char *flag_string(char *buf, int buf_len, u64 flags,
5185                          struct flag_table *table, int table_size)
5186 {
5187         char extra[32];
5188         char *p = buf;
5189         int len = buf_len;
5190         int no_room = 0;
5191         int i;
5192
5193         /* make sure there is at least 2 so we can form "*" */
5194         if (len < 2)
5195                 return "";
5196
5197         len--;  /* leave room for a nul */
5198         for (i = 0; i < table_size; i++) {
5199                 if (flags & table[i].flag) {
5200                         no_room = append_str(buf, &p, &len, table[i].str);
5201                         if (no_room)
5202                                 break;
5203                         flags &= ~table[i].flag;
5204                 }
5205         }
5206
5207         /* any undocumented bits left? */
5208         if (!no_room && flags) {
5209                 snprintf(extra, sizeof(extra), "bits 0x%llx", flags);
5210                 no_room = append_str(buf, &p, &len, extra);
5211         }
5212
5213         /* add * if ran out of room */
5214         if (no_room) {
5215                 /* may need to back up to add space for a '*' */
5216                 if (len == 0)
5217                         --p;
5218                 *p++ = '*';
5219         }
5220
5221         /* add final nul - space already allocated above */
5222         *p = 0;
5223         return buf;
5224 }
5225
5226 /* first 8 CCE error interrupt source names */
5227 static const char * const cce_misc_names[] = {
5228         "CceErrInt",            /* 0 */
5229         "RxeErrInt",            /* 1 */
5230         "MiscErrInt",           /* 2 */
5231         "Reserved3",            /* 3 */
5232         "PioErrInt",            /* 4 */
5233         "SDmaErrInt",           /* 5 */
5234         "EgressErrInt",         /* 6 */
5235         "TxeErrInt"             /* 7 */
5236 };
5237
5238 /*
5239  * Return the miscellaneous error interrupt name.
5240  */
5241 static char *is_misc_err_name(char *buf, size_t bsize, unsigned int source)
5242 {
5243         if (source < ARRAY_SIZE(cce_misc_names))
5244                 strncpy(buf, cce_misc_names[source], bsize);
5245         else
5246                 snprintf(buf, bsize, "Reserved%u",
5247                          source + IS_GENERAL_ERR_START);
5248
5249         return buf;
5250 }
5251
5252 /*
5253  * Return the SDMA engine error interrupt name.
5254  */
5255 static char *is_sdma_eng_err_name(char *buf, size_t bsize, unsigned int source)
5256 {
5257         snprintf(buf, bsize, "SDmaEngErrInt%u", source);
5258         return buf;
5259 }
5260
5261 /*
5262  * Return the send context error interrupt name.
5263  */
5264 static char *is_sendctxt_err_name(char *buf, size_t bsize, unsigned int source)
5265 {
5266         snprintf(buf, bsize, "SendCtxtErrInt%u", source);
5267         return buf;
5268 }
5269
5270 static const char * const various_names[] = {
5271         "PbcInt",
5272         "GpioAssertInt",
5273         "Qsfp1Int",
5274         "Qsfp2Int",
5275         "TCritInt"
5276 };
5277
5278 /*
5279  * Return the various interrupt name.
5280  */
5281 static char *is_various_name(char *buf, size_t bsize, unsigned int source)
5282 {
5283         if (source < ARRAY_SIZE(various_names))
5284                 strncpy(buf, various_names[source], bsize);
5285         else
5286                 snprintf(buf, bsize, "Reserved%u", source + IS_VARIOUS_START);
5287         return buf;
5288 }
5289
5290 /*
5291  * Return the DC interrupt name.
5292  */
5293 static char *is_dc_name(char *buf, size_t bsize, unsigned int source)
5294 {
5295         static const char * const dc_int_names[] = {
5296                 "common",
5297                 "lcb",
5298                 "8051",
5299                 "lbm"   /* local block merge */
5300         };
5301
5302         if (source < ARRAY_SIZE(dc_int_names))
5303                 snprintf(buf, bsize, "dc_%s_int", dc_int_names[source]);
5304         else
5305                 snprintf(buf, bsize, "DCInt%u", source);
5306         return buf;
5307 }
5308
5309 static const char * const sdma_int_names[] = {
5310         "SDmaInt",
5311         "SdmaIdleInt",
5312         "SdmaProgressInt",
5313 };
5314
5315 /*
5316  * Return the SDMA engine interrupt name.
5317  */
5318 static char *is_sdma_eng_name(char *buf, size_t bsize, unsigned int source)
5319 {
5320         /* what interrupt */
5321         unsigned int what  = source / TXE_NUM_SDMA_ENGINES;
5322         /* which engine */
5323         unsigned int which = source % TXE_NUM_SDMA_ENGINES;
5324
5325         if (likely(what < 3))
5326                 snprintf(buf, bsize, "%s%u", sdma_int_names[what], which);
5327         else
5328                 snprintf(buf, bsize, "Invalid SDMA interrupt %u", source);
5329         return buf;
5330 }
5331
5332 /*
5333  * Return the receive available interrupt name.
5334  */
5335 static char *is_rcv_avail_name(char *buf, size_t bsize, unsigned int source)
5336 {
5337         snprintf(buf, bsize, "RcvAvailInt%u", source);
5338         return buf;
5339 }
5340
5341 /*
5342  * Return the receive urgent interrupt name.
5343  */
5344 static char *is_rcv_urgent_name(char *buf, size_t bsize, unsigned int source)
5345 {
5346         snprintf(buf, bsize, "RcvUrgentInt%u", source);
5347         return buf;
5348 }
5349
5350 /*
5351  * Return the send credit interrupt name.
5352  */
5353 static char *is_send_credit_name(char *buf, size_t bsize, unsigned int source)
5354 {
5355         snprintf(buf, bsize, "SendCreditInt%u", source);
5356         return buf;
5357 }
5358
5359 /*
5360  * Return the reserved interrupt name.
5361  */
5362 static char *is_reserved_name(char *buf, size_t bsize, unsigned int source)
5363 {
5364         snprintf(buf, bsize, "Reserved%u", source + IS_RESERVED_START);
5365         return buf;
5366 }
5367
5368 static char *cce_err_status_string(char *buf, int buf_len, u64 flags)
5369 {
5370         return flag_string(buf, buf_len, flags,
5371                            cce_err_status_flags,
5372                            ARRAY_SIZE(cce_err_status_flags));
5373 }
5374
5375 static char *rxe_err_status_string(char *buf, int buf_len, u64 flags)
5376 {
5377         return flag_string(buf, buf_len, flags,
5378                            rxe_err_status_flags,
5379                            ARRAY_SIZE(rxe_err_status_flags));
5380 }
5381
5382 static char *misc_err_status_string(char *buf, int buf_len, u64 flags)
5383 {
5384         return flag_string(buf, buf_len, flags, misc_err_status_flags,
5385                            ARRAY_SIZE(misc_err_status_flags));
5386 }
5387
5388 static char *pio_err_status_string(char *buf, int buf_len, u64 flags)
5389 {
5390         return flag_string(buf, buf_len, flags,
5391                            pio_err_status_flags,
5392                            ARRAY_SIZE(pio_err_status_flags));
5393 }
5394
5395 static char *sdma_err_status_string(char *buf, int buf_len, u64 flags)
5396 {
5397         return flag_string(buf, buf_len, flags,
5398                            sdma_err_status_flags,
5399                            ARRAY_SIZE(sdma_err_status_flags));
5400 }
5401
5402 static char *egress_err_status_string(char *buf, int buf_len, u64 flags)
5403 {
5404         return flag_string(buf, buf_len, flags,
5405                            egress_err_status_flags,
5406                            ARRAY_SIZE(egress_err_status_flags));
5407 }
5408
5409 static char *egress_err_info_string(char *buf, int buf_len, u64 flags)
5410 {
5411         return flag_string(buf, buf_len, flags,
5412                            egress_err_info_flags,
5413                            ARRAY_SIZE(egress_err_info_flags));
5414 }
5415
5416 static char *send_err_status_string(char *buf, int buf_len, u64 flags)
5417 {
5418         return flag_string(buf, buf_len, flags,
5419                            send_err_status_flags,
5420                            ARRAY_SIZE(send_err_status_flags));
5421 }
5422
5423 static void handle_cce_err(struct hfi1_devdata *dd, u32 unused, u64 reg)
5424 {
5425         char buf[96];
5426         int i = 0;
5427
5428         /*
5429          * For most these errors, there is nothing that can be done except
5430          * report or record it.
5431          */
5432         dd_dev_info(dd, "CCE Error: %s\n",
5433                     cce_err_status_string(buf, sizeof(buf), reg));
5434
5435         if ((reg & CCE_ERR_STATUS_CCE_CLI2_ASYNC_FIFO_PARITY_ERR_SMASK) &&
5436             is_ax(dd) && (dd->icode != ICODE_FUNCTIONAL_SIMULATOR)) {
5437                 /* this error requires a manual drop into SPC freeze mode */
5438                 /* then a fix up */
5439                 start_freeze_handling(dd->pport, FREEZE_SELF);
5440         }
5441
5442         for (i = 0; i < NUM_CCE_ERR_STATUS_COUNTERS; i++) {
5443                 if (reg & (1ull << i)) {
5444                         incr_cntr64(&dd->cce_err_status_cnt[i]);
5445                         /* maintain a counter over all cce_err_status errors */
5446                         incr_cntr64(&dd->sw_cce_err_status_aggregate);
5447                 }
5448         }
5449 }
5450
5451 /*
5452  * Check counters for receive errors that do not have an interrupt
5453  * associated with them.
5454  */
5455 #define RCVERR_CHECK_TIME 10
5456 static void update_rcverr_timer(unsigned long opaque)
5457 {
5458         struct hfi1_devdata *dd = (struct hfi1_devdata *)opaque;
5459         struct hfi1_pportdata *ppd = dd->pport;
5460         u32 cur_ovfl_cnt = read_dev_cntr(dd, C_RCV_OVF, CNTR_INVALID_VL);
5461
5462         if (dd->rcv_ovfl_cnt < cur_ovfl_cnt &&
5463             ppd->port_error_action & OPA_PI_MASK_EX_BUFFER_OVERRUN) {
5464                 dd_dev_info(dd, "%s: PortErrorAction bounce\n", __func__);
5465                 set_link_down_reason(
5466                 ppd, OPA_LINKDOWN_REASON_EXCESSIVE_BUFFER_OVERRUN, 0,
5467                 OPA_LINKDOWN_REASON_EXCESSIVE_BUFFER_OVERRUN);
5468                 queue_work(ppd->hfi1_wq, &ppd->link_bounce_work);
5469         }
5470         dd->rcv_ovfl_cnt = (u32)cur_ovfl_cnt;
5471
5472         mod_timer(&dd->rcverr_timer, jiffies + HZ * RCVERR_CHECK_TIME);
5473 }
5474
5475 static int init_rcverr(struct hfi1_devdata *dd)
5476 {
5477         setup_timer(&dd->rcverr_timer, update_rcverr_timer, (unsigned long)dd);
5478         /* Assume the hardware counter has been reset */
5479         dd->rcv_ovfl_cnt = 0;
5480         return mod_timer(&dd->rcverr_timer, jiffies + HZ * RCVERR_CHECK_TIME);
5481 }
5482
5483 static void free_rcverr(struct hfi1_devdata *dd)
5484 {
5485         if (dd->rcverr_timer.data)
5486                 del_timer_sync(&dd->rcverr_timer);
5487         dd->rcverr_timer.data = 0;
5488 }
5489
5490 static void handle_rxe_err(struct hfi1_devdata *dd, u32 unused, u64 reg)
5491 {
5492         char buf[96];
5493         int i = 0;
5494
5495         dd_dev_info(dd, "Receive Error: %s\n",
5496                     rxe_err_status_string(buf, sizeof(buf), reg));
5497
5498         if (reg & ALL_RXE_FREEZE_ERR) {
5499                 int flags = 0;
5500
5501                 /*
5502                  * Freeze mode recovery is disabled for the errors
5503                  * in RXE_FREEZE_ABORT_MASK
5504                  */
5505                 if (is_ax(dd) && (reg & RXE_FREEZE_ABORT_MASK))
5506                         flags = FREEZE_ABORT;
5507
5508                 start_freeze_handling(dd->pport, flags);
5509         }
5510
5511         for (i = 0; i < NUM_RCV_ERR_STATUS_COUNTERS; i++) {
5512                 if (reg & (1ull << i))
5513                         incr_cntr64(&dd->rcv_err_status_cnt[i]);
5514         }
5515 }
5516
5517 static void handle_misc_err(struct hfi1_devdata *dd, u32 unused, u64 reg)
5518 {
5519         char buf[96];
5520         int i = 0;
5521
5522         dd_dev_info(dd, "Misc Error: %s",
5523                     misc_err_status_string(buf, sizeof(buf), reg));
5524         for (i = 0; i < NUM_MISC_ERR_STATUS_COUNTERS; i++) {
5525                 if (reg & (1ull << i))
5526                         incr_cntr64(&dd->misc_err_status_cnt[i]);
5527         }
5528 }
5529
5530 static void handle_pio_err(struct hfi1_devdata *dd, u32 unused, u64 reg)
5531 {
5532         char buf[96];
5533         int i = 0;
5534
5535         dd_dev_info(dd, "PIO Error: %s\n",
5536                     pio_err_status_string(buf, sizeof(buf), reg));
5537
5538         if (reg & ALL_PIO_FREEZE_ERR)
5539                 start_freeze_handling(dd->pport, 0);
5540
5541         for (i = 0; i < NUM_SEND_PIO_ERR_STATUS_COUNTERS; i++) {
5542                 if (reg & (1ull << i))
5543                         incr_cntr64(&dd->send_pio_err_status_cnt[i]);
5544         }
5545 }
5546
5547 static void handle_sdma_err(struct hfi1_devdata *dd, u32 unused, u64 reg)
5548 {
5549         char buf[96];
5550         int i = 0;
5551
5552         dd_dev_info(dd, "SDMA Error: %s\n",
5553                     sdma_err_status_string(buf, sizeof(buf), reg));
5554
5555         if (reg & ALL_SDMA_FREEZE_ERR)
5556                 start_freeze_handling(dd->pport, 0);
5557
5558         for (i = 0; i < NUM_SEND_DMA_ERR_STATUS_COUNTERS; i++) {
5559                 if (reg & (1ull << i))
5560                         incr_cntr64(&dd->send_dma_err_status_cnt[i]);
5561         }
5562 }
5563
5564 static inline void __count_port_discards(struct hfi1_pportdata *ppd)
5565 {
5566         incr_cntr64(&ppd->port_xmit_discards);
5567 }
5568
5569 static void count_port_inactive(struct hfi1_devdata *dd)
5570 {
5571         __count_port_discards(dd->pport);
5572 }
5573
5574 /*
5575  * We have had a "disallowed packet" error during egress. Determine the
5576  * integrity check which failed, and update relevant error counter, etc.
5577  *
5578  * Note that the SEND_EGRESS_ERR_INFO register has only a single
5579  * bit of state per integrity check, and so we can miss the reason for an
5580  * egress error if more than one packet fails the same integrity check
5581  * since we cleared the corresponding bit in SEND_EGRESS_ERR_INFO.
5582  */
5583 static void handle_send_egress_err_info(struct hfi1_devdata *dd,
5584                                         int vl)
5585 {
5586         struct hfi1_pportdata *ppd = dd->pport;
5587         u64 src = read_csr(dd, SEND_EGRESS_ERR_SOURCE); /* read first */
5588         u64 info = read_csr(dd, SEND_EGRESS_ERR_INFO);
5589         char buf[96];
5590
5591         /* clear down all observed info as quickly as possible after read */
5592         write_csr(dd, SEND_EGRESS_ERR_INFO, info);
5593
5594         dd_dev_info(dd,
5595                     "Egress Error Info: 0x%llx, %s Egress Error Src 0x%llx\n",
5596                     info, egress_err_info_string(buf, sizeof(buf), info), src);
5597
5598         /* Eventually add other counters for each bit */
5599         if (info & PORT_DISCARD_EGRESS_ERRS) {
5600                 int weight, i;
5601
5602                 /*
5603                  * Count all applicable bits as individual errors and
5604                  * attribute them to the packet that triggered this handler.
5605                  * This may not be completely accurate due to limitations
5606                  * on the available hardware error information.  There is
5607                  * a single information register and any number of error
5608                  * packets may have occurred and contributed to it before
5609                  * this routine is called.  This means that:
5610                  * a) If multiple packets with the same error occur before
5611                  *    this routine is called, earlier packets are missed.
5612                  *    There is only a single bit for each error type.
5613                  * b) Errors may not be attributed to the correct VL.
5614                  *    The driver is attributing all bits in the info register
5615                  *    to the packet that triggered this call, but bits
5616                  *    could be an accumulation of different packets with
5617                  *    different VLs.
5618                  * c) A single error packet may have multiple counts attached
5619                  *    to it.  There is no way for the driver to know if
5620                  *    multiple bits set in the info register are due to a
5621                  *    single packet or multiple packets.  The driver assumes
5622                  *    multiple packets.
5623                  */
5624                 weight = hweight64(info & PORT_DISCARD_EGRESS_ERRS);
5625                 for (i = 0; i < weight; i++) {
5626                         __count_port_discards(ppd);
5627                         if (vl >= 0 && vl < TXE_NUM_DATA_VL)
5628                                 incr_cntr64(&ppd->port_xmit_discards_vl[vl]);
5629                         else if (vl == 15)
5630                                 incr_cntr64(&ppd->port_xmit_discards_vl
5631                                             [C_VL_15]);
5632                 }
5633         }
5634 }
5635
5636 /*
5637  * Input value is a bit position within the SEND_EGRESS_ERR_STATUS
5638  * register. Does it represent a 'port inactive' error?
5639  */
5640 static inline int port_inactive_err(u64 posn)
5641 {
5642         return (posn >= SEES(TX_LINKDOWN) &&
5643                 posn <= SEES(TX_INCORRECT_LINK_STATE));
5644 }
5645
5646 /*
5647  * Input value is a bit position within the SEND_EGRESS_ERR_STATUS
5648  * register. Does it represent a 'disallowed packet' error?
5649  */
5650 static inline int disallowed_pkt_err(int posn)
5651 {
5652         return (posn >= SEES(TX_SDMA0_DISALLOWED_PACKET) &&
5653                 posn <= SEES(TX_SDMA15_DISALLOWED_PACKET));
5654 }
5655
5656 /*
5657  * Input value is a bit position of one of the SDMA engine disallowed
5658  * packet errors.  Return which engine.  Use of this must be guarded by
5659  * disallowed_pkt_err().
5660  */
5661 static inline int disallowed_pkt_engine(int posn)
5662 {
5663         return posn - SEES(TX_SDMA0_DISALLOWED_PACKET);
5664 }
5665
5666 /*
5667  * Translate an SDMA engine to a VL.  Return -1 if the tranlation cannot
5668  * be done.
5669  */
5670 static int engine_to_vl(struct hfi1_devdata *dd, int engine)
5671 {
5672         struct sdma_vl_map *m;
5673         int vl;
5674
5675         /* range check */
5676         if (engine < 0 || engine >= TXE_NUM_SDMA_ENGINES)
5677                 return -1;
5678
5679         rcu_read_lock();
5680         m = rcu_dereference(dd->sdma_map);
5681         vl = m->engine_to_vl[engine];
5682         rcu_read_unlock();
5683
5684         return vl;
5685 }
5686
5687 /*
5688  * Translate the send context (sofware index) into a VL.  Return -1 if the
5689  * translation cannot be done.
5690  */
5691 static int sc_to_vl(struct hfi1_devdata *dd, int sw_index)
5692 {
5693         struct send_context_info *sci;
5694         struct send_context *sc;
5695         int i;
5696
5697         sci = &dd->send_contexts[sw_index];
5698
5699         /* there is no information for user (PSM) and ack contexts */
5700         if ((sci->type != SC_KERNEL) && (sci->type != SC_VL15))
5701                 return -1;
5702
5703         sc = sci->sc;
5704         if (!sc)
5705                 return -1;
5706         if (dd->vld[15].sc == sc)
5707                 return 15;
5708         for (i = 0; i < num_vls; i++)
5709                 if (dd->vld[i].sc == sc)
5710                         return i;
5711
5712         return -1;
5713 }
5714
5715 static void handle_egress_err(struct hfi1_devdata *dd, u32 unused, u64 reg)
5716 {
5717         u64 reg_copy = reg, handled = 0;
5718         char buf[96];
5719         int i = 0;
5720
5721         if (reg & ALL_TXE_EGRESS_FREEZE_ERR)
5722                 start_freeze_handling(dd->pport, 0);
5723         else if (is_ax(dd) &&
5724                  (reg & SEND_EGRESS_ERR_STATUS_TX_CREDIT_RETURN_VL_ERR_SMASK) &&
5725                  (dd->icode != ICODE_FUNCTIONAL_SIMULATOR))
5726                 start_freeze_handling(dd->pport, 0);
5727
5728         while (reg_copy) {
5729                 int posn = fls64(reg_copy);
5730                 /* fls64() returns a 1-based offset, we want it zero based */
5731                 int shift = posn - 1;
5732                 u64 mask = 1ULL << shift;
5733
5734                 if (port_inactive_err(shift)) {
5735                         count_port_inactive(dd);
5736                         handled |= mask;
5737                 } else if (disallowed_pkt_err(shift)) {
5738                         int vl = engine_to_vl(dd, disallowed_pkt_engine(shift));
5739
5740                         handle_send_egress_err_info(dd, vl);
5741                         handled |= mask;
5742                 }
5743                 reg_copy &= ~mask;
5744         }
5745
5746         reg &= ~handled;
5747
5748         if (reg)
5749                 dd_dev_info(dd, "Egress Error: %s\n",
5750                             egress_err_status_string(buf, sizeof(buf), reg));
5751
5752         for (i = 0; i < NUM_SEND_EGRESS_ERR_STATUS_COUNTERS; i++) {
5753                 if (reg & (1ull << i))
5754                         incr_cntr64(&dd->send_egress_err_status_cnt[i]);
5755         }
5756 }
5757
5758 static void handle_txe_err(struct hfi1_devdata *dd, u32 unused, u64 reg)
5759 {
5760         char buf[96];
5761         int i = 0;
5762
5763         dd_dev_info(dd, "Send Error: %s\n",
5764                     send_err_status_string(buf, sizeof(buf), reg));
5765
5766         for (i = 0; i < NUM_SEND_ERR_STATUS_COUNTERS; i++) {
5767                 if (reg & (1ull << i))
5768                         incr_cntr64(&dd->send_err_status_cnt[i]);
5769         }
5770 }
5771
5772 /*
5773  * The maximum number of times the error clear down will loop before
5774  * blocking a repeating error.  This value is arbitrary.
5775  */
5776 #define MAX_CLEAR_COUNT 20
5777
5778 /*
5779  * Clear and handle an error register.  All error interrupts are funneled
5780  * through here to have a central location to correctly handle single-
5781  * or multi-shot errors.
5782  *
5783  * For non per-context registers, call this routine with a context value
5784  * of 0 so the per-context offset is zero.
5785  *
5786  * If the handler loops too many times, assume that something is wrong
5787  * and can't be fixed, so mask the error bits.
5788  */
5789 static void interrupt_clear_down(struct hfi1_devdata *dd,
5790                                  u32 context,
5791                                  const struct err_reg_info *eri)
5792 {
5793         u64 reg;
5794         u32 count;
5795
5796         /* read in a loop until no more errors are seen */
5797         count = 0;
5798         while (1) {
5799                 reg = read_kctxt_csr(dd, context, eri->status);
5800                 if (reg == 0)
5801                         break;
5802                 write_kctxt_csr(dd, context, eri->clear, reg);
5803                 if (likely(eri->handler))
5804                         eri->handler(dd, context, reg);
5805                 count++;
5806                 if (count > MAX_CLEAR_COUNT) {
5807                         u64 mask;
5808
5809                         dd_dev_err(dd, "Repeating %s bits 0x%llx - masking\n",
5810                                    eri->desc, reg);
5811                         /*
5812                          * Read-modify-write so any other masked bits
5813                          * remain masked.
5814                          */
5815                         mask = read_kctxt_csr(dd, context, eri->mask);
5816                         mask &= ~reg;
5817                         write_kctxt_csr(dd, context, eri->mask, mask);
5818                         break;
5819                 }
5820         }
5821 }
5822
5823 /*
5824  * CCE block "misc" interrupt.  Source is < 16.
5825  */
5826 static void is_misc_err_int(struct hfi1_devdata *dd, unsigned int source)
5827 {
5828         const struct err_reg_info *eri = &misc_errs[source];
5829
5830         if (eri->handler) {
5831                 interrupt_clear_down(dd, 0, eri);
5832         } else {
5833                 dd_dev_err(dd, "Unexpected misc interrupt (%u) - reserved\n",
5834                            source);
5835         }
5836 }
5837
5838 static char *send_context_err_status_string(char *buf, int buf_len, u64 flags)
5839 {
5840         return flag_string(buf, buf_len, flags,
5841                            sc_err_status_flags,
5842                            ARRAY_SIZE(sc_err_status_flags));
5843 }
5844
5845 /*
5846  * Send context error interrupt.  Source (hw_context) is < 160.
5847  *
5848  * All send context errors cause the send context to halt.  The normal
5849  * clear-down mechanism cannot be used because we cannot clear the
5850  * error bits until several other long-running items are done first.
5851  * This is OK because with the context halted, nothing else is going
5852  * to happen on it anyway.
5853  */
5854 static void is_sendctxt_err_int(struct hfi1_devdata *dd,
5855                                 unsigned int hw_context)
5856 {
5857         struct send_context_info *sci;
5858         struct send_context *sc;
5859         char flags[96];
5860         u64 status;
5861         u32 sw_index;
5862         int i = 0;
5863         unsigned long irq_flags;
5864
5865         sw_index = dd->hw_to_sw[hw_context];
5866         if (sw_index >= dd->num_send_contexts) {
5867                 dd_dev_err(dd,
5868                            "out of range sw index %u for send context %u\n",
5869                            sw_index, hw_context);
5870                 return;
5871         }
5872         sci = &dd->send_contexts[sw_index];
5873         spin_lock_irqsave(&dd->sc_lock, irq_flags);
5874         sc = sci->sc;
5875         if (!sc) {
5876                 dd_dev_err(dd, "%s: context %u(%u): no sc?\n", __func__,
5877                            sw_index, hw_context);
5878                 spin_unlock_irqrestore(&dd->sc_lock, irq_flags);
5879                 return;
5880         }
5881
5882         /* tell the software that a halt has begun */
5883         sc_stop(sc, SCF_HALTED);
5884
5885         status = read_kctxt_csr(dd, hw_context, SEND_CTXT_ERR_STATUS);
5886
5887         dd_dev_info(dd, "Send Context %u(%u) Error: %s\n", sw_index, hw_context,
5888                     send_context_err_status_string(flags, sizeof(flags),
5889                                                    status));
5890
5891         if (status & SEND_CTXT_ERR_STATUS_PIO_DISALLOWED_PACKET_ERR_SMASK)
5892                 handle_send_egress_err_info(dd, sc_to_vl(dd, sw_index));
5893
5894         /*
5895          * Automatically restart halted kernel contexts out of interrupt
5896          * context.  User contexts must ask the driver to restart the context.
5897          */
5898         if (sc->type != SC_USER)
5899                 queue_work(dd->pport->hfi1_wq, &sc->halt_work);
5900         spin_unlock_irqrestore(&dd->sc_lock, irq_flags);
5901
5902         /*
5903          * Update the counters for the corresponding status bits.
5904          * Note that these particular counters are aggregated over all
5905          * 160 contexts.
5906          */
5907         for (i = 0; i < NUM_SEND_CTXT_ERR_STATUS_COUNTERS; i++) {
5908                 if (status & (1ull << i))
5909                         incr_cntr64(&dd->sw_ctxt_err_status_cnt[i]);
5910         }
5911 }
5912
5913 static void handle_sdma_eng_err(struct hfi1_devdata *dd,
5914                                 unsigned int source, u64 status)
5915 {
5916         struct sdma_engine *sde;
5917         int i = 0;
5918
5919         sde = &dd->per_sdma[source];
5920 #ifdef CONFIG_SDMA_VERBOSITY
5921         dd_dev_err(sde->dd, "CONFIG SDMA(%u) %s:%d %s()\n", sde->this_idx,
5922                    slashstrip(__FILE__), __LINE__, __func__);
5923         dd_dev_err(sde->dd, "CONFIG SDMA(%u) source: %u status 0x%llx\n",
5924                    sde->this_idx, source, (unsigned long long)status);
5925 #endif
5926         sde->err_cnt++;
5927         sdma_engine_error(sde, status);
5928
5929         /*
5930         * Update the counters for the corresponding status bits.
5931         * Note that these particular counters are aggregated over
5932         * all 16 DMA engines.
5933         */
5934         for (i = 0; i < NUM_SEND_DMA_ENG_ERR_STATUS_COUNTERS; i++) {
5935                 if (status & (1ull << i))
5936                         incr_cntr64(&dd->sw_send_dma_eng_err_status_cnt[i]);
5937         }
5938 }
5939
5940 /*
5941  * CCE block SDMA error interrupt.  Source is < 16.
5942  */
5943 static void is_sdma_eng_err_int(struct hfi1_devdata *dd, unsigned int source)
5944 {
5945 #ifdef CONFIG_SDMA_VERBOSITY
5946         struct sdma_engine *sde = &dd->per_sdma[source];
5947
5948         dd_dev_err(dd, "CONFIG SDMA(%u) %s:%d %s()\n", sde->this_idx,
5949                    slashstrip(__FILE__), __LINE__, __func__);
5950         dd_dev_err(dd, "CONFIG SDMA(%u) source: %u\n", sde->this_idx,
5951                    source);
5952         sdma_dumpstate(sde);
5953 #endif
5954         interrupt_clear_down(dd, source, &sdma_eng_err);
5955 }
5956
5957 /*
5958  * CCE block "various" interrupt.  Source is < 8.
5959  */
5960 static void is_various_int(struct hfi1_devdata *dd, unsigned int source)
5961 {
5962         const struct err_reg_info *eri = &various_err[source];
5963
5964         /*
5965          * TCritInt cannot go through interrupt_clear_down()
5966          * because it is not a second tier interrupt. The handler
5967          * should be called directly.
5968          */
5969         if (source == TCRIT_INT_SOURCE)
5970                 handle_temp_err(dd);
5971         else if (eri->handler)
5972                 interrupt_clear_down(dd, 0, eri);
5973         else
5974                 dd_dev_info(dd,
5975                             "%s: Unimplemented/reserved interrupt %d\n",
5976                             __func__, source);
5977 }
5978
5979 static void handle_qsfp_int(struct hfi1_devdata *dd, u32 src_ctx, u64 reg)
5980 {
5981         /* src_ctx is always zero */
5982         struct hfi1_pportdata *ppd = dd->pport;
5983         unsigned long flags;
5984         u64 qsfp_int_mgmt = (u64)(QSFP_HFI0_INT_N | QSFP_HFI0_MODPRST_N);
5985
5986         if (reg & QSFP_HFI0_MODPRST_N) {
5987                 if (!qsfp_mod_present(ppd)) {
5988                         dd_dev_info(dd, "%s: QSFP module removed\n",
5989                                     __func__);
5990
5991                         ppd->driver_link_ready = 0;
5992                         /*
5993                          * Cable removed, reset all our information about the
5994                          * cache and cable capabilities
5995                          */
5996
5997                         spin_lock_irqsave(&ppd->qsfp_info.qsfp_lock, flags);
5998                         /*
5999                          * We don't set cache_refresh_required here as we expect
6000                          * an interrupt when a cable is inserted
6001                          */
6002                         ppd->qsfp_info.cache_valid = 0;
6003                         ppd->qsfp_info.reset_needed = 0;
6004                         ppd->qsfp_info.limiting_active = 0;
6005                         spin_unlock_irqrestore(&ppd->qsfp_info.qsfp_lock,
6006                                                flags);
6007                         /* Invert the ModPresent pin now to detect plug-in */
6008                         write_csr(dd, dd->hfi1_id ? ASIC_QSFP2_INVERT :
6009                                   ASIC_QSFP1_INVERT, qsfp_int_mgmt);
6010
6011                         if ((ppd->offline_disabled_reason >
6012                           HFI1_ODR_MASK(
6013                           OPA_LINKDOWN_REASON_LOCAL_MEDIA_NOT_INSTALLED)) ||
6014                           (ppd->offline_disabled_reason ==
6015                           HFI1_ODR_MASK(OPA_LINKDOWN_REASON_NONE)))
6016                                 ppd->offline_disabled_reason =
6017                                 HFI1_ODR_MASK(
6018                                 OPA_LINKDOWN_REASON_LOCAL_MEDIA_NOT_INSTALLED);
6019
6020                         if (ppd->host_link_state == HLS_DN_POLL) {
6021                                 /*
6022                                  * The link is still in POLL. This means
6023                                  * that the normal link down processing
6024                                  * will not happen. We have to do it here
6025                                  * before turning the DC off.
6026                                  */
6027                                 queue_work(ppd->hfi1_wq, &ppd->link_down_work);
6028                         }
6029                 } else {
6030                         dd_dev_info(dd, "%s: QSFP module inserted\n",
6031                                     __func__);
6032
6033                         spin_lock_irqsave(&ppd->qsfp_info.qsfp_lock, flags);
6034                         ppd->qsfp_info.cache_valid = 0;
6035                         ppd->qsfp_info.cache_refresh_required = 1;
6036                         spin_unlock_irqrestore(&ppd->qsfp_info.qsfp_lock,
6037                                                flags);
6038
6039                         /*
6040                          * Stop inversion of ModPresent pin to detect
6041                          * removal of the cable
6042                          */
6043                         qsfp_int_mgmt &= ~(u64)QSFP_HFI0_MODPRST_N;
6044                         write_csr(dd, dd->hfi1_id ? ASIC_QSFP2_INVERT :
6045                                   ASIC_QSFP1_INVERT, qsfp_int_mgmt);
6046
6047                         ppd->offline_disabled_reason =
6048                                 HFI1_ODR_MASK(OPA_LINKDOWN_REASON_TRANSIENT);
6049                 }
6050         }
6051
6052         if (reg & QSFP_HFI0_INT_N) {
6053                 dd_dev_info(dd, "%s: Interrupt received from QSFP module\n",
6054                             __func__);
6055                 spin_lock_irqsave(&ppd->qsfp_info.qsfp_lock, flags);
6056                 ppd->qsfp_info.check_interrupt_flags = 1;
6057                 spin_unlock_irqrestore(&ppd->qsfp_info.qsfp_lock, flags);
6058         }
6059
6060         /* Schedule the QSFP work only if there is a cable attached. */
6061         if (qsfp_mod_present(ppd))
6062                 queue_work(ppd->hfi1_wq, &ppd->qsfp_info.qsfp_work);
6063 }
6064
6065 static int request_host_lcb_access(struct hfi1_devdata *dd)
6066 {
6067         int ret;
6068
6069         ret = do_8051_command(dd, HCMD_MISC,
6070                               (u64)HCMD_MISC_REQUEST_LCB_ACCESS <<
6071                               LOAD_DATA_FIELD_ID_SHIFT, NULL);
6072         if (ret != HCMD_SUCCESS) {
6073                 dd_dev_err(dd, "%s: command failed with error %d\n",
6074                            __func__, ret);
6075         }
6076         return ret == HCMD_SUCCESS ? 0 : -EBUSY;
6077 }
6078
6079 static int request_8051_lcb_access(struct hfi1_devdata *dd)
6080 {
6081         int ret;
6082
6083         ret = do_8051_command(dd, HCMD_MISC,
6084                               (u64)HCMD_MISC_GRANT_LCB_ACCESS <<
6085                               LOAD_DATA_FIELD_ID_SHIFT, NULL);
6086         if (ret != HCMD_SUCCESS) {
6087                 dd_dev_err(dd, "%s: command failed with error %d\n",
6088                            __func__, ret);
6089         }
6090         return ret == HCMD_SUCCESS ? 0 : -EBUSY;
6091 }
6092
6093 /*
6094  * Set the LCB selector - allow host access.  The DCC selector always
6095  * points to the host.
6096  */
6097 static inline void set_host_lcb_access(struct hfi1_devdata *dd)
6098 {
6099         write_csr(dd, DC_DC8051_CFG_CSR_ACCESS_SEL,
6100                   DC_DC8051_CFG_CSR_ACCESS_SEL_DCC_SMASK |
6101                   DC_DC8051_CFG_CSR_ACCESS_SEL_LCB_SMASK);
6102 }
6103
6104 /*
6105  * Clear the LCB selector - allow 8051 access.  The DCC selector always
6106  * points to the host.
6107  */
6108 static inline void set_8051_lcb_access(struct hfi1_devdata *dd)
6109 {
6110         write_csr(dd, DC_DC8051_CFG_CSR_ACCESS_SEL,
6111                   DC_DC8051_CFG_CSR_ACCESS_SEL_DCC_SMASK);
6112 }
6113
6114 /*
6115  * Acquire LCB access from the 8051.  If the host already has access,
6116  * just increment a counter.  Otherwise, inform the 8051 that the
6117  * host is taking access.
6118  *
6119  * Returns:
6120  *      0 on success
6121  *      -EBUSY if the 8051 has control and cannot be disturbed
6122  *      -errno if unable to acquire access from the 8051
6123  */
6124 int acquire_lcb_access(struct hfi1_devdata *dd, int sleep_ok)
6125 {
6126         struct hfi1_pportdata *ppd = dd->pport;
6127         int ret = 0;
6128
6129         /*
6130          * Use the host link state lock so the operation of this routine
6131          * { link state check, selector change, count increment } can occur
6132          * as a unit against a link state change.  Otherwise there is a
6133          * race between the state change and the count increment.
6134          */
6135         if (sleep_ok) {
6136                 mutex_lock(&ppd->hls_lock);
6137         } else {
6138                 while (!mutex_trylock(&ppd->hls_lock))
6139                         udelay(1);
6140         }
6141
6142         /* this access is valid only when the link is up */
6143         if (ppd->host_link_state & HLS_DOWN) {
6144                 dd_dev_info(dd, "%s: link state %s not up\n",
6145                             __func__, link_state_name(ppd->host_link_state));
6146                 ret = -EBUSY;
6147                 goto done;
6148         }
6149
6150         if (dd->lcb_access_count == 0) {
6151                 ret = request_host_lcb_access(dd);
6152                 if (ret) {
6153                         dd_dev_err(dd,
6154                                    "%s: unable to acquire LCB access, err %d\n",
6155                                    __func__, ret);
6156                         goto done;
6157                 }
6158                 set_host_lcb_access(dd);
6159         }
6160         dd->lcb_access_count++;
6161 done:
6162         mutex_unlock(&ppd->hls_lock);
6163         return ret;
6164 }
6165
6166 /*
6167  * Release LCB access by decrementing the use count.  If the count is moving
6168  * from 1 to 0, inform 8051 that it has control back.
6169  *
6170  * Returns:
6171  *      0 on success
6172  *      -errno if unable to release access to the 8051
6173  */
6174 int release_lcb_access(struct hfi1_devdata *dd, int sleep_ok)
6175 {
6176         int ret = 0;
6177
6178         /*
6179          * Use the host link state lock because the acquire needed it.
6180          * Here, we only need to keep { selector change, count decrement }
6181          * as a unit.
6182          */
6183         if (sleep_ok) {
6184                 mutex_lock(&dd->pport->hls_lock);
6185         } else {
6186                 while (!mutex_trylock(&dd->pport->hls_lock))
6187                         udelay(1);
6188         }
6189
6190         if (dd->lcb_access_count == 0) {
6191                 dd_dev_err(dd, "%s: LCB access count is zero.  Skipping.\n",
6192                            __func__);
6193                 goto done;
6194         }
6195
6196         if (dd->lcb_access_count == 1) {
6197                 set_8051_lcb_access(dd);
6198                 ret = request_8051_lcb_access(dd);
6199                 if (ret) {
6200                         dd_dev_err(dd,
6201                                    "%s: unable to release LCB access, err %d\n",
6202                                    __func__, ret);
6203                         /* restore host access if the grant didn't work */
6204                         set_host_lcb_access(dd);
6205                         goto done;
6206                 }
6207         }
6208         dd->lcb_access_count--;
6209 done:
6210         mutex_unlock(&dd->pport->hls_lock);
6211         return ret;
6212 }
6213
6214 /*
6215  * Initialize LCB access variables and state.  Called during driver load,
6216  * after most of the initialization is finished.
6217  *
6218  * The DC default is LCB access on for the host.  The driver defaults to
6219  * leaving access to the 8051.  Assign access now - this constrains the call
6220  * to this routine to be after all LCB set-up is done.  In particular, after
6221  * hf1_init_dd() -> set_up_interrupts() -> clear_all_interrupts()
6222  */
6223 static void init_lcb_access(struct hfi1_devdata *dd)
6224 {
6225         dd->lcb_access_count = 0;
6226 }
6227
6228 /*
6229  * Write a response back to a 8051 request.
6230  */
6231 static void hreq_response(struct hfi1_devdata *dd, u8 return_code, u16 rsp_data)
6232 {
6233         write_csr(dd, DC_DC8051_CFG_EXT_DEV_0,
6234                   DC_DC8051_CFG_EXT_DEV_0_COMPLETED_SMASK |
6235                   (u64)return_code <<
6236                   DC_DC8051_CFG_EXT_DEV_0_RETURN_CODE_SHIFT |
6237                   (u64)rsp_data << DC_DC8051_CFG_EXT_DEV_0_RSP_DATA_SHIFT);
6238 }
6239
6240 /*
6241  * Handle host requests from the 8051.
6242  */
6243 static void handle_8051_request(struct hfi1_pportdata *ppd)
6244 {
6245         struct hfi1_devdata *dd = ppd->dd;
6246         u64 reg;
6247         u16 data = 0;
6248         u8 type;
6249
6250         reg = read_csr(dd, DC_DC8051_CFG_EXT_DEV_1);
6251         if ((reg & DC_DC8051_CFG_EXT_DEV_1_REQ_NEW_SMASK) == 0)
6252                 return; /* no request */
6253
6254         /* zero out COMPLETED so the response is seen */
6255         write_csr(dd, DC_DC8051_CFG_EXT_DEV_0, 0);
6256
6257         /* extract request details */
6258         type = (reg >> DC_DC8051_CFG_EXT_DEV_1_REQ_TYPE_SHIFT)
6259                         & DC_DC8051_CFG_EXT_DEV_1_REQ_TYPE_MASK;
6260         data = (reg >> DC_DC8051_CFG_EXT_DEV_1_REQ_DATA_SHIFT)
6261                         & DC_DC8051_CFG_EXT_DEV_1_REQ_DATA_MASK;
6262
6263         switch (type) {
6264         case HREQ_LOAD_CONFIG:
6265         case HREQ_SAVE_CONFIG:
6266         case HREQ_READ_CONFIG:
6267         case HREQ_SET_TX_EQ_ABS:
6268         case HREQ_SET_TX_EQ_REL:
6269         case HREQ_ENABLE:
6270                 dd_dev_info(dd, "8051 request: request 0x%x not supported\n",
6271                             type);
6272                 hreq_response(dd, HREQ_NOT_SUPPORTED, 0);
6273                 break;
6274         case HREQ_CONFIG_DONE:
6275                 hreq_response(dd, HREQ_SUCCESS, 0);
6276                 break;
6277
6278         case HREQ_INTERFACE_TEST:
6279                 hreq_response(dd, HREQ_SUCCESS, data);
6280                 break;
6281         default:
6282                 dd_dev_err(dd, "8051 request: unknown request 0x%x\n", type);
6283                 hreq_response(dd, HREQ_NOT_SUPPORTED, 0);
6284                 break;
6285         }
6286 }
6287
6288 static void write_global_credit(struct hfi1_devdata *dd,
6289                                 u8 vau, u16 total, u16 shared)
6290 {
6291         write_csr(dd, SEND_CM_GLOBAL_CREDIT,
6292                   ((u64)total <<
6293                    SEND_CM_GLOBAL_CREDIT_TOTAL_CREDIT_LIMIT_SHIFT) |
6294                   ((u64)shared <<
6295                    SEND_CM_GLOBAL_CREDIT_SHARED_LIMIT_SHIFT) |
6296                   ((u64)vau << SEND_CM_GLOBAL_CREDIT_AU_SHIFT));
6297 }
6298
6299 /*
6300  * Set up initial VL15 credits of the remote.  Assumes the rest of
6301  * the CM credit registers are zero from a previous global or credit reset .
6302  */
6303 void set_up_vl15(struct hfi1_devdata *dd, u8 vau, u16 vl15buf)
6304 {
6305         /* leave shared count at zero for both global and VL15 */
6306         write_global_credit(dd, vau, vl15buf, 0);
6307
6308         write_csr(dd, SEND_CM_CREDIT_VL15, (u64)vl15buf
6309                   << SEND_CM_CREDIT_VL15_DEDICATED_LIMIT_VL_SHIFT);
6310 }
6311
6312 /*
6313  * Zero all credit details from the previous connection and
6314  * reset the CM manager's internal counters.
6315  */
6316 void reset_link_credits(struct hfi1_devdata *dd)
6317 {
6318         int i;
6319
6320         /* remove all previous VL credit limits */
6321         for (i = 0; i < TXE_NUM_DATA_VL; i++)
6322                 write_csr(dd, SEND_CM_CREDIT_VL + (8 * i), 0);
6323         write_csr(dd, SEND_CM_CREDIT_VL15, 0);
6324         write_global_credit(dd, 0, 0, 0);
6325         /* reset the CM block */
6326         pio_send_control(dd, PSC_CM_RESET);
6327 }
6328
6329 /* convert a vCU to a CU */
6330 static u32 vcu_to_cu(u8 vcu)
6331 {
6332         return 1 << vcu;
6333 }
6334
6335 /* convert a CU to a vCU */
6336 static u8 cu_to_vcu(u32 cu)
6337 {
6338         return ilog2(cu);
6339 }
6340
6341 /* convert a vAU to an AU */
6342 static u32 vau_to_au(u8 vau)
6343 {
6344         return 8 * (1 << vau);
6345 }
6346
6347 static void set_linkup_defaults(struct hfi1_pportdata *ppd)
6348 {
6349         ppd->sm_trap_qp = 0x0;
6350         ppd->sa_qp = 0x1;
6351 }
6352
6353 /*
6354  * Graceful LCB shutdown.  This leaves the LCB FIFOs in reset.
6355  */
6356 static void lcb_shutdown(struct hfi1_devdata *dd, int abort)
6357 {
6358         u64 reg;
6359
6360         /* clear lcb run: LCB_CFG_RUN.EN = 0 */
6361         write_csr(dd, DC_LCB_CFG_RUN, 0);
6362         /* set tx fifo reset: LCB_CFG_TX_FIFOS_RESET.VAL = 1 */
6363         write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET,
6364                   1ull << DC_LCB_CFG_TX_FIFOS_RESET_VAL_SHIFT);
6365         /* set dcc reset csr: DCC_CFG_RESET.{reset_lcb,reset_rx_fpe} = 1 */
6366         dd->lcb_err_en = read_csr(dd, DC_LCB_ERR_EN);
6367         reg = read_csr(dd, DCC_CFG_RESET);
6368         write_csr(dd, DCC_CFG_RESET, reg |
6369                   (1ull << DCC_CFG_RESET_RESET_LCB_SHIFT) |
6370                   (1ull << DCC_CFG_RESET_RESET_RX_FPE_SHIFT));
6371         (void)read_csr(dd, DCC_CFG_RESET); /* make sure the write completed */
6372         if (!abort) {
6373                 udelay(1);    /* must hold for the longer of 16cclks or 20ns */
6374                 write_csr(dd, DCC_CFG_RESET, reg);
6375                 write_csr(dd, DC_LCB_ERR_EN, dd->lcb_err_en);
6376         }
6377 }
6378
6379 /*
6380  * This routine should be called after the link has been transitioned to
6381  * OFFLINE (OFFLINE state has the side effect of putting the SerDes into
6382  * reset).
6383  *
6384  * The expectation is that the caller of this routine would have taken
6385  * care of properly transitioning the link into the correct state.
6386  * NOTE: the caller needs to acquire the dd->dc8051_lock lock
6387  *       before calling this function.
6388  */
6389 static void _dc_shutdown(struct hfi1_devdata *dd)
6390 {
6391         lockdep_assert_held(&dd->dc8051_lock);
6392
6393         if (dd->dc_shutdown)
6394                 return;
6395
6396         dd->dc_shutdown = 1;
6397         /* Shutdown the LCB */
6398         lcb_shutdown(dd, 1);
6399         /*
6400          * Going to OFFLINE would have causes the 8051 to put the
6401          * SerDes into reset already. Just need to shut down the 8051,
6402          * itself.
6403          */
6404         write_csr(dd, DC_DC8051_CFG_RST, 0x1);
6405 }
6406
6407 static void dc_shutdown(struct hfi1_devdata *dd)
6408 {
6409         mutex_lock(&dd->dc8051_lock);
6410         _dc_shutdown(dd);
6411         mutex_unlock(&dd->dc8051_lock);
6412 }
6413
6414 /*
6415  * Calling this after the DC has been brought out of reset should not
6416  * do any damage.
6417  * NOTE: the caller needs to acquire the dd->dc8051_lock lock
6418  *       before calling this function.
6419  */
6420 static void _dc_start(struct hfi1_devdata *dd)
6421 {
6422         lockdep_assert_held(&dd->dc8051_lock);
6423
6424         if (!dd->dc_shutdown)
6425                 return;
6426
6427         /* Take the 8051 out of reset */
6428         write_csr(dd, DC_DC8051_CFG_RST, 0ull);
6429         /* Wait until 8051 is ready */
6430         if (wait_fm_ready(dd, TIMEOUT_8051_START))
6431                 dd_dev_err(dd, "%s: timeout starting 8051 firmware\n",
6432                            __func__);
6433
6434         /* Take away reset for LCB and RX FPE (set in lcb_shutdown). */
6435         write_csr(dd, DCC_CFG_RESET, 0x10);
6436         /* lcb_shutdown() with abort=1 does not restore these */
6437         write_csr(dd, DC_LCB_ERR_EN, dd->lcb_err_en);
6438         dd->dc_shutdown = 0;
6439 }
6440
6441 static void dc_start(struct hfi1_devdata *dd)
6442 {
6443         mutex_lock(&dd->dc8051_lock);
6444         _dc_start(dd);
6445         mutex_unlock(&dd->dc8051_lock);
6446 }
6447
6448 /*
6449  * These LCB adjustments are for the Aurora SerDes core in the FPGA.
6450  */
6451 static void adjust_lcb_for_fpga_serdes(struct hfi1_devdata *dd)
6452 {
6453         u64 rx_radr, tx_radr;
6454         u32 version;
6455
6456         if (dd->icode != ICODE_FPGA_EMULATION)
6457                 return;
6458
6459         /*
6460          * These LCB defaults on emulator _s are good, nothing to do here:
6461          *      LCB_CFG_TX_FIFOS_RADR
6462          *      LCB_CFG_RX_FIFOS_RADR
6463          *      LCB_CFG_LN_DCLK
6464          *      LCB_CFG_IGNORE_LOST_RCLK
6465          */
6466         if (is_emulator_s(dd))
6467                 return;
6468         /* else this is _p */
6469
6470         version = emulator_rev(dd);
6471         if (!is_ax(dd))
6472                 version = 0x2d; /* all B0 use 0x2d or higher settings */
6473
6474         if (version <= 0x12) {
6475                 /* release 0x12 and below */
6476
6477                 /*
6478                  * LCB_CFG_RX_FIFOS_RADR.RST_VAL = 0x9
6479                  * LCB_CFG_RX_FIFOS_RADR.OK_TO_JUMP_VAL = 0x9
6480                  * LCB_CFG_RX_FIFOS_RADR.DO_NOT_JUMP_VAL = 0xa
6481                  */
6482                 rx_radr =
6483                       0xaull << DC_LCB_CFG_RX_FIFOS_RADR_DO_NOT_JUMP_VAL_SHIFT
6484                     | 0x9ull << DC_LCB_CFG_RX_FIFOS_RADR_OK_TO_JUMP_VAL_SHIFT
6485                     | 0x9ull << DC_LCB_CFG_RX_FIFOS_RADR_RST_VAL_SHIFT;
6486                 /*
6487                  * LCB_CFG_TX_FIFOS_RADR.ON_REINIT = 0 (default)
6488                  * LCB_CFG_TX_FIFOS_RADR.RST_VAL = 6
6489                  */
6490                 tx_radr = 6ull << DC_LCB_CFG_TX_FIFOS_RADR_RST_VAL_SHIFT;
6491         } else if (version <= 0x18) {
6492                 /* release 0x13 up to 0x18 */
6493                 /* LCB_CFG_RX_FIFOS_RADR = 0x988 */
6494                 rx_radr =
6495                       0x9ull << DC_LCB_CFG_RX_FIFOS_RADR_DO_NOT_JUMP_VAL_SHIFT
6496                     | 0x8ull << DC_LCB_CFG_RX_FIFOS_RADR_OK_TO_JUMP_VAL_SHIFT
6497                     | 0x8ull << DC_LCB_CFG_RX_FIFOS_RADR_RST_VAL_SHIFT;
6498                 tx_radr = 7ull << DC_LCB_CFG_TX_FIFOS_RADR_RST_VAL_SHIFT;
6499         } else if (version == 0x19) {
6500                 /* release 0x19 */
6501                 /* LCB_CFG_RX_FIFOS_RADR = 0xa99 */
6502                 rx_radr =
6503                       0xAull << DC_LCB_CFG_RX_FIFOS_RADR_DO_NOT_JUMP_VAL_SHIFT
6504                     | 0x9ull << DC_LCB_CFG_RX_FIFOS_RADR_OK_TO_JUMP_VAL_SHIFT
6505                     | 0x9ull << DC_LCB_CFG_RX_FIFOS_RADR_RST_VAL_SHIFT;
6506                 tx_radr = 3ull << DC_LCB_CFG_TX_FIFOS_RADR_RST_VAL_SHIFT;
6507         } else if (version == 0x1a) {
6508                 /* release 0x1a */
6509                 /* LCB_CFG_RX_FIFOS_RADR = 0x988 */
6510                 rx_radr =
6511                       0x9ull << DC_LCB_CFG_RX_FIFOS_RADR_DO_NOT_JUMP_VAL_SHIFT
6512                     | 0x8ull << DC_LCB_CFG_RX_FIFOS_RADR_OK_TO_JUMP_VAL_SHIFT
6513                     | 0x8ull << DC_LCB_CFG_RX_FIFOS_RADR_RST_VAL_SHIFT;
6514                 tx_radr = 7ull << DC_LCB_CFG_TX_FIFOS_RADR_RST_VAL_SHIFT;
6515                 write_csr(dd, DC_LCB_CFG_LN_DCLK, 1ull);
6516         } else {
6517                 /* release 0x1b and higher */
6518                 /* LCB_CFG_RX_FIFOS_RADR = 0x877 */
6519                 rx_radr =
6520                       0x8ull << DC_LCB_CFG_RX_FIFOS_RADR_DO_NOT_JUMP_VAL_SHIFT
6521                     | 0x7ull << DC_LCB_CFG_RX_FIFOS_RADR_OK_TO_JUMP_VAL_SHIFT
6522                     | 0x7ull << DC_LCB_CFG_RX_FIFOS_RADR_RST_VAL_SHIFT;
6523                 tx_radr = 3ull << DC_LCB_CFG_TX_FIFOS_RADR_RST_VAL_SHIFT;
6524         }
6525
6526         write_csr(dd, DC_LCB_CFG_RX_FIFOS_RADR, rx_radr);
6527         /* LCB_CFG_IGNORE_LOST_RCLK.EN = 1 */
6528         write_csr(dd, DC_LCB_CFG_IGNORE_LOST_RCLK,
6529                   DC_LCB_CFG_IGNORE_LOST_RCLK_EN_SMASK);
6530         write_csr(dd, DC_LCB_CFG_TX_FIFOS_RADR, tx_radr);
6531 }
6532
6533 /*
6534  * Handle a SMA idle message
6535  *
6536  * This is a work-queue function outside of the interrupt.
6537  */
6538 void handle_sma_message(struct work_struct *work)
6539 {
6540         struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata,
6541                                                         sma_message_work);
6542         struct hfi1_devdata *dd = ppd->dd;
6543         u64 msg;
6544         int ret;
6545
6546         /*
6547          * msg is bytes 1-4 of the 40-bit idle message - the command code
6548          * is stripped off
6549          */
6550         ret = read_idle_sma(dd, &msg);
6551         if (ret)
6552                 return;
6553         dd_dev_info(dd, "%s: SMA message 0x%llx\n", __func__, msg);
6554         /*
6555          * React to the SMA message.  Byte[1] (0 for us) is the command.
6556          */
6557         switch (msg & 0xff) {
6558         case SMA_IDLE_ARM:
6559                 /*
6560                  * See OPAv1 table 9-14 - HFI and External Switch Ports Key
6561                  * State Transitions
6562                  *
6563                  * Only expected in INIT or ARMED, discard otherwise.
6564                  */
6565                 if (ppd->host_link_state & (HLS_UP_INIT | HLS_UP_ARMED))
6566                         ppd->neighbor_normal = 1;
6567                 break;
6568         case SMA_IDLE_ACTIVE:
6569                 /*
6570                  * See OPAv1 table 9-14 - HFI and External Switch Ports Key
6571                  * State Transitions
6572                  *
6573                  * Can activate the node.  Discard otherwise.
6574                  */
6575                 if (ppd->host_link_state == HLS_UP_ARMED &&
6576                     ppd->is_active_optimize_enabled) {
6577                         ppd->neighbor_normal = 1;
6578                         ret = set_link_state(ppd, HLS_UP_ACTIVE);
6579                         if (ret)
6580                                 dd_dev_err(
6581                                         dd,
6582                                         "%s: received Active SMA idle message, couldn't set link to Active\n",
6583                                         __func__);
6584                 }
6585                 break;
6586         default:
6587                 dd_dev_err(dd,
6588                            "%s: received unexpected SMA idle message 0x%llx\n",
6589                            __func__, msg);
6590                 break;
6591         }
6592 }
6593
6594 static void adjust_rcvctrl(struct hfi1_devdata *dd, u64 add, u64 clear)
6595 {
6596         u64 rcvctrl;
6597         unsigned long flags;
6598
6599         spin_lock_irqsave(&dd->rcvctrl_lock, flags);
6600         rcvctrl = read_csr(dd, RCV_CTRL);
6601         rcvctrl |= add;
6602         rcvctrl &= ~clear;
6603         write_csr(dd, RCV_CTRL, rcvctrl);
6604         spin_unlock_irqrestore(&dd->rcvctrl_lock, flags);
6605 }
6606
6607 static inline void add_rcvctrl(struct hfi1_devdata *dd, u64 add)
6608 {
6609         adjust_rcvctrl(dd, add, 0);
6610 }
6611
6612 static inline void clear_rcvctrl(struct hfi1_devdata *dd, u64 clear)
6613 {
6614         adjust_rcvctrl(dd, 0, clear);
6615 }
6616
6617 /*
6618  * Called from all interrupt handlers to start handling an SPC freeze.
6619  */
6620 void start_freeze_handling(struct hfi1_pportdata *ppd, int flags)
6621 {
6622         struct hfi1_devdata *dd = ppd->dd;
6623         struct send_context *sc;
6624         int i;
6625
6626         if (flags & FREEZE_SELF)
6627                 write_csr(dd, CCE_CTRL, CCE_CTRL_SPC_FREEZE_SMASK);
6628
6629         /* enter frozen mode */
6630         dd->flags |= HFI1_FROZEN;
6631
6632         /* notify all SDMA engines that they are going into a freeze */
6633         sdma_freeze_notify(dd, !!(flags & FREEZE_LINK_DOWN));
6634
6635         /* do halt pre-handling on all enabled send contexts */
6636         for (i = 0; i < dd->num_send_contexts; i++) {
6637                 sc = dd->send_contexts[i].sc;
6638                 if (sc && (sc->flags & SCF_ENABLED))
6639                         sc_stop(sc, SCF_FROZEN | SCF_HALTED);
6640         }
6641
6642         /* Send context are frozen. Notify user space */
6643         hfi1_set_uevent_bits(ppd, _HFI1_EVENT_FROZEN_BIT);
6644
6645         if (flags & FREEZE_ABORT) {
6646                 dd_dev_err(dd,
6647                            "Aborted freeze recovery. Please REBOOT system\n");
6648                 return;
6649         }
6650         /* queue non-interrupt handler */
6651         queue_work(ppd->hfi1_wq, &ppd->freeze_work);
6652 }
6653
6654 /*
6655  * Wait until all 4 sub-blocks indicate that they have frozen or unfrozen,
6656  * depending on the "freeze" parameter.
6657  *
6658  * No need to return an error if it times out, our only option
6659  * is to proceed anyway.
6660  */
6661 static void wait_for_freeze_status(struct hfi1_devdata *dd, int freeze)
6662 {
6663         unsigned long timeout;
6664         u64 reg;
6665
6666         timeout = jiffies + msecs_to_jiffies(FREEZE_STATUS_TIMEOUT);
6667         while (1) {
6668                 reg = read_csr(dd, CCE_STATUS);
6669                 if (freeze) {
6670                         /* waiting until all indicators are set */
6671                         if ((reg & ALL_FROZE) == ALL_FROZE)
6672                                 return; /* all done */
6673                 } else {
6674                         /* waiting until all indicators are clear */
6675                         if ((reg & ALL_FROZE) == 0)
6676                                 return; /* all done */
6677                 }
6678
6679                 if (time_after(jiffies, timeout)) {
6680                         dd_dev_err(dd,
6681                                    "Time out waiting for SPC %sfreeze, bits 0x%llx, expecting 0x%llx, continuing",
6682                                    freeze ? "" : "un", reg & ALL_FROZE,
6683                                    freeze ? ALL_FROZE : 0ull);
6684                         return;
6685                 }
6686                 usleep_range(80, 120);
6687         }
6688 }
6689
6690 /*
6691  * Do all freeze handling for the RXE block.
6692  */
6693 static void rxe_freeze(struct hfi1_devdata *dd)
6694 {
6695         int i;
6696
6697         /* disable port */
6698         clear_rcvctrl(dd, RCV_CTRL_RCV_PORT_ENABLE_SMASK);
6699
6700         /* disable all receive contexts */
6701         for (i = 0; i < dd->num_rcv_contexts; i++)
6702                 hfi1_rcvctrl(dd, HFI1_RCVCTRL_CTXT_DIS, i);
6703 }
6704
6705 /*
6706  * Unfreeze handling for the RXE block - kernel contexts only.
6707  * This will also enable the port.  User contexts will do unfreeze
6708  * handling on a per-context basis as they call into the driver.
6709  *
6710  */
6711 static void rxe_kernel_unfreeze(struct hfi1_devdata *dd)
6712 {
6713         u32 rcvmask;
6714         int i;
6715
6716         /* enable all kernel contexts */
6717         for (i = 0; i < dd->n_krcv_queues; i++) {
6718                 rcvmask = HFI1_RCVCTRL_CTXT_ENB;
6719                 /* HFI1_RCVCTRL_TAILUPD_[ENB|DIS] needs to be set explicitly */
6720                 rcvmask |= dd->rcd[i]->rcvhdrtail_kvaddr ?
6721                         HFI1_RCVCTRL_TAILUPD_ENB : HFI1_RCVCTRL_TAILUPD_DIS;
6722                 hfi1_rcvctrl(dd, rcvmask, i);
6723         }
6724
6725         /* enable port */
6726         add_rcvctrl(dd, RCV_CTRL_RCV_PORT_ENABLE_SMASK);
6727 }
6728
6729 /*
6730  * Non-interrupt SPC freeze handling.
6731  *
6732  * This is a work-queue function outside of the triggering interrupt.
6733  */
6734 void handle_freeze(struct work_struct *work)
6735 {
6736         struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata,
6737                                                                 freeze_work);
6738         struct hfi1_devdata *dd = ppd->dd;
6739
6740         /* wait for freeze indicators on all affected blocks */
6741         wait_for_freeze_status(dd, 1);
6742
6743         /* SPC is now frozen */
6744
6745         /* do send PIO freeze steps */
6746         pio_freeze(dd);
6747
6748         /* do send DMA freeze steps */
6749         sdma_freeze(dd);
6750
6751         /* do send egress freeze steps - nothing to do */
6752
6753         /* do receive freeze steps */
6754         rxe_freeze(dd);
6755
6756         /*
6757          * Unfreeze the hardware - clear the freeze, wait for each
6758          * block's frozen bit to clear, then clear the frozen flag.
6759          */
6760         write_csr(dd, CCE_CTRL, CCE_CTRL_SPC_UNFREEZE_SMASK);
6761         wait_for_freeze_status(dd, 0);
6762
6763         if (is_ax(dd)) {
6764                 write_csr(dd, CCE_CTRL, CCE_CTRL_SPC_FREEZE_SMASK);
6765                 wait_for_freeze_status(dd, 1);
6766                 write_csr(dd, CCE_CTRL, CCE_CTRL_SPC_UNFREEZE_SMASK);
6767                 wait_for_freeze_status(dd, 0);
6768         }
6769
6770         /* do send PIO unfreeze steps for kernel contexts */
6771         pio_kernel_unfreeze(dd);
6772
6773         /* do send DMA unfreeze steps */
6774         sdma_unfreeze(dd);
6775
6776         /* do send egress unfreeze steps - nothing to do */
6777
6778         /* do receive unfreeze steps for kernel contexts */
6779         rxe_kernel_unfreeze(dd);
6780
6781         /*
6782          * The unfreeze procedure touches global device registers when
6783          * it disables and re-enables RXE. Mark the device unfrozen
6784          * after all that is done so other parts of the driver waiting
6785          * for the device to unfreeze don't do things out of order.
6786          *
6787          * The above implies that the meaning of HFI1_FROZEN flag is
6788          * "Device has gone into freeze mode and freeze mode handling
6789          * is still in progress."
6790          *
6791          * The flag will be removed when freeze mode processing has
6792          * completed.
6793          */
6794         dd->flags &= ~HFI1_FROZEN;
6795         wake_up(&dd->event_queue);
6796
6797         /* no longer frozen */
6798 }
6799
6800 /*
6801  * Handle a link up interrupt from the 8051.
6802  *
6803  * This is a work-queue function outside of the interrupt.
6804  */
6805 void handle_link_up(struct work_struct *work)
6806 {
6807         struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata,
6808                                                   link_up_work);
6809         set_link_state(ppd, HLS_UP_INIT);
6810
6811         /* cache the read of DC_LCB_STS_ROUND_TRIP_LTP_CNT */
6812         read_ltp_rtt(ppd->dd);
6813         /*
6814          * OPA specifies that certain counters are cleared on a transition
6815          * to link up, so do that.
6816          */
6817         clear_linkup_counters(ppd->dd);
6818         /*
6819          * And (re)set link up default values.
6820          */
6821         set_linkup_defaults(ppd);
6822
6823         /* enforce link speed enabled */
6824         if ((ppd->link_speed_active & ppd->link_speed_enabled) == 0) {
6825                 /* oops - current speed is not enabled, bounce */
6826                 dd_dev_err(ppd->dd,
6827                            "Link speed active 0x%x is outside enabled 0x%x, downing link\n",
6828                            ppd->link_speed_active, ppd->link_speed_enabled);
6829                 set_link_down_reason(ppd, OPA_LINKDOWN_REASON_SPEED_POLICY, 0,
6830                                      OPA_LINKDOWN_REASON_SPEED_POLICY);
6831                 set_link_state(ppd, HLS_DN_OFFLINE);
6832                 start_link(ppd);
6833         }
6834 }
6835
6836 /*
6837  * Several pieces of LNI information were cached for SMA in ppd.
6838  * Reset these on link down
6839  */
6840 static void reset_neighbor_info(struct hfi1_pportdata *ppd)
6841 {
6842         ppd->neighbor_guid = 0;
6843         ppd->neighbor_port_number = 0;
6844         ppd->neighbor_type = 0;
6845         ppd->neighbor_fm_security = 0;
6846 }
6847
6848 static const char * const link_down_reason_strs[] = {
6849         [OPA_LINKDOWN_REASON_NONE] = "None",
6850         [OPA_LINKDOWN_REASON_RCV_ERROR_0] = "Recive error 0",
6851         [OPA_LINKDOWN_REASON_BAD_PKT_LEN] = "Bad packet length",
6852         [OPA_LINKDOWN_REASON_PKT_TOO_LONG] = "Packet too long",
6853         [OPA_LINKDOWN_REASON_PKT_TOO_SHORT] = "Packet too short",
6854         [OPA_LINKDOWN_REASON_BAD_SLID] = "Bad SLID",
6855         [OPA_LINKDOWN_REASON_BAD_DLID] = "Bad DLID",
6856         [OPA_LINKDOWN_REASON_BAD_L2] = "Bad L2",
6857         [OPA_LINKDOWN_REASON_BAD_SC] = "Bad SC",
6858         [OPA_LINKDOWN_REASON_RCV_ERROR_8] = "Receive error 8",
6859         [OPA_LINKDOWN_REASON_BAD_MID_TAIL] = "Bad mid tail",
6860         [OPA_LINKDOWN_REASON_RCV_ERROR_10] = "Receive error 10",
6861         [OPA_LINKDOWN_REASON_PREEMPT_ERROR] = "Preempt error",
6862         [OPA_LINKDOWN_REASON_PREEMPT_VL15] = "Preempt vl15",
6863         [OPA_LINKDOWN_REASON_BAD_VL_MARKER] = "Bad VL marker",
6864         [OPA_LINKDOWN_REASON_RCV_ERROR_14] = "Receive error 14",
6865         [OPA_LINKDOWN_REASON_RCV_ERROR_15] = "Receive error 15",
6866         [OPA_LINKDOWN_REASON_BAD_HEAD_DIST] = "Bad head distance",
6867         [OPA_LINKDOWN_REASON_BAD_TAIL_DIST] = "Bad tail distance",
6868         [OPA_LINKDOWN_REASON_BAD_CTRL_DIST] = "Bad control distance",
6869         [OPA_LINKDOWN_REASON_BAD_CREDIT_ACK] = "Bad credit ack",
6870         [OPA_LINKDOWN_REASON_UNSUPPORTED_VL_MARKER] = "Unsupported VL marker",
6871         [OPA_LINKDOWN_REASON_BAD_PREEMPT] = "Bad preempt",
6872         [OPA_LINKDOWN_REASON_BAD_CONTROL_FLIT] = "Bad control flit",
6873         [OPA_LINKDOWN_REASON_EXCEED_MULTICAST_LIMIT] = "Exceed multicast limit",
6874         [OPA_LINKDOWN_REASON_RCV_ERROR_24] = "Receive error 24",
6875         [OPA_LINKDOWN_REASON_RCV_ERROR_25] = "Receive error 25",
6876         [OPA_LINKDOWN_REASON_RCV_ERROR_26] = "Receive error 26",
6877         [OPA_LINKDOWN_REASON_RCV_ERROR_27] = "Receive error 27",
6878         [OPA_LINKDOWN_REASON_RCV_ERROR_28] = "Receive error 28",
6879         [OPA_LINKDOWN_REASON_RCV_ERROR_29] = "Receive error 29",
6880         [OPA_LINKDOWN_REASON_RCV_ERROR_30] = "Receive error 30",
6881         [OPA_LINKDOWN_REASON_EXCESSIVE_BUFFER_OVERRUN] =
6882                                         "Excessive buffer overrun",
6883         [OPA_LINKDOWN_REASON_UNKNOWN] = "Unknown",
6884         [OPA_LINKDOWN_REASON_REBOOT] = "Reboot",
6885         [OPA_LINKDOWN_REASON_NEIGHBOR_UNKNOWN] = "Neighbor unknown",
6886         [OPA_LINKDOWN_REASON_FM_BOUNCE] = "FM bounce",
6887         [OPA_LINKDOWN_REASON_SPEED_POLICY] = "Speed policy",
6888         [OPA_LINKDOWN_REASON_WIDTH_POLICY] = "Width policy",
6889         [OPA_LINKDOWN_REASON_DISCONNECTED] = "Disconnected",
6890         [OPA_LINKDOWN_REASON_LOCAL_MEDIA_NOT_INSTALLED] =
6891                                         "Local media not installed",
6892         [OPA_LINKDOWN_REASON_NOT_INSTALLED] = "Not installed",
6893         [OPA_LINKDOWN_REASON_CHASSIS_CONFIG] = "Chassis config",
6894         [OPA_LINKDOWN_REASON_END_TO_END_NOT_INSTALLED] =
6895                                         "End to end not installed",
6896         [OPA_LINKDOWN_REASON_POWER_POLICY] = "Power policy",
6897         [OPA_LINKDOWN_REASON_LINKSPEED_POLICY] = "Link speed policy",
6898         [OPA_LINKDOWN_REASON_LINKWIDTH_POLICY] = "Link width policy",
6899         [OPA_LINKDOWN_REASON_SWITCH_MGMT] = "Switch management",
6900         [OPA_LINKDOWN_REASON_SMA_DISABLED] = "SMA disabled",
6901         [OPA_LINKDOWN_REASON_TRANSIENT] = "Transient"
6902 };
6903
6904 /* return the neighbor link down reason string */
6905 static const char *link_down_reason_str(u8 reason)
6906 {
6907         const char *str = NULL;
6908
6909         if (reason < ARRAY_SIZE(link_down_reason_strs))
6910                 str = link_down_reason_strs[reason];
6911         if (!str)
6912                 str = "(invalid)";
6913
6914         return str;
6915 }
6916
6917 /*
6918  * Handle a link down interrupt from the 8051.
6919  *
6920  * This is a work-queue function outside of the interrupt.
6921  */
6922 void handle_link_down(struct work_struct *work)
6923 {
6924         u8 lcl_reason, neigh_reason = 0;
6925         u8 link_down_reason;
6926         struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata,
6927                                                   link_down_work);
6928         int was_up;
6929         static const char ldr_str[] = "Link down reason: ";
6930
6931         if ((ppd->host_link_state &
6932              (HLS_DN_POLL | HLS_VERIFY_CAP | HLS_GOING_UP)) &&
6933              ppd->port_type == PORT_TYPE_FIXED)
6934                 ppd->offline_disabled_reason =
6935                         HFI1_ODR_MASK(OPA_LINKDOWN_REASON_NOT_INSTALLED);
6936
6937         /* Go offline first, then deal with reading/writing through 8051 */
6938         was_up = !!(ppd->host_link_state & HLS_UP);
6939         set_link_state(ppd, HLS_DN_OFFLINE);
6940
6941         if (was_up) {
6942                 lcl_reason = 0;
6943                 /* link down reason is only valid if the link was up */
6944                 read_link_down_reason(ppd->dd, &link_down_reason);
6945                 switch (link_down_reason) {
6946                 case LDR_LINK_TRANSFER_ACTIVE_LOW:
6947                         /* the link went down, no idle message reason */
6948                         dd_dev_info(ppd->dd, "%sUnexpected link down\n",
6949                                     ldr_str);
6950                         break;
6951                 case LDR_RECEIVED_LINKDOWN_IDLE_MSG:
6952                         /*
6953                          * The neighbor reason is only valid if an idle message
6954                          * was received for it.
6955                          */
6956                         read_planned_down_reason_code(ppd->dd, &neigh_reason);
6957                         dd_dev_info(ppd->dd,
6958                                     "%sNeighbor link down message %d, %s\n",
6959                                     ldr_str, neigh_reason,
6960                                     link_down_reason_str(neigh_reason));
6961                         break;
6962                 case LDR_RECEIVED_HOST_OFFLINE_REQ:
6963                         dd_dev_info(ppd->dd,
6964                                     "%sHost requested link to go offline\n",
6965                                     ldr_str);
6966                         break;
6967                 default:
6968                         dd_dev_info(ppd->dd, "%sUnknown reason 0x%x\n",
6969                                     ldr_str, link_down_reason);
6970                         break;
6971                 }
6972
6973                 /*
6974                  * If no reason, assume peer-initiated but missed
6975                  * LinkGoingDown idle flits.
6976                  */
6977                 if (neigh_reason == 0)
6978                         lcl_reason = OPA_LINKDOWN_REASON_NEIGHBOR_UNKNOWN;
6979         } else {
6980                 /* went down while polling or going up */
6981                 lcl_reason = OPA_LINKDOWN_REASON_TRANSIENT;
6982         }
6983
6984         set_link_down_reason(ppd, lcl_reason, neigh_reason, 0);
6985
6986         /* inform the SMA when the link transitions from up to down */
6987         if (was_up && ppd->local_link_down_reason.sma == 0 &&
6988             ppd->neigh_link_down_reason.sma == 0) {
6989                 ppd->local_link_down_reason.sma =
6990                                         ppd->local_link_down_reason.latest;
6991                 ppd->neigh_link_down_reason.sma =
6992                                         ppd->neigh_link_down_reason.latest;
6993         }
6994
6995         reset_neighbor_info(ppd);
6996
6997         /* disable the port */
6998         clear_rcvctrl(ppd->dd, RCV_CTRL_RCV_PORT_ENABLE_SMASK);
6999
7000         /*
7001          * If there is no cable attached, turn the DC off. Otherwise,
7002          * start the link bring up.
7003          */
7004         if (ppd->port_type == PORT_TYPE_QSFP && !qsfp_mod_present(ppd))
7005                 dc_shutdown(ppd->dd);
7006         else
7007                 start_link(ppd);
7008 }
7009
7010 void handle_link_bounce(struct work_struct *work)
7011 {
7012         struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata,
7013                                                         link_bounce_work);
7014
7015         /*
7016          * Only do something if the link is currently up.
7017          */
7018         if (ppd->host_link_state & HLS_UP) {
7019                 set_link_state(ppd, HLS_DN_OFFLINE);
7020                 start_link(ppd);
7021         } else {
7022                 dd_dev_info(ppd->dd, "%s: link not up (%s), nothing to do\n",
7023                             __func__, link_state_name(ppd->host_link_state));
7024         }
7025 }
7026
7027 /*
7028  * Mask conversion: Capability exchange to Port LTP.  The capability
7029  * exchange has an implicit 16b CRC that is mandatory.
7030  */
7031 static int cap_to_port_ltp(int cap)
7032 {
7033         int port_ltp = PORT_LTP_CRC_MODE_16; /* this mode is mandatory */
7034
7035         if (cap & CAP_CRC_14B)
7036                 port_ltp |= PORT_LTP_CRC_MODE_14;
7037         if (cap & CAP_CRC_48B)
7038                 port_ltp |= PORT_LTP_CRC_MODE_48;
7039         if (cap & CAP_CRC_12B_16B_PER_LANE)
7040                 port_ltp |= PORT_LTP_CRC_MODE_PER_LANE;
7041
7042         return port_ltp;
7043 }
7044
7045 /*
7046  * Convert an OPA Port LTP mask to capability mask
7047  */
7048 int port_ltp_to_cap(int port_ltp)
7049 {
7050         int cap_mask = 0;
7051
7052         if (port_ltp & PORT_LTP_CRC_MODE_14)
7053                 cap_mask |= CAP_CRC_14B;
7054         if (port_ltp & PORT_LTP_CRC_MODE_48)
7055                 cap_mask |= CAP_CRC_48B;
7056         if (port_ltp & PORT_LTP_CRC_MODE_PER_LANE)
7057                 cap_mask |= CAP_CRC_12B_16B_PER_LANE;
7058
7059         return cap_mask;
7060 }
7061
7062 /*
7063  * Convert a single DC LCB CRC mode to an OPA Port LTP mask.
7064  */
7065 static int lcb_to_port_ltp(int lcb_crc)
7066 {
7067         int port_ltp = 0;
7068
7069         if (lcb_crc == LCB_CRC_12B_16B_PER_LANE)
7070                 port_ltp = PORT_LTP_CRC_MODE_PER_LANE;
7071         else if (lcb_crc == LCB_CRC_48B)
7072                 port_ltp = PORT_LTP_CRC_MODE_48;
7073         else if (lcb_crc == LCB_CRC_14B)
7074                 port_ltp = PORT_LTP_CRC_MODE_14;
7075         else
7076                 port_ltp = PORT_LTP_CRC_MODE_16;
7077
7078         return port_ltp;
7079 }
7080
7081 /*
7082  * Our neighbor has indicated that we are allowed to act as a fabric
7083  * manager, so place the full management partition key in the second
7084  * (0-based) pkey array position (see OPAv1, section 20.2.2.6.8). Note
7085  * that we should already have the limited management partition key in
7086  * array element 1, and also that the port is not yet up when
7087  * add_full_mgmt_pkey() is invoked.
7088  */
7089 static void add_full_mgmt_pkey(struct hfi1_pportdata *ppd)
7090 {
7091         struct hfi1_devdata *dd = ppd->dd;
7092
7093         /* Sanity check - ppd->pkeys[2] should be 0, or already initalized */
7094         if (!((ppd->pkeys[2] == 0) || (ppd->pkeys[2] == FULL_MGMT_P_KEY)))
7095                 dd_dev_warn(dd, "%s pkey[2] already set to 0x%x, resetting it to 0x%x\n",
7096                             __func__, ppd->pkeys[2], FULL_MGMT_P_KEY);
7097         ppd->pkeys[2] = FULL_MGMT_P_KEY;
7098         (void)hfi1_set_ib_cfg(ppd, HFI1_IB_CFG_PKEYS, 0);
7099         hfi1_event_pkey_change(ppd->dd, ppd->port);
7100 }
7101
7102 static void clear_full_mgmt_pkey(struct hfi1_pportdata *ppd)
7103 {
7104         if (ppd->pkeys[2] != 0) {
7105                 ppd->pkeys[2] = 0;
7106                 (void)hfi1_set_ib_cfg(ppd, HFI1_IB_CFG_PKEYS, 0);
7107                 hfi1_event_pkey_change(ppd->dd, ppd->port);
7108         }
7109 }
7110
7111 /*
7112  * Convert the given link width to the OPA link width bitmask.
7113  */
7114 static u16 link_width_to_bits(struct hfi1_devdata *dd, u16 width)
7115 {
7116         switch (width) {
7117         case 0:
7118                 /*
7119                  * Simulator and quick linkup do not set the width.
7120                  * Just set it to 4x without complaint.
7121                  */
7122                 if (dd->icode == ICODE_FUNCTIONAL_SIMULATOR || quick_linkup)
7123                         return OPA_LINK_WIDTH_4X;
7124                 return 0; /* no lanes up */
7125         case 1: return OPA_LINK_WIDTH_1X;
7126         case 2: return OPA_LINK_WIDTH_2X;
7127         case 3: return OPA_LINK_WIDTH_3X;
7128         default:
7129                 dd_dev_info(dd, "%s: invalid width %d, using 4\n",
7130                             __func__, width);
7131                 /* fall through */
7132         case 4: return OPA_LINK_WIDTH_4X;
7133         }
7134 }
7135
7136 /*
7137  * Do a population count on the bottom nibble.
7138  */
7139 static const u8 bit_counts[16] = {
7140         0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4
7141 };
7142
7143 static inline u8 nibble_to_count(u8 nibble)
7144 {
7145         return bit_counts[nibble & 0xf];
7146 }
7147
7148 /*
7149  * Read the active lane information from the 8051 registers and return
7150  * their widths.
7151  *
7152  * Active lane information is found in these 8051 registers:
7153  *      enable_lane_tx
7154  *      enable_lane_rx
7155  */
7156 static void get_link_widths(struct hfi1_devdata *dd, u16 *tx_width,
7157                             u16 *rx_width)
7158 {
7159         u16 tx, rx;
7160         u8 enable_lane_rx;
7161         u8 enable_lane_tx;
7162         u8 tx_polarity_inversion;
7163         u8 rx_polarity_inversion;
7164         u8 max_rate;
7165
7166         /* read the active lanes */
7167         read_tx_settings(dd, &enable_lane_tx, &tx_polarity_inversion,
7168                          &rx_polarity_inversion, &max_rate);
7169         read_local_lni(dd, &enable_lane_rx);
7170
7171         /* convert to counts */
7172         tx = nibble_to_count(enable_lane_tx);
7173         rx = nibble_to_count(enable_lane_rx);
7174
7175         /*
7176          * Set link_speed_active here, overriding what was set in
7177          * handle_verify_cap().  The ASIC 8051 firmware does not correctly
7178          * set the max_rate field in handle_verify_cap until v0.19.
7179          */
7180         if ((dd->icode == ICODE_RTL_SILICON) &&
7181             (dd->dc8051_ver < dc8051_ver(0, 19))) {
7182                 /* max_rate: 0 = 12.5G, 1 = 25G */
7183                 switch (max_rate) {
7184                 case 0:
7185                         dd->pport[0].link_speed_active = OPA_LINK_SPEED_12_5G;
7186                         break;
7187                 default:
7188                         dd_dev_err(dd,
7189                                    "%s: unexpected max rate %d, using 25Gb\n",
7190                                    __func__, (int)max_rate);
7191                         /* fall through */
7192                 case 1:
7193                         dd->pport[0].link_speed_active = OPA_LINK_SPEED_25G;
7194                         break;
7195                 }
7196         }
7197
7198         dd_dev_info(dd,
7199                     "Fabric active lanes (width): tx 0x%x (%d), rx 0x%x (%d)\n",
7200                     enable_lane_tx, tx, enable_lane_rx, rx);
7201         *tx_width = link_width_to_bits(dd, tx);
7202         *rx_width = link_width_to_bits(dd, rx);
7203 }
7204
7205 /*
7206  * Read verify_cap_local_fm_link_width[1] to obtain the link widths.
7207  * Valid after the end of VerifyCap and during LinkUp.  Does not change
7208  * after link up.  I.e. look elsewhere for downgrade information.
7209  *
7210  * Bits are:
7211  *      + bits [7:4] contain the number of active transmitters
7212  *      + bits [3:0] contain the number of active receivers
7213  * These are numbers 1 through 4 and can be different values if the
7214  * link is asymmetric.
7215  *
7216  * verify_cap_local_fm_link_width[0] retains its original value.
7217  */
7218 static void get_linkup_widths(struct hfi1_devdata *dd, u16 *tx_width,
7219                               u16 *rx_width)
7220 {
7221         u16 widths, tx, rx;
7222         u8 misc_bits, local_flags;
7223         u16 active_tx, active_rx;
7224
7225         read_vc_local_link_width(dd, &misc_bits, &local_flags, &widths);
7226         tx = widths >> 12;
7227         rx = (widths >> 8) & 0xf;
7228
7229         *tx_width = link_width_to_bits(dd, tx);
7230         *rx_width = link_width_to_bits(dd, rx);
7231
7232         /* print the active widths */
7233         get_link_widths(dd, &active_tx, &active_rx);
7234 }
7235
7236 /*
7237  * Set ppd->link_width_active and ppd->link_width_downgrade_active using
7238  * hardware information when the link first comes up.
7239  *
7240  * The link width is not available until after VerifyCap.AllFramesReceived
7241  * (the trigger for handle_verify_cap), so this is outside that routine
7242  * and should be called when the 8051 signals linkup.
7243  */
7244 void get_linkup_link_widths(struct hfi1_pportdata *ppd)
7245 {
7246         u16 tx_width, rx_width;
7247
7248         /* get end-of-LNI link widths */
7249         get_linkup_widths(ppd->dd, &tx_width, &rx_width);
7250
7251         /* use tx_width as the link is supposed to be symmetric on link up */
7252         ppd->link_width_active = tx_width;
7253         /* link width downgrade active (LWD.A) starts out matching LW.A */
7254         ppd->link_width_downgrade_tx_active = ppd->link_width_active;
7255         ppd->link_width_downgrade_rx_active = ppd->link_width_active;
7256         /* per OPA spec, on link up LWD.E resets to LWD.S */
7257         ppd->link_width_downgrade_enabled = ppd->link_width_downgrade_supported;
7258         /* cache the active egress rate (units {10^6 bits/sec]) */
7259         ppd->current_egress_rate = active_egress_rate(ppd);
7260 }
7261
7262 /*
7263  * Handle a verify capabilities interrupt from the 8051.
7264  *
7265  * This is a work-queue function outside of the interrupt.
7266  */
7267 void handle_verify_cap(struct work_struct *work)
7268 {
7269         struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata,
7270                                                                 link_vc_work);
7271         struct hfi1_devdata *dd = ppd->dd;
7272         u64 reg;
7273         u8 power_management;
7274         u8 continious;
7275         u8 vcu;
7276         u8 vau;
7277         u8 z;
7278         u16 vl15buf;
7279         u16 link_widths;
7280         u16 crc_mask;
7281         u16 crc_val;
7282         u16 device_id;
7283         u16 active_tx, active_rx;
7284         u8 partner_supported_crc;
7285         u8 remote_tx_rate;
7286         u8 device_rev;
7287
7288         set_link_state(ppd, HLS_VERIFY_CAP);
7289
7290         lcb_shutdown(dd, 0);
7291         adjust_lcb_for_fpga_serdes(dd);
7292
7293         /*
7294          * These are now valid:
7295          *      remote VerifyCap fields in the general LNI config
7296          *      CSR DC8051_STS_REMOTE_GUID
7297          *      CSR DC8051_STS_REMOTE_NODE_TYPE
7298          *      CSR DC8051_STS_REMOTE_FM_SECURITY
7299          *      CSR DC8051_STS_REMOTE_PORT_NO
7300          */
7301
7302         read_vc_remote_phy(dd, &power_management, &continious);
7303         read_vc_remote_fabric(dd, &vau, &z, &vcu, &vl15buf,
7304                               &partner_supported_crc);
7305         read_vc_remote_link_width(dd, &remote_tx_rate, &link_widths);
7306         read_remote_device_id(dd, &device_id, &device_rev);
7307         /*
7308          * And the 'MgmtAllowed' information, which is exchanged during
7309          * LNI, is also be available at this point.
7310          */
7311         read_mgmt_allowed(dd, &ppd->mgmt_allowed);
7312         /* print the active widths */
7313         get_link_widths(dd, &active_tx, &active_rx);
7314         dd_dev_info(dd,
7315                     "Peer PHY: power management 0x%x, continuous updates 0x%x\n",
7316                     (int)power_management, (int)continious);
7317         dd_dev_info(dd,
7318                     "Peer Fabric: vAU %d, Z %d, vCU %d, vl15 credits 0x%x, CRC sizes 0x%x\n",
7319                     (int)vau, (int)z, (int)vcu, (int)vl15buf,
7320                     (int)partner_supported_crc);
7321         dd_dev_info(dd, "Peer Link Width: tx rate 0x%x, widths 0x%x\n",
7322                     (u32)remote_tx_rate, (u32)link_widths);
7323         dd_dev_info(dd, "Peer Device ID: 0x%04x, Revision 0x%02x\n",
7324                     (u32)device_id, (u32)device_rev);
7325         /*
7326          * The peer vAU value just read is the peer receiver value.  HFI does
7327          * not support a transmit vAU of 0 (AU == 8).  We advertised that
7328          * with Z=1 in the fabric capabilities sent to the peer.  The peer
7329          * will see our Z=1, and, if it advertised a vAU of 0, will move its
7330          * receive to vAU of 1 (AU == 16).  Do the same here.  We do not care
7331          * about the peer Z value - our sent vAU is 3 (hardwired) and is not
7332          * subject to the Z value exception.
7333          */
7334         if (vau == 0)
7335                 vau = 1;
7336         set_up_vl15(dd, vau, vl15buf);
7337
7338         /* set up the LCB CRC mode */
7339         crc_mask = ppd->port_crc_mode_enabled & partner_supported_crc;
7340
7341         /* order is important: use the lowest bit in common */
7342         if (crc_mask & CAP_CRC_14B)
7343                 crc_val = LCB_CRC_14B;
7344         else if (crc_mask & CAP_CRC_48B)
7345                 crc_val = LCB_CRC_48B;
7346         else if (crc_mask & CAP_CRC_12B_16B_PER_LANE)
7347                 crc_val = LCB_CRC_12B_16B_PER_LANE;
7348         else
7349                 crc_val = LCB_CRC_16B;
7350
7351         dd_dev_info(dd, "Final LCB CRC mode: %d\n", (int)crc_val);
7352         write_csr(dd, DC_LCB_CFG_CRC_MODE,
7353                   (u64)crc_val << DC_LCB_CFG_CRC_MODE_TX_VAL_SHIFT);
7354
7355         /* set (14b only) or clear sideband credit */
7356         reg = read_csr(dd, SEND_CM_CTRL);
7357         if (crc_val == LCB_CRC_14B && crc_14b_sideband) {
7358                 write_csr(dd, SEND_CM_CTRL,
7359                           reg | SEND_CM_CTRL_FORCE_CREDIT_MODE_SMASK);
7360         } else {
7361                 write_csr(dd, SEND_CM_CTRL,
7362                           reg & ~SEND_CM_CTRL_FORCE_CREDIT_MODE_SMASK);
7363         }
7364
7365         ppd->link_speed_active = 0;     /* invalid value */
7366         if (dd->dc8051_ver < dc8051_ver(0, 20)) {
7367                 /* remote_tx_rate: 0 = 12.5G, 1 = 25G */
7368                 switch (remote_tx_rate) {
7369                 case 0:
7370                         ppd->link_speed_active = OPA_LINK_SPEED_12_5G;
7371                         break;
7372                 case 1:
7373                         ppd->link_speed_active = OPA_LINK_SPEED_25G;
7374                         break;
7375                 }
7376         } else {
7377                 /* actual rate is highest bit of the ANDed rates */
7378                 u8 rate = remote_tx_rate & ppd->local_tx_rate;
7379
7380                 if (rate & 2)
7381                         ppd->link_speed_active = OPA_LINK_SPEED_25G;
7382                 else if (rate & 1)
7383                         ppd->link_speed_active = OPA_LINK_SPEED_12_5G;
7384         }
7385         if (ppd->link_speed_active == 0) {
7386                 dd_dev_err(dd, "%s: unexpected remote tx rate %d, using 25Gb\n",
7387                            __func__, (int)remote_tx_rate);
7388                 ppd->link_speed_active = OPA_LINK_SPEED_25G;
7389         }
7390
7391         /*
7392          * Cache the values of the supported, enabled, and active
7393          * LTP CRC modes to return in 'portinfo' queries. But the bit
7394          * flags that are returned in the portinfo query differ from
7395          * what's in the link_crc_mask, crc_sizes, and crc_val
7396          * variables. Convert these here.
7397          */
7398         ppd->port_ltp_crc_mode = cap_to_port_ltp(link_crc_mask) << 8;
7399                 /* supported crc modes */
7400         ppd->port_ltp_crc_mode |=
7401                 cap_to_port_ltp(ppd->port_crc_mode_enabled) << 4;
7402                 /* enabled crc modes */
7403         ppd->port_ltp_crc_mode |= lcb_to_port_ltp(crc_val);
7404                 /* active crc mode */
7405
7406         /* set up the remote credit return table */
7407         assign_remote_cm_au_table(dd, vcu);
7408
7409         /*
7410          * The LCB is reset on entry to handle_verify_cap(), so this must
7411          * be applied on every link up.
7412          *
7413          * Adjust LCB error kill enable to kill the link if
7414          * these RBUF errors are seen:
7415          *      REPLAY_BUF_MBE_SMASK
7416          *      FLIT_INPUT_BUF_MBE_SMASK
7417          */
7418         if (is_ax(dd)) {                        /* fixed in B0 */
7419                 reg = read_csr(dd, DC_LCB_CFG_LINK_KILL_EN);
7420                 reg |= DC_LCB_CFG_LINK_KILL_EN_REPLAY_BUF_MBE_SMASK
7421                         | DC_LCB_CFG_LINK_KILL_EN_FLIT_INPUT_BUF_MBE_SMASK;
7422                 write_csr(dd, DC_LCB_CFG_LINK_KILL_EN, reg);
7423         }
7424
7425         /* pull LCB fifos out of reset - all fifo clocks must be stable */
7426         write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET, 0);
7427
7428         /* give 8051 access to the LCB CSRs */
7429         write_csr(dd, DC_LCB_ERR_EN, 0); /* mask LCB errors */
7430         set_8051_lcb_access(dd);
7431
7432         ppd->neighbor_guid =
7433                 read_csr(dd, DC_DC8051_STS_REMOTE_GUID);
7434         ppd->neighbor_port_number = read_csr(dd, DC_DC8051_STS_REMOTE_PORT_NO) &
7435                                         DC_DC8051_STS_REMOTE_PORT_NO_VAL_SMASK;
7436         ppd->neighbor_type =
7437                 read_csr(dd, DC_DC8051_STS_REMOTE_NODE_TYPE) &
7438                 DC_DC8051_STS_REMOTE_NODE_TYPE_VAL_MASK;
7439         ppd->neighbor_fm_security =
7440                 read_csr(dd, DC_DC8051_STS_REMOTE_FM_SECURITY) &
7441                 DC_DC8051_STS_LOCAL_FM_SECURITY_DISABLED_MASK;
7442         dd_dev_info(dd,
7443                     "Neighbor Guid: %llx Neighbor type %d MgmtAllowed %d FM security bypass %d\n",
7444                     ppd->neighbor_guid, ppd->neighbor_type,
7445                     ppd->mgmt_allowed, ppd->neighbor_fm_security);
7446         if (ppd->mgmt_allowed)
7447                 add_full_mgmt_pkey(ppd);
7448
7449         /* tell the 8051 to go to LinkUp */
7450         set_link_state(ppd, HLS_GOING_UP);
7451 }
7452
7453 /*
7454  * Apply the link width downgrade enabled policy against the current active
7455  * link widths.
7456  *
7457  * Called when the enabled policy changes or the active link widths change.
7458  */
7459 void apply_link_downgrade_policy(struct hfi1_pportdata *ppd, int refresh_widths)
7460 {
7461         int do_bounce = 0;
7462         int tries;
7463         u16 lwde;
7464         u16 tx, rx;
7465
7466         /* use the hls lock to avoid a race with actual link up */
7467         tries = 0;
7468 retry:
7469         mutex_lock(&ppd->hls_lock);
7470         /* only apply if the link is up */
7471         if (ppd->host_link_state & HLS_DOWN) {
7472                 /* still going up..wait and retry */
7473                 if (ppd->host_link_state & HLS_GOING_UP) {
7474                         if (++tries < 1000) {
7475                                 mutex_unlock(&ppd->hls_lock);
7476                                 usleep_range(100, 120); /* arbitrary */
7477                                 goto retry;
7478                         }
7479                         dd_dev_err(ppd->dd,
7480                                    "%s: giving up waiting for link state change\n",
7481                                    __func__);
7482                 }
7483                 goto done;
7484         }
7485
7486         lwde = ppd->link_width_downgrade_enabled;
7487
7488         if (refresh_widths) {
7489                 get_link_widths(ppd->dd, &tx, &rx);
7490                 ppd->link_width_downgrade_tx_active = tx;
7491                 ppd->link_width_downgrade_rx_active = rx;
7492         }
7493
7494         if (ppd->link_width_downgrade_tx_active == 0 ||
7495             ppd->link_width_downgrade_rx_active == 0) {
7496                 /* the 8051 reported a dead link as a downgrade */
7497                 dd_dev_err(ppd->dd, "Link downgrade is really a link down, ignoring\n");
7498         } else if (lwde == 0) {
7499                 /* downgrade is disabled */
7500
7501                 /* bounce if not at starting active width */
7502                 if ((ppd->link_width_active !=
7503                      ppd->link_width_downgrade_tx_active) ||
7504                     (ppd->link_width_active !=
7505                      ppd->link_width_downgrade_rx_active)) {
7506                         dd_dev_err(ppd->dd,
7507                                    "Link downgrade is disabled and link has downgraded, downing link\n");
7508                         dd_dev_err(ppd->dd,
7509                                    "  original 0x%x, tx active 0x%x, rx active 0x%x\n",
7510                                    ppd->link_width_active,
7511                                    ppd->link_width_downgrade_tx_active,
7512                                    ppd->link_width_downgrade_rx_active);
7513                         do_bounce = 1;
7514                 }
7515         } else if ((lwde & ppd->link_width_downgrade_tx_active) == 0 ||
7516                    (lwde & ppd->link_width_downgrade_rx_active) == 0) {
7517                 /* Tx or Rx is outside the enabled policy */
7518                 dd_dev_err(ppd->dd,
7519                            "Link is outside of downgrade allowed, downing link\n");
7520                 dd_dev_err(ppd->dd,
7521                            "  enabled 0x%x, tx active 0x%x, rx active 0x%x\n",
7522                            lwde, ppd->link_width_downgrade_tx_active,
7523                            ppd->link_width_downgrade_rx_active);
7524                 do_bounce = 1;
7525         }
7526
7527 done:
7528         mutex_unlock(&ppd->hls_lock);
7529
7530         if (do_bounce) {
7531                 set_link_down_reason(ppd, OPA_LINKDOWN_REASON_WIDTH_POLICY, 0,
7532                                      OPA_LINKDOWN_REASON_WIDTH_POLICY);
7533                 set_link_state(ppd, HLS_DN_OFFLINE);
7534                 start_link(ppd);
7535         }
7536 }
7537
7538 /*
7539  * Handle a link downgrade interrupt from the 8051.
7540  *
7541  * This is a work-queue function outside of the interrupt.
7542  */
7543 void handle_link_downgrade(struct work_struct *work)
7544 {
7545         struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata,
7546                                                         link_downgrade_work);
7547
7548         dd_dev_info(ppd->dd, "8051: Link width downgrade\n");
7549         apply_link_downgrade_policy(ppd, 1);
7550 }
7551
7552 static char *dcc_err_string(char *buf, int buf_len, u64 flags)
7553 {
7554         return flag_string(buf, buf_len, flags, dcc_err_flags,
7555                 ARRAY_SIZE(dcc_err_flags));
7556 }
7557
7558 static char *lcb_err_string(char *buf, int buf_len, u64 flags)
7559 {
7560         return flag_string(buf, buf_len, flags, lcb_err_flags,
7561                 ARRAY_SIZE(lcb_err_flags));
7562 }
7563
7564 static char *dc8051_err_string(char *buf, int buf_len, u64 flags)
7565 {
7566         return flag_string(buf, buf_len, flags, dc8051_err_flags,
7567                 ARRAY_SIZE(dc8051_err_flags));
7568 }
7569
7570 static char *dc8051_info_err_string(char *buf, int buf_len, u64 flags)
7571 {
7572         return flag_string(buf, buf_len, flags, dc8051_info_err_flags,
7573                 ARRAY_SIZE(dc8051_info_err_flags));
7574 }
7575
7576 static char *dc8051_info_host_msg_string(char *buf, int buf_len, u64 flags)
7577 {
7578         return flag_string(buf, buf_len, flags, dc8051_info_host_msg_flags,
7579                 ARRAY_SIZE(dc8051_info_host_msg_flags));
7580 }
7581
7582 static void handle_8051_interrupt(struct hfi1_devdata *dd, u32 unused, u64 reg)
7583 {
7584         struct hfi1_pportdata *ppd = dd->pport;
7585         u64 info, err, host_msg;
7586         int queue_link_down = 0;
7587         char buf[96];
7588
7589         /* look at the flags */
7590         if (reg & DC_DC8051_ERR_FLG_SET_BY_8051_SMASK) {
7591                 /* 8051 information set by firmware */
7592                 /* read DC8051_DBG_ERR_INFO_SET_BY_8051 for details */
7593                 info = read_csr(dd, DC_DC8051_DBG_ERR_INFO_SET_BY_8051);
7594                 err = (info >> DC_DC8051_DBG_ERR_INFO_SET_BY_8051_ERROR_SHIFT)
7595                         & DC_DC8051_DBG_ERR_INFO_SET_BY_8051_ERROR_MASK;
7596                 host_msg = (info >>
7597                         DC_DC8051_DBG_ERR_INFO_SET_BY_8051_HOST_MSG_SHIFT)
7598                         & DC_DC8051_DBG_ERR_INFO_SET_BY_8051_HOST_MSG_MASK;
7599
7600                 /*
7601                  * Handle error flags.
7602                  */
7603                 if (err & FAILED_LNI) {
7604                         /*
7605                          * LNI error indications are cleared by the 8051
7606                          * only when starting polling.  Only pay attention
7607                          * to them when in the states that occur during
7608                          * LNI.
7609                          */
7610                         if (ppd->host_link_state
7611                             & (HLS_DN_POLL | HLS_VERIFY_CAP | HLS_GOING_UP)) {
7612                                 queue_link_down = 1;
7613                                 dd_dev_info(dd, "Link error: %s\n",
7614                                             dc8051_info_err_string(buf,
7615                                                                    sizeof(buf),
7616                                                                    err &
7617                                                                    FAILED_LNI));
7618                         }
7619                         err &= ~(u64)FAILED_LNI;
7620                 }
7621                 /* unknown frames can happen durning LNI, just count */
7622                 if (err & UNKNOWN_FRAME) {
7623                         ppd->unknown_frame_count++;
7624                         err &= ~(u64)UNKNOWN_FRAME;
7625                 }
7626                 if (err) {
7627                         /* report remaining errors, but do not do anything */
7628                         dd_dev_err(dd, "8051 info error: %s\n",
7629                                    dc8051_info_err_string(buf, sizeof(buf),
7630                                                           err));
7631                 }
7632
7633                 /*
7634                  * Handle host message flags.
7635                  */
7636                 if (host_msg & HOST_REQ_DONE) {
7637                         /*
7638                          * Presently, the driver does a busy wait for
7639                          * host requests to complete.  This is only an
7640                          * informational message.
7641                          * NOTE: The 8051 clears the host message
7642                          * information *on the next 8051 command*.
7643                          * Therefore, when linkup is achieved,
7644                          * this flag will still be set.
7645                          */
7646                         host_msg &= ~(u64)HOST_REQ_DONE;
7647                 }
7648                 if (host_msg & BC_SMA_MSG) {
7649                         queue_work(ppd->hfi1_wq, &ppd->sma_message_work);
7650                         host_msg &= ~(u64)BC_SMA_MSG;
7651                 }
7652                 if (host_msg & LINKUP_ACHIEVED) {
7653                         dd_dev_info(dd, "8051: Link up\n");
7654                         queue_work(ppd->hfi1_wq, &ppd->link_up_work);
7655                         host_msg &= ~(u64)LINKUP_ACHIEVED;
7656                 }
7657                 if (host_msg & EXT_DEVICE_CFG_REQ) {
7658                         handle_8051_request(ppd);
7659                         host_msg &= ~(u64)EXT_DEVICE_CFG_REQ;
7660                 }
7661                 if (host_msg & VERIFY_CAP_FRAME) {
7662                         queue_work(ppd->hfi1_wq, &ppd->link_vc_work);
7663                         host_msg &= ~(u64)VERIFY_CAP_FRAME;
7664                 }
7665                 if (host_msg & LINK_GOING_DOWN) {
7666                         const char *extra = "";
7667                         /* no downgrade action needed if going down */
7668                         if (host_msg & LINK_WIDTH_DOWNGRADED) {
7669                                 host_msg &= ~(u64)LINK_WIDTH_DOWNGRADED;
7670                                 extra = " (ignoring downgrade)";
7671                         }
7672                         dd_dev_info(dd, "8051: Link down%s\n", extra);
7673                         queue_link_down = 1;
7674                         host_msg &= ~(u64)LINK_GOING_DOWN;
7675                 }
7676                 if (host_msg & LINK_WIDTH_DOWNGRADED) {
7677                         queue_work(ppd->hfi1_wq, &ppd->link_downgrade_work);
7678                         host_msg &= ~(u64)LINK_WIDTH_DOWNGRADED;
7679                 }
7680                 if (host_msg) {
7681                         /* report remaining messages, but do not do anything */
7682                         dd_dev_info(dd, "8051 info host message: %s\n",
7683                                     dc8051_info_host_msg_string(buf,
7684                                                                 sizeof(buf),
7685                                                                 host_msg));
7686                 }
7687
7688                 reg &= ~DC_DC8051_ERR_FLG_SET_BY_8051_SMASK;
7689         }
7690         if (reg & DC_DC8051_ERR_FLG_LOST_8051_HEART_BEAT_SMASK) {
7691                 /*
7692                  * Lost the 8051 heartbeat.  If this happens, we
7693                  * receive constant interrupts about it.  Disable
7694                  * the interrupt after the first.
7695                  */
7696                 dd_dev_err(dd, "Lost 8051 heartbeat\n");
7697                 write_csr(dd, DC_DC8051_ERR_EN,
7698                           read_csr(dd, DC_DC8051_ERR_EN) &
7699                           ~DC_DC8051_ERR_EN_LOST_8051_HEART_BEAT_SMASK);
7700
7701                 reg &= ~DC_DC8051_ERR_FLG_LOST_8051_HEART_BEAT_SMASK;
7702         }
7703         if (reg) {
7704                 /* report the error, but do not do anything */
7705                 dd_dev_err(dd, "8051 error: %s\n",
7706                            dc8051_err_string(buf, sizeof(buf), reg));
7707         }
7708
7709         if (queue_link_down) {
7710                 /*
7711                  * if the link is already going down or disabled, do not
7712                  * queue another
7713                  */
7714                 if ((ppd->host_link_state &
7715                     (HLS_GOING_OFFLINE | HLS_LINK_COOLDOWN)) ||
7716                     ppd->link_enabled == 0) {
7717                         dd_dev_info(dd, "%s: not queuing link down\n",
7718                                     __func__);
7719                 } else {
7720                         queue_work(ppd->hfi1_wq, &ppd->link_down_work);
7721                 }
7722         }
7723 }
7724
7725 static const char * const fm_config_txt[] = {
7726 [0] =
7727         "BadHeadDist: Distance violation between two head flits",
7728 [1] =
7729         "BadTailDist: Distance violation between two tail flits",
7730 [2] =
7731         "BadCtrlDist: Distance violation between two credit control flits",
7732 [3] =
7733         "BadCrdAck: Credits return for unsupported VL",
7734 [4] =
7735         "UnsupportedVLMarker: Received VL Marker",
7736 [5] =
7737         "BadPreempt: Exceeded the preemption nesting level",
7738 [6] =
7739         "BadControlFlit: Received unsupported control flit",
7740 /* no 7 */
7741 [8] =
7742         "UnsupportedVLMarker: Received VL Marker for unconfigured or disabled VL",
7743 };
7744
7745 static const char * const port_rcv_txt[] = {
7746 [1] =
7747         "BadPktLen: Illegal PktLen",
7748 [2] =
7749         "PktLenTooLong: Packet longer than PktLen",
7750 [3] =
7751         "PktLenTooShort: Packet shorter than PktLen",
7752 [4] =
7753         "BadSLID: Illegal SLID (0, using multicast as SLID, does not include security validation of SLID)",
7754 [5] =
7755         "BadDLID: Illegal DLID (0, doesn't match HFI)",
7756 [6] =
7757         "BadL2: Illegal L2 opcode",
7758 [7] =
7759         "BadSC: Unsupported SC",
7760 [9] =
7761         "BadRC: Illegal RC",
7762 [11] =
7763         "PreemptError: Preempting with same VL",
7764 [12] =
7765         "PreemptVL15: Preempting a VL15 packet",
7766 };
7767
7768 #define OPA_LDR_FMCONFIG_OFFSET 16
7769 #define OPA_LDR_PORTRCV_OFFSET 0
7770 static void handle_dcc_err(struct hfi1_devdata *dd, u32 unused, u64 reg)
7771 {
7772         u64 info, hdr0, hdr1;
7773         const char *extra;
7774         char buf[96];
7775         struct hfi1_pportdata *ppd = dd->pport;
7776         u8 lcl_reason = 0;
7777         int do_bounce = 0;
7778
7779         if (reg & DCC_ERR_FLG_UNCORRECTABLE_ERR_SMASK) {
7780                 if (!(dd->err_info_uncorrectable & OPA_EI_STATUS_SMASK)) {
7781                         info = read_csr(dd, DCC_ERR_INFO_UNCORRECTABLE);
7782                         dd->err_info_uncorrectable = info & OPA_EI_CODE_SMASK;
7783                         /* set status bit */
7784                         dd->err_info_uncorrectable |= OPA_EI_STATUS_SMASK;
7785                 }
7786                 reg &= ~DCC_ERR_FLG_UNCORRECTABLE_ERR_SMASK;
7787         }
7788
7789         if (reg & DCC_ERR_FLG_LINK_ERR_SMASK) {
7790                 struct hfi1_pportdata *ppd = dd->pport;
7791                 /* this counter saturates at (2^32) - 1 */
7792                 if (ppd->link_downed < (u32)UINT_MAX)
7793                         ppd->link_downed++;
7794                 reg &= ~DCC_ERR_FLG_LINK_ERR_SMASK;
7795         }
7796
7797         if (reg & DCC_ERR_FLG_FMCONFIG_ERR_SMASK) {
7798                 u8 reason_valid = 1;
7799
7800                 info = read_csr(dd, DCC_ERR_INFO_FMCONFIG);
7801                 if (!(dd->err_info_fmconfig & OPA_EI_STATUS_SMASK)) {
7802                         dd->err_info_fmconfig = info & OPA_EI_CODE_SMASK;
7803                         /* set status bit */
7804                         dd->err_info_fmconfig |= OPA_EI_STATUS_SMASK;
7805                 }
7806                 switch (info) {
7807                 case 0:
7808                 case 1:
7809                 case 2:
7810                 case 3:
7811                 case 4:
7812                 case 5:
7813                 case 6:
7814                         extra = fm_config_txt[info];
7815                         break;
7816                 case 8:
7817                         extra = fm_config_txt[info];
7818                         if (ppd->port_error_action &
7819                             OPA_PI_MASK_FM_CFG_UNSUPPORTED_VL_MARKER) {
7820                                 do_bounce = 1;
7821                                 /*
7822                                  * lcl_reason cannot be derived from info
7823                                  * for this error
7824                                  */
7825                                 lcl_reason =
7826                                   OPA_LINKDOWN_REASON_UNSUPPORTED_VL_MARKER;
7827                         }
7828                         break;
7829                 default:
7830                         reason_valid = 0;
7831                         snprintf(buf, sizeof(buf), "reserved%lld", info);
7832                         extra = buf;
7833                         break;
7834                 }
7835
7836                 if (reason_valid && !do_bounce) {
7837                         do_bounce = ppd->port_error_action &
7838                                         (1 << (OPA_LDR_FMCONFIG_OFFSET + info));
7839                         lcl_reason = info + OPA_LINKDOWN_REASON_BAD_HEAD_DIST;
7840                 }
7841
7842                 /* just report this */
7843                 dd_dev_info(dd, "DCC Error: fmconfig error: %s\n", extra);
7844                 reg &= ~DCC_ERR_FLG_FMCONFIG_ERR_SMASK;
7845         }
7846
7847         if (reg & DCC_ERR_FLG_RCVPORT_ERR_SMASK) {
7848                 u8 reason_valid = 1;
7849
7850                 info = read_csr(dd, DCC_ERR_INFO_PORTRCV);
7851                 hdr0 = read_csr(dd, DCC_ERR_INFO_PORTRCV_HDR0);
7852                 hdr1 = read_csr(dd, DCC_ERR_INFO_PORTRCV_HDR1);
7853                 if (!(dd->err_info_rcvport.status_and_code &
7854                       OPA_EI_STATUS_SMASK)) {
7855                         dd->err_info_rcvport.status_and_code =
7856                                 info & OPA_EI_CODE_SMASK;
7857                         /* set status bit */
7858                         dd->err_info_rcvport.status_and_code |=
7859                                 OPA_EI_STATUS_SMASK;
7860                         /*
7861                          * save first 2 flits in the packet that caused
7862                          * the error
7863                          */
7864                         dd->err_info_rcvport.packet_flit1 = hdr0;
7865                         dd->err_info_rcvport.packet_flit2 = hdr1;
7866                 }
7867                 switch (info) {
7868                 case 1:
7869                 case 2:
7870                 case 3:
7871                 case 4:
7872                 case 5:
7873                 case 6:
7874                 case 7:
7875                 case 9:
7876                 case 11:
7877                 case 12:
7878                         extra = port_rcv_txt[info];
7879                         break;
7880                 default:
7881                         reason_valid = 0;
7882                         snprintf(buf, sizeof(buf), "reserved%lld", info);
7883                         extra = buf;
7884                         break;
7885                 }
7886
7887                 if (reason_valid && !do_bounce) {
7888                         do_bounce = ppd->port_error_action &
7889                                         (1 << (OPA_LDR_PORTRCV_OFFSET + info));
7890                         lcl_reason = info + OPA_LINKDOWN_REASON_RCV_ERROR_0;
7891                 }
7892
7893                 /* just report this */
7894                 dd_dev_info(dd, "DCC Error: PortRcv error: %s\n", extra);
7895                 dd_dev_info(dd, "           hdr0 0x%llx, hdr1 0x%llx\n",
7896                             hdr0, hdr1);
7897
7898                 reg &= ~DCC_ERR_FLG_RCVPORT_ERR_SMASK;
7899         }
7900
7901         if (reg & DCC_ERR_FLG_EN_CSR_ACCESS_BLOCKED_UC_SMASK) {
7902                 /* informative only */
7903                 dd_dev_info(dd, "8051 access to LCB blocked\n");
7904                 reg &= ~DCC_ERR_FLG_EN_CSR_ACCESS_BLOCKED_UC_SMASK;
7905         }
7906         if (reg & DCC_ERR_FLG_EN_CSR_ACCESS_BLOCKED_HOST_SMASK) {
7907                 /* informative only */
7908                 dd_dev_info(dd, "host access to LCB blocked\n");
7909                 reg &= ~DCC_ERR_FLG_EN_CSR_ACCESS_BLOCKED_HOST_SMASK;
7910         }
7911
7912         /* report any remaining errors */
7913         if (reg)
7914                 dd_dev_info(dd, "DCC Error: %s\n",
7915                             dcc_err_string(buf, sizeof(buf), reg));
7916
7917         if (lcl_reason == 0)
7918                 lcl_reason = OPA_LINKDOWN_REASON_UNKNOWN;
7919
7920         if (do_bounce) {
7921                 dd_dev_info(dd, "%s: PortErrorAction bounce\n", __func__);
7922                 set_link_down_reason(ppd, lcl_reason, 0, lcl_reason);
7923                 queue_work(ppd->hfi1_wq, &ppd->link_bounce_work);
7924         }
7925 }
7926
7927 static void handle_lcb_err(struct hfi1_devdata *dd, u32 unused, u64 reg)
7928 {
7929         char buf[96];
7930
7931         dd_dev_info(dd, "LCB Error: %s\n",
7932                     lcb_err_string(buf, sizeof(buf), reg));
7933 }
7934
7935 /*
7936  * CCE block DC interrupt.  Source is < 8.
7937  */
7938 static void is_dc_int(struct hfi1_devdata *dd, unsigned int source)
7939 {
7940         const struct err_reg_info *eri = &dc_errs[source];
7941
7942         if (eri->handler) {
7943                 interrupt_clear_down(dd, 0, eri);
7944         } else if (source == 3 /* dc_lbm_int */) {
7945                 /*
7946                  * This indicates that a parity error has occurred on the
7947                  * address/control lines presented to the LBM.  The error
7948                  * is a single pulse, there is no associated error flag,
7949                  * and it is non-maskable.  This is because if a parity
7950                  * error occurs on the request the request is dropped.
7951                  * This should never occur, but it is nice to know if it
7952                  * ever does.
7953                  */
7954                 dd_dev_err(dd, "Parity error in DC LBM block\n");
7955         } else {
7956                 dd_dev_err(dd, "Invalid DC interrupt %u\n", source);
7957         }
7958 }
7959
7960 /*
7961  * TX block send credit interrupt.  Source is < 160.
7962  */
7963 static void is_send_credit_int(struct hfi1_devdata *dd, unsigned int source)
7964 {
7965         sc_group_release_update(dd, source);
7966 }
7967
7968 /*
7969  * TX block SDMA interrupt.  Source is < 48.
7970  *
7971  * SDMA interrupts are grouped by type:
7972  *
7973  *       0 -  N-1 = SDma
7974  *       N - 2N-1 = SDmaProgress
7975  *      2N - 3N-1 = SDmaIdle
7976  */
7977 static void is_sdma_eng_int(struct hfi1_devdata *dd, unsigned int source)
7978 {
7979         /* what interrupt */
7980         unsigned int what  = source / TXE_NUM_SDMA_ENGINES;
7981         /* which engine */
7982         unsigned int which = source % TXE_NUM_SDMA_ENGINES;
7983
7984 #ifdef CONFIG_SDMA_VERBOSITY
7985         dd_dev_err(dd, "CONFIG SDMA(%u) %s:%d %s()\n", which,
7986                    slashstrip(__FILE__), __LINE__, __func__);
7987         sdma_dumpstate(&dd->per_sdma[which]);
7988 #endif
7989
7990         if (likely(what < 3 && which < dd->num_sdma)) {
7991                 sdma_engine_interrupt(&dd->per_sdma[which], 1ull << source);
7992         } else {
7993                 /* should not happen */
7994                 dd_dev_err(dd, "Invalid SDMA interrupt 0x%x\n", source);
7995         }
7996 }
7997
7998 /*
7999  * RX block receive available interrupt.  Source is < 160.
8000  */
8001 static void is_rcv_avail_int(struct hfi1_devdata *dd, unsigned int source)
8002 {
8003         struct hfi1_ctxtdata *rcd;
8004         char *err_detail;
8005
8006         if (likely(source < dd->num_rcv_contexts)) {
8007                 rcd = dd->rcd[source];
8008                 if (rcd) {
8009                         if (source < dd->first_user_ctxt)
8010                                 rcd->do_interrupt(rcd, 0);
8011                         else
8012                                 handle_user_interrupt(rcd);
8013                         return; /* OK */
8014                 }
8015                 /* received an interrupt, but no rcd */
8016                 err_detail = "dataless";
8017         } else {
8018                 /* received an interrupt, but are not using that context */
8019                 err_detail = "out of range";
8020         }
8021         dd_dev_err(dd, "unexpected %s receive available context interrupt %u\n",
8022                    err_detail, source);
8023 }
8024
8025 /*
8026  * RX block receive urgent interrupt.  Source is < 160.
8027  */
8028 static void is_rcv_urgent_int(struct hfi1_devdata *dd, unsigned int source)
8029 {
8030         struct hfi1_ctxtdata *rcd;
8031         char *err_detail;
8032
8033         if (likely(source < dd->num_rcv_contexts)) {
8034                 rcd = dd->rcd[source];
8035                 if (rcd) {
8036                         /* only pay attention to user urgent interrupts */
8037                         if (source >= dd->first_user_ctxt)
8038                                 handle_user_interrupt(rcd);
8039                         return; /* OK */
8040                 }
8041                 /* received an interrupt, but no rcd */
8042                 err_detail = "dataless";
8043         } else {
8044                 /* received an interrupt, but are not using that context */
8045                 err_detail = "out of range";
8046         }
8047         dd_dev_err(dd, "unexpected %s receive urgent context interrupt %u\n",
8048                    err_detail, source);
8049 }
8050
8051 /*
8052  * Reserved range interrupt.  Should not be called in normal operation.
8053  */
8054 static void is_reserved_int(struct hfi1_devdata *dd, unsigned int source)
8055 {
8056         char name[64];
8057
8058         dd_dev_err(dd, "unexpected %s interrupt\n",
8059                    is_reserved_name(name, sizeof(name), source));
8060 }
8061
8062 static const struct is_table is_table[] = {
8063 /*
8064  * start                 end
8065  *                              name func               interrupt func
8066  */
8067 { IS_GENERAL_ERR_START,  IS_GENERAL_ERR_END,
8068                                 is_misc_err_name,       is_misc_err_int },
8069 { IS_SDMAENG_ERR_START,  IS_SDMAENG_ERR_END,
8070                                 is_sdma_eng_err_name,   is_sdma_eng_err_int },
8071 { IS_SENDCTXT_ERR_START, IS_SENDCTXT_ERR_END,
8072                                 is_sendctxt_err_name,   is_sendctxt_err_int },
8073 { IS_SDMA_START,             IS_SDMA_END,
8074                                 is_sdma_eng_name,       is_sdma_eng_int },
8075 { IS_VARIOUS_START,          IS_VARIOUS_END,
8076                                 is_various_name,        is_various_int },
8077 { IS_DC_START,       IS_DC_END,
8078                                 is_dc_name,             is_dc_int },
8079 { IS_RCVAVAIL_START,     IS_RCVAVAIL_END,
8080                                 is_rcv_avail_name,      is_rcv_avail_int },
8081 { IS_RCVURGENT_START,    IS_RCVURGENT_END,
8082                                 is_rcv_urgent_name,     is_rcv_urgent_int },
8083 { IS_SENDCREDIT_START,   IS_SENDCREDIT_END,
8084                                 is_send_credit_name,    is_send_credit_int},
8085 { IS_RESERVED_START,     IS_RESERVED_END,
8086                                 is_reserved_name,       is_reserved_int},
8087 };
8088
8089 /*
8090  * Interrupt source interrupt - called when the given source has an interrupt.
8091  * Source is a bit index into an array of 64-bit integers.
8092  */
8093 static void is_interrupt(struct hfi1_devdata *dd, unsigned int source)
8094 {
8095         const struct is_table *entry;
8096
8097         /* avoids a double compare by walking the table in-order */
8098         for (entry = &is_table[0]; entry->is_name; entry++) {
8099                 if (source < entry->end) {
8100                         trace_hfi1_interrupt(dd, entry, source);
8101                         entry->is_int(dd, source - entry->start);
8102                         return;
8103                 }
8104         }
8105         /* fell off the end */
8106         dd_dev_err(dd, "invalid interrupt source %u\n", source);
8107 }
8108
8109 /*
8110  * General interrupt handler.  This is able to correctly handle
8111  * all interrupts in case INTx is used.
8112  */
8113 static irqreturn_t general_interrupt(int irq, void *data)
8114 {
8115         struct hfi1_devdata *dd = data;
8116         u64 regs[CCE_NUM_INT_CSRS];
8117         u32 bit;
8118         int i;
8119
8120         this_cpu_inc(*dd->int_counter);
8121
8122         /* phase 1: scan and clear all handled interrupts */
8123         for (i = 0; i < CCE_NUM_INT_CSRS; i++) {
8124                 if (dd->gi_mask[i] == 0) {
8125                         regs[i] = 0;    /* used later */
8126                         continue;
8127                 }
8128                 regs[i] = read_csr(dd, CCE_INT_STATUS + (8 * i)) &
8129                                 dd->gi_mask[i];
8130                 /* only clear if anything is set */
8131                 if (regs[i])
8132                         write_csr(dd, CCE_INT_CLEAR + (8 * i), regs[i]);
8133         }
8134
8135         /* phase 2: call the appropriate handler */
8136         for_each_set_bit(bit, (unsigned long *)&regs[0],
8137                          CCE_NUM_INT_CSRS * 64) {
8138                 is_interrupt(dd, bit);
8139         }
8140
8141         return IRQ_HANDLED;
8142 }
8143
8144 static irqreturn_t sdma_interrupt(int irq, void *data)
8145 {
8146         struct sdma_engine *sde = data;
8147         struct hfi1_devdata *dd = sde->dd;
8148         u64 status;
8149
8150 #ifdef CONFIG_SDMA_VERBOSITY
8151         dd_dev_err(dd, "CONFIG SDMA(%u) %s:%d %s()\n", sde->this_idx,
8152                    slashstrip(__FILE__), __LINE__, __func__);
8153         sdma_dumpstate(sde);
8154 #endif
8155
8156         this_cpu_inc(*dd->int_counter);
8157
8158         /* This read_csr is really bad in the hot path */
8159         status = read_csr(dd,
8160                           CCE_INT_STATUS + (8 * (IS_SDMA_START / 64)))
8161                           & sde->imask;
8162         if (likely(status)) {
8163                 /* clear the interrupt(s) */
8164                 write_csr(dd,
8165                           CCE_INT_CLEAR + (8 * (IS_SDMA_START / 64)),
8166                           status);
8167
8168                 /* handle the interrupt(s) */
8169                 sdma_engine_interrupt(sde, status);
8170         } else
8171                 dd_dev_err(dd, "SDMA engine %u interrupt, but no status bits set\n",
8172                            sde->this_idx);
8173
8174         return IRQ_HANDLED;
8175 }
8176
8177 /*
8178  * Clear the receive interrupt.  Use a read of the interrupt clear CSR
8179  * to insure that the write completed.  This does NOT guarantee that
8180  * queued DMA writes to memory from the chip are pushed.
8181  */
8182 static inline void clear_recv_intr(struct hfi1_ctxtdata *rcd)
8183 {
8184         struct hfi1_devdata *dd = rcd->dd;
8185         u32 addr = CCE_INT_CLEAR + (8 * rcd->ireg);
8186
8187         mmiowb();       /* make sure everything before is written */
8188         write_csr(dd, addr, rcd->imask);
8189         /* force the above write on the chip and get a value back */
8190         (void)read_csr(dd, addr);
8191 }
8192
8193 /* force the receive interrupt */
8194 void force_recv_intr(struct hfi1_ctxtdata *rcd)
8195 {
8196         write_csr(rcd->dd, CCE_INT_FORCE + (8 * rcd->ireg), rcd->imask);
8197 }
8198
8199 /*
8200  * Return non-zero if a packet is present.
8201  *
8202  * This routine is called when rechecking for packets after the RcvAvail
8203  * interrupt has been cleared down.  First, do a quick check of memory for
8204  * a packet present.  If not found, use an expensive CSR read of the context
8205  * tail to determine the actual tail.  The CSR read is necessary because there
8206  * is no method to push pending DMAs to memory other than an interrupt and we
8207  * are trying to determine if we need to force an interrupt.
8208  */
8209 static inline int check_packet_present(struct hfi1_ctxtdata *rcd)
8210 {
8211         u32 tail;
8212         int present;
8213
8214         if (!rcd->rcvhdrtail_kvaddr)
8215                 present = (rcd->seq_cnt ==
8216                                 rhf_rcv_seq(rhf_to_cpu(get_rhf_addr(rcd))));
8217         else /* is RDMA rtail */
8218                 present = (rcd->head != get_rcvhdrtail(rcd));
8219
8220         if (present)
8221                 return 1;
8222
8223         /* fall back to a CSR read, correct indpendent of DMA_RTAIL */
8224         tail = (u32)read_uctxt_csr(rcd->dd, rcd->ctxt, RCV_HDR_TAIL);
8225         return rcd->head != tail;
8226 }
8227
8228 /*
8229  * Receive packet IRQ handler.  This routine expects to be on its own IRQ.
8230  * This routine will try to handle packets immediately (latency), but if
8231  * it finds too many, it will invoke the thread handler (bandwitdh).  The
8232  * chip receive interrupt is *not* cleared down until this or the thread (if
8233  * invoked) is finished.  The intent is to avoid extra interrupts while we
8234  * are processing packets anyway.
8235  */
8236 static irqreturn_t receive_context_interrupt(int irq, void *data)
8237 {
8238         struct hfi1_ctxtdata *rcd = data;
8239         struct hfi1_devdata *dd = rcd->dd;
8240         int disposition;
8241         int present;
8242
8243         trace_hfi1_receive_interrupt(dd, rcd->ctxt);
8244         this_cpu_inc(*dd->int_counter);
8245         aspm_ctx_disable(rcd);
8246
8247         /* receive interrupt remains blocked while processing packets */
8248         disposition = rcd->do_interrupt(rcd, 0);
8249
8250         /*
8251          * Too many packets were seen while processing packets in this
8252          * IRQ handler.  Invoke the handler thread.  The receive interrupt
8253          * remains blocked.
8254          */
8255         if (disposition == RCV_PKT_LIMIT)
8256                 return IRQ_WAKE_THREAD;
8257
8258         /*
8259          * The packet processor detected no more packets.  Clear the receive
8260          * interrupt and recheck for a packet packet that may have arrived
8261          * after the previous check and interrupt clear.  If a packet arrived,
8262          * force another interrupt.
8263          */
8264         clear_recv_intr(rcd);
8265         present = check_packet_present(rcd);
8266         if (present)
8267                 force_recv_intr(rcd);
8268
8269         return IRQ_HANDLED;
8270 }
8271
8272 /*
8273  * Receive packet thread handler.  This expects to be invoked with the
8274  * receive interrupt still blocked.
8275  */
8276 static irqreturn_t receive_context_thread(int irq, void *data)
8277 {
8278         struct hfi1_ctxtdata *rcd = data;
8279         int present;
8280
8281         /* receive interrupt is still blocked from the IRQ handler */
8282         (void)rcd->do_interrupt(rcd, 1);
8283
8284         /*
8285          * The packet processor will only return if it detected no more
8286          * packets.  Hold IRQs here so we can safely clear the interrupt and
8287          * recheck for a packet that may have arrived after the previous
8288          * check and the interrupt clear.  If a packet arrived, force another
8289          * interrupt.
8290          */
8291         local_irq_disable();
8292         clear_recv_intr(rcd);
8293         present = check_packet_present(rcd);
8294         if (present)
8295                 force_recv_intr(rcd);
8296         local_irq_enable();
8297
8298         return IRQ_HANDLED;
8299 }
8300
8301 /* ========================================================================= */
8302
8303 u32 read_physical_state(struct hfi1_devdata *dd)
8304 {
8305         u64 reg;
8306
8307         reg = read_csr(dd, DC_DC8051_STS_CUR_STATE);
8308         return (reg >> DC_DC8051_STS_CUR_STATE_PORT_SHIFT)
8309                                 & DC_DC8051_STS_CUR_STATE_PORT_MASK;
8310 }
8311
8312 u32 read_logical_state(struct hfi1_devdata *dd)
8313 {
8314         u64 reg;
8315
8316         reg = read_csr(dd, DCC_CFG_PORT_CONFIG);
8317         return (reg >> DCC_CFG_PORT_CONFIG_LINK_STATE_SHIFT)
8318                                 & DCC_CFG_PORT_CONFIG_LINK_STATE_MASK;
8319 }
8320
8321 static void set_logical_state(struct hfi1_devdata *dd, u32 chip_lstate)
8322 {
8323         u64 reg;
8324
8325         reg = read_csr(dd, DCC_CFG_PORT_CONFIG);
8326         /* clear current state, set new state */
8327         reg &= ~DCC_CFG_PORT_CONFIG_LINK_STATE_SMASK;
8328         reg |= (u64)chip_lstate << DCC_CFG_PORT_CONFIG_LINK_STATE_SHIFT;
8329         write_csr(dd, DCC_CFG_PORT_CONFIG, reg);
8330 }
8331
8332 /*
8333  * Use the 8051 to read a LCB CSR.
8334  */
8335 static int read_lcb_via_8051(struct hfi1_devdata *dd, u32 addr, u64 *data)
8336 {
8337         u32 regno;
8338         int ret;
8339
8340         if (dd->icode == ICODE_FUNCTIONAL_SIMULATOR) {
8341                 if (acquire_lcb_access(dd, 0) == 0) {
8342                         *data = read_csr(dd, addr);
8343                         release_lcb_access(dd, 0);
8344                         return 0;
8345                 }
8346                 return -EBUSY;
8347         }
8348
8349         /* register is an index of LCB registers: (offset - base) / 8 */
8350         regno = (addr - DC_LCB_CFG_RUN) >> 3;
8351         ret = do_8051_command(dd, HCMD_READ_LCB_CSR, regno, data);
8352         if (ret != HCMD_SUCCESS)
8353                 return -EBUSY;
8354         return 0;
8355 }
8356
8357 /*
8358  * Read an LCB CSR.  Access may not be in host control, so check.
8359  * Return 0 on success, -EBUSY on failure.
8360  */
8361 int read_lcb_csr(struct hfi1_devdata *dd, u32 addr, u64 *data)
8362 {
8363         struct hfi1_pportdata *ppd = dd->pport;
8364
8365         /* if up, go through the 8051 for the value */
8366         if (ppd->host_link_state & HLS_UP)
8367                 return read_lcb_via_8051(dd, addr, data);
8368         /* if going up or down, no access */
8369         if (ppd->host_link_state & (HLS_GOING_UP | HLS_GOING_OFFLINE))
8370                 return -EBUSY;
8371         /* otherwise, host has access */
8372         *data = read_csr(dd, addr);
8373         return 0;
8374 }
8375
8376 /*
8377  * Use the 8051 to write a LCB CSR.
8378  */
8379 static int write_lcb_via_8051(struct hfi1_devdata *dd, u32 addr, u64 data)
8380 {
8381         u32 regno;
8382         int ret;
8383
8384         if (dd->icode == ICODE_FUNCTIONAL_SIMULATOR ||
8385             (dd->dc8051_ver < dc8051_ver(0, 20))) {
8386                 if (acquire_lcb_access(dd, 0) == 0) {
8387                         write_csr(dd, addr, data);
8388                         release_lcb_access(dd, 0);
8389                         return 0;
8390                 }
8391                 return -EBUSY;
8392         }
8393
8394         /* register is an index of LCB registers: (offset - base) / 8 */
8395         regno = (addr - DC_LCB_CFG_RUN) >> 3;
8396         ret = do_8051_command(dd, HCMD_WRITE_LCB_CSR, regno, &data);
8397         if (ret != HCMD_SUCCESS)
8398                 return -EBUSY;
8399         return 0;
8400 }
8401
8402 /*
8403  * Write an LCB CSR.  Access may not be in host control, so check.
8404  * Return 0 on success, -EBUSY on failure.
8405  */
8406 int write_lcb_csr(struct hfi1_devdata *dd, u32 addr, u64 data)
8407 {
8408         struct hfi1_pportdata *ppd = dd->pport;
8409
8410         /* if up, go through the 8051 for the value */
8411         if (ppd->host_link_state & HLS_UP)
8412                 return write_lcb_via_8051(dd, addr, data);
8413         /* if going up or down, no access */
8414         if (ppd->host_link_state & (HLS_GOING_UP | HLS_GOING_OFFLINE))
8415                 return -EBUSY;
8416         /* otherwise, host has access */
8417         write_csr(dd, addr, data);
8418         return 0;
8419 }
8420
8421 /*
8422  * Returns:
8423  *      < 0 = Linux error, not able to get access
8424  *      > 0 = 8051 command RETURN_CODE
8425  */
8426 static int do_8051_command(
8427         struct hfi1_devdata *dd,
8428         u32 type,
8429         u64 in_data,
8430         u64 *out_data)
8431 {
8432         u64 reg, completed;
8433         int return_code;
8434         unsigned long timeout;
8435
8436         hfi1_cdbg(DC8051, "type %d, data 0x%012llx", type, in_data);
8437
8438         mutex_lock(&dd->dc8051_lock);
8439
8440         /* We can't send any commands to the 8051 if it's in reset */
8441         if (dd->dc_shutdown) {
8442                 return_code = -ENODEV;
8443                 goto fail;
8444         }
8445
8446         /*
8447          * If an 8051 host command timed out previously, then the 8051 is
8448          * stuck.
8449          *
8450          * On first timeout, attempt to reset and restart the entire DC
8451          * block (including 8051). (Is this too big of a hammer?)
8452          *
8453          * If the 8051 times out a second time, the reset did not bring it
8454          * back to healthy life. In that case, fail any subsequent commands.
8455          */
8456         if (dd->dc8051_timed_out) {
8457                 if (dd->dc8051_timed_out > 1) {
8458                         dd_dev_err(dd,
8459                                    "Previous 8051 host command timed out, skipping command %u\n",
8460                                    type);
8461                         return_code = -ENXIO;
8462                         goto fail;
8463                 }
8464                 _dc_shutdown(dd);
8465                 _dc_start(dd);
8466         }
8467
8468         /*
8469          * If there is no timeout, then the 8051 command interface is
8470          * waiting for a command.
8471          */
8472
8473         /*
8474          * When writing a LCB CSR, out_data contains the full value to
8475          * to be written, while in_data contains the relative LCB
8476          * address in 7:0.  Do the work here, rather than the caller,
8477          * of distrubting the write data to where it needs to go:
8478          *
8479          * Write data
8480          *   39:00 -> in_data[47:8]
8481          *   47:40 -> DC8051_CFG_EXT_DEV_0.RETURN_CODE
8482          *   63:48 -> DC8051_CFG_EXT_DEV_0.RSP_DATA
8483          */
8484         if (type == HCMD_WRITE_LCB_CSR) {
8485                 in_data |= ((*out_data) & 0xffffffffffull) << 8;
8486                 reg = ((((*out_data) >> 40) & 0xff) <<
8487                                 DC_DC8051_CFG_EXT_DEV_0_RETURN_CODE_SHIFT)
8488                       | ((((*out_data) >> 48) & 0xffff) <<
8489                                 DC_DC8051_CFG_EXT_DEV_0_RSP_DATA_SHIFT);
8490                 write_csr(dd, DC_DC8051_CFG_EXT_DEV_0, reg);
8491         }
8492
8493         /*
8494          * Do two writes: the first to stabilize the type and req_data, the
8495          * second to activate.
8496          */
8497         reg = ((u64)type & DC_DC8051_CFG_HOST_CMD_0_REQ_TYPE_MASK)
8498                         << DC_DC8051_CFG_HOST_CMD_0_REQ_TYPE_SHIFT
8499                 | (in_data & DC_DC8051_CFG_HOST_CMD_0_REQ_DATA_MASK)
8500                         << DC_DC8051_CFG_HOST_CMD_0_REQ_DATA_SHIFT;
8501         write_csr(dd, DC_DC8051_CFG_HOST_CMD_0, reg);
8502         reg |= DC_DC8051_CFG_HOST_CMD_0_REQ_NEW_SMASK;
8503         write_csr(dd, DC_DC8051_CFG_HOST_CMD_0, reg);
8504
8505         /* wait for completion, alternate: interrupt */
8506         timeout = jiffies + msecs_to_jiffies(DC8051_COMMAND_TIMEOUT);
8507         while (1) {
8508                 reg = read_csr(dd, DC_DC8051_CFG_HOST_CMD_1);
8509                 completed = reg & DC_DC8051_CFG_HOST_CMD_1_COMPLETED_SMASK;
8510                 if (completed)
8511                         break;
8512                 if (time_after(jiffies, timeout)) {
8513                         dd->dc8051_timed_out++;
8514                         dd_dev_err(dd, "8051 host command %u timeout\n", type);
8515                         if (out_data)
8516                                 *out_data = 0;
8517                         return_code = -ETIMEDOUT;
8518                         goto fail;
8519                 }
8520                 udelay(2);
8521         }
8522
8523         if (out_data) {
8524                 *out_data = (reg >> DC_DC8051_CFG_HOST_CMD_1_RSP_DATA_SHIFT)
8525                                 & DC_DC8051_CFG_HOST_CMD_1_RSP_DATA_MASK;
8526                 if (type == HCMD_READ_LCB_CSR) {
8527                         /* top 16 bits are in a different register */
8528                         *out_data |= (read_csr(dd, DC_DC8051_CFG_EXT_DEV_1)
8529                                 & DC_DC8051_CFG_EXT_DEV_1_REQ_DATA_SMASK)
8530                                 << (48
8531                                     - DC_DC8051_CFG_EXT_DEV_1_REQ_DATA_SHIFT);
8532                 }
8533         }
8534         return_code = (reg >> DC_DC8051_CFG_HOST_CMD_1_RETURN_CODE_SHIFT)
8535                                 & DC_DC8051_CFG_HOST_CMD_1_RETURN_CODE_MASK;
8536         dd->dc8051_timed_out = 0;
8537         /*
8538          * Clear command for next user.
8539          */
8540         write_csr(dd, DC_DC8051_CFG_HOST_CMD_0, 0);
8541
8542 fail:
8543         mutex_unlock(&dd->dc8051_lock);
8544         return return_code;
8545 }
8546
8547 static int set_physical_link_state(struct hfi1_devdata *dd, u64 state)
8548 {
8549         return do_8051_command(dd, HCMD_CHANGE_PHY_STATE, state, NULL);
8550 }
8551
8552 int load_8051_config(struct hfi1_devdata *dd, u8 field_id,
8553                      u8 lane_id, u32 config_data)
8554 {
8555         u64 data;
8556         int ret;
8557
8558         data = (u64)field_id << LOAD_DATA_FIELD_ID_SHIFT
8559                 | (u64)lane_id << LOAD_DATA_LANE_ID_SHIFT
8560                 | (u64)config_data << LOAD_DATA_DATA_SHIFT;
8561         ret = do_8051_command(dd, HCMD_LOAD_CONFIG_DATA, data, NULL);
8562         if (ret != HCMD_SUCCESS) {
8563                 dd_dev_err(dd,
8564                            "load 8051 config: field id %d, lane %d, err %d\n",
8565                            (int)field_id, (int)lane_id, ret);
8566         }
8567         return ret;
8568 }
8569
8570 /*
8571  * Read the 8051 firmware "registers".  Use the RAM directly.  Always
8572  * set the result, even on error.
8573  * Return 0 on success, -errno on failure
8574  */
8575 int read_8051_config(struct hfi1_devdata *dd, u8 field_id, u8 lane_id,
8576                      u32 *result)
8577 {
8578         u64 big_data;
8579         u32 addr;
8580         int ret;
8581
8582         /* address start depends on the lane_id */
8583         if (lane_id < 4)
8584                 addr = (4 * NUM_GENERAL_FIELDS)
8585                         + (lane_id * 4 * NUM_LANE_FIELDS);
8586         else
8587                 addr = 0;
8588         addr += field_id * 4;
8589
8590         /* read is in 8-byte chunks, hardware will truncate the address down */
8591         ret = read_8051_data(dd, addr, 8, &big_data);
8592
8593         if (ret == 0) {
8594                 /* extract the 4 bytes we want */
8595                 if (addr & 0x4)
8596                         *result = (u32)(big_data >> 32);
8597                 else
8598                         *result = (u32)big_data;
8599         } else {
8600                 *result = 0;
8601                 dd_dev_err(dd, "%s: direct read failed, lane %d, field %d!\n",
8602                            __func__, lane_id, field_id);
8603         }
8604
8605         return ret;
8606 }
8607
8608 static int write_vc_local_phy(struct hfi1_devdata *dd, u8 power_management,
8609                               u8 continuous)
8610 {
8611         u32 frame;
8612
8613         frame = continuous << CONTINIOUS_REMOTE_UPDATE_SUPPORT_SHIFT
8614                 | power_management << POWER_MANAGEMENT_SHIFT;
8615         return load_8051_config(dd, VERIFY_CAP_LOCAL_PHY,
8616                                 GENERAL_CONFIG, frame);
8617 }
8618
8619 static int write_vc_local_fabric(struct hfi1_devdata *dd, u8 vau, u8 z, u8 vcu,
8620                                  u16 vl15buf, u8 crc_sizes)
8621 {
8622         u32 frame;
8623
8624         frame = (u32)vau << VAU_SHIFT
8625                 | (u32)z << Z_SHIFT
8626                 | (u32)vcu << VCU_SHIFT
8627                 | (u32)vl15buf << VL15BUF_SHIFT
8628                 | (u32)crc_sizes << CRC_SIZES_SHIFT;
8629         return load_8051_config(dd, VERIFY_CAP_LOCAL_FABRIC,
8630                                 GENERAL_CONFIG, frame);
8631 }
8632
8633 static void read_vc_local_link_width(struct hfi1_devdata *dd, u8 *misc_bits,
8634                                      u8 *flag_bits, u16 *link_widths)
8635 {
8636         u32 frame;
8637
8638         read_8051_config(dd, VERIFY_CAP_LOCAL_LINK_WIDTH, GENERAL_CONFIG,
8639                          &frame);
8640         *misc_bits = (frame >> MISC_CONFIG_BITS_SHIFT) & MISC_CONFIG_BITS_MASK;
8641         *flag_bits = (frame >> LOCAL_FLAG_BITS_SHIFT) & LOCAL_FLAG_BITS_MASK;
8642         *link_widths = (frame >> LINK_WIDTH_SHIFT) & LINK_WIDTH_MASK;
8643 }
8644
8645 static int write_vc_local_link_width(struct hfi1_devdata *dd,
8646                                      u8 misc_bits,
8647                                      u8 flag_bits,
8648                                      u16 link_widths)
8649 {
8650         u32 frame;
8651
8652         frame = (u32)misc_bits << MISC_CONFIG_BITS_SHIFT
8653                 | (u32)flag_bits << LOCAL_FLAG_BITS_SHIFT
8654                 | (u32)link_widths << LINK_WIDTH_SHIFT;
8655         return load_8051_config(dd, VERIFY_CAP_LOCAL_LINK_WIDTH, GENERAL_CONFIG,
8656                      frame);
8657 }
8658
8659 static int write_local_device_id(struct hfi1_devdata *dd, u16 device_id,
8660                                  u8 device_rev)
8661 {
8662         u32 frame;
8663
8664         frame = ((u32)device_id << LOCAL_DEVICE_ID_SHIFT)
8665                 | ((u32)device_rev << LOCAL_DEVICE_REV_SHIFT);
8666         return load_8051_config(dd, LOCAL_DEVICE_ID, GENERAL_CONFIG, frame);
8667 }
8668
8669 static void read_remote_device_id(struct hfi1_devdata *dd, u16 *device_id,
8670                                   u8 *device_rev)
8671 {
8672         u32 frame;
8673
8674         read_8051_config(dd, REMOTE_DEVICE_ID, GENERAL_CONFIG, &frame);
8675         *device_id = (frame >> REMOTE_DEVICE_ID_SHIFT) & REMOTE_DEVICE_ID_MASK;
8676         *device_rev = (frame >> REMOTE_DEVICE_REV_SHIFT)
8677                         & REMOTE_DEVICE_REV_MASK;
8678 }
8679
8680 void read_misc_status(struct hfi1_devdata *dd, u8 *ver_a, u8 *ver_b)
8681 {
8682         u32 frame;
8683
8684         read_8051_config(dd, MISC_STATUS, GENERAL_CONFIG, &frame);
8685         *ver_a = (frame >> STS_FM_VERSION_A_SHIFT) & STS_FM_VERSION_A_MASK;
8686         *ver_b = (frame >> STS_FM_VERSION_B_SHIFT) & STS_FM_VERSION_B_MASK;
8687 }
8688
8689 static void read_vc_remote_phy(struct hfi1_devdata *dd, u8 *power_management,
8690                                u8 *continuous)
8691 {
8692         u32 frame;
8693
8694         read_8051_config(dd, VERIFY_CAP_REMOTE_PHY, GENERAL_CONFIG, &frame);
8695         *power_management = (frame >> POWER_MANAGEMENT_SHIFT)
8696                                         & POWER_MANAGEMENT_MASK;
8697         *continuous = (frame >> CONTINIOUS_REMOTE_UPDATE_SUPPORT_SHIFT)
8698                                         & CONTINIOUS_REMOTE_UPDATE_SUPPORT_MASK;
8699 }
8700
8701 static void read_vc_remote_fabric(struct hfi1_devdata *dd, u8 *vau, u8 *z,
8702                                   u8 *vcu, u16 *vl15buf, u8 *crc_sizes)
8703 {
8704         u32 frame;
8705
8706         read_8051_config(dd, VERIFY_CAP_REMOTE_FABRIC, GENERAL_CONFIG, &frame);
8707         *vau = (frame >> VAU_SHIFT) & VAU_MASK;
8708         *z = (frame >> Z_SHIFT) & Z_MASK;
8709         *vcu = (frame >> VCU_SHIFT) & VCU_MASK;
8710         *vl15buf = (frame >> VL15BUF_SHIFT) & VL15BUF_MASK;
8711         *crc_sizes = (frame >> CRC_SIZES_SHIFT) & CRC_SIZES_MASK;
8712 }
8713
8714 static void read_vc_remote_link_width(struct hfi1_devdata *dd,
8715                                       u8 *remote_tx_rate,
8716                                       u16 *link_widths)
8717 {
8718         u32 frame;
8719
8720         read_8051_config(dd, VERIFY_CAP_REMOTE_LINK_WIDTH, GENERAL_CONFIG,
8721                          &frame);
8722         *remote_tx_rate = (frame >> REMOTE_TX_RATE_SHIFT)
8723                                 & REMOTE_TX_RATE_MASK;
8724         *link_widths = (frame >> LINK_WIDTH_SHIFT) & LINK_WIDTH_MASK;
8725 }
8726
8727 static void read_local_lni(struct hfi1_devdata *dd, u8 *enable_lane_rx)
8728 {
8729         u32 frame;
8730
8731         read_8051_config(dd, LOCAL_LNI_INFO, GENERAL_CONFIG, &frame);
8732         *enable_lane_rx = (frame >> ENABLE_LANE_RX_SHIFT) & ENABLE_LANE_RX_MASK;
8733 }
8734
8735 static void read_mgmt_allowed(struct hfi1_devdata *dd, u8 *mgmt_allowed)
8736 {
8737         u32 frame;
8738
8739         read_8051_config(dd, REMOTE_LNI_INFO, GENERAL_CONFIG, &frame);
8740         *mgmt_allowed = (frame >> MGMT_ALLOWED_SHIFT) & MGMT_ALLOWED_MASK;
8741 }
8742
8743 static void read_last_local_state(struct hfi1_devdata *dd, u32 *lls)
8744 {
8745         read_8051_config(dd, LAST_LOCAL_STATE_COMPLETE, GENERAL_CONFIG, lls);
8746 }
8747
8748 static void read_last_remote_state(struct hfi1_devdata *dd, u32 *lrs)
8749 {
8750         read_8051_config(dd, LAST_REMOTE_STATE_COMPLETE, GENERAL_CONFIG, lrs);
8751 }
8752
8753 void hfi1_read_link_quality(struct hfi1_devdata *dd, u8 *link_quality)
8754 {
8755         u32 frame;
8756         int ret;
8757
8758         *link_quality = 0;
8759         if (dd->pport->host_link_state & HLS_UP) {
8760                 ret = read_8051_config(dd, LINK_QUALITY_INFO, GENERAL_CONFIG,
8761                                        &frame);
8762                 if (ret == 0)
8763                         *link_quality = (frame >> LINK_QUALITY_SHIFT)
8764                                                 & LINK_QUALITY_MASK;
8765         }
8766 }
8767
8768 static void read_planned_down_reason_code(struct hfi1_devdata *dd, u8 *pdrrc)
8769 {
8770         u32 frame;
8771
8772         read_8051_config(dd, LINK_QUALITY_INFO, GENERAL_CONFIG, &frame);
8773         *pdrrc = (frame >> DOWN_REMOTE_REASON_SHIFT) & DOWN_REMOTE_REASON_MASK;
8774 }
8775
8776 static void read_link_down_reason(struct hfi1_devdata *dd, u8 *ldr)
8777 {
8778         u32 frame;
8779
8780         read_8051_config(dd, LINK_DOWN_REASON, GENERAL_CONFIG, &frame);
8781         *ldr = (frame & 0xff);
8782 }
8783
8784 static int read_tx_settings(struct hfi1_devdata *dd,
8785                             u8 *enable_lane_tx,
8786                             u8 *tx_polarity_inversion,
8787                             u8 *rx_polarity_inversion,
8788                             u8 *max_rate)
8789 {
8790         u32 frame;
8791         int ret;
8792
8793         ret = read_8051_config(dd, TX_SETTINGS, GENERAL_CONFIG, &frame);
8794         *enable_lane_tx = (frame >> ENABLE_LANE_TX_SHIFT)
8795                                 & ENABLE_LANE_TX_MASK;
8796         *tx_polarity_inversion = (frame >> TX_POLARITY_INVERSION_SHIFT)
8797                                 & TX_POLARITY_INVERSION_MASK;
8798         *rx_polarity_inversion = (frame >> RX_POLARITY_INVERSION_SHIFT)
8799                                 & RX_POLARITY_INVERSION_MASK;
8800         *max_rate = (frame >> MAX_RATE_SHIFT) & MAX_RATE_MASK;
8801         return ret;
8802 }
8803
8804 static int write_tx_settings(struct hfi1_devdata *dd,
8805                              u8 enable_lane_tx,
8806                              u8 tx_polarity_inversion,
8807                              u8 rx_polarity_inversion,
8808                              u8 max_rate)
8809 {
8810         u32 frame;
8811
8812         /* no need to mask, all variable sizes match field widths */
8813         frame = enable_lane_tx << ENABLE_LANE_TX_SHIFT
8814                 | tx_polarity_inversion << TX_POLARITY_INVERSION_SHIFT
8815                 | rx_polarity_inversion << RX_POLARITY_INVERSION_SHIFT
8816                 | max_rate << MAX_RATE_SHIFT;
8817         return load_8051_config(dd, TX_SETTINGS, GENERAL_CONFIG, frame);
8818 }
8819
8820 /*
8821  * Read an idle LCB message.
8822  *
8823  * Returns 0 on success, -EINVAL on error
8824  */
8825 static int read_idle_message(struct hfi1_devdata *dd, u64 type, u64 *data_out)
8826 {
8827         int ret;
8828
8829         ret = do_8051_command(dd, HCMD_READ_LCB_IDLE_MSG, type, data_out);
8830         if (ret != HCMD_SUCCESS) {
8831                 dd_dev_err(dd, "read idle message: type %d, err %d\n",
8832                            (u32)type, ret);
8833                 return -EINVAL;
8834         }
8835         dd_dev_info(dd, "%s: read idle message 0x%llx\n", __func__, *data_out);
8836         /* return only the payload as we already know the type */
8837         *data_out >>= IDLE_PAYLOAD_SHIFT;
8838         return 0;
8839 }
8840
8841 /*
8842  * Read an idle SMA message.  To be done in response to a notification from
8843  * the 8051.
8844  *
8845  * Returns 0 on success, -EINVAL on error
8846  */
8847 static int read_idle_sma(struct hfi1_devdata *dd, u64 *data)
8848 {
8849         return read_idle_message(dd, (u64)IDLE_SMA << IDLE_MSG_TYPE_SHIFT,
8850                                  data);
8851 }
8852
8853 /*
8854  * Send an idle LCB message.
8855  *
8856  * Returns 0 on success, -EINVAL on error
8857  */
8858 static int send_idle_message(struct hfi1_devdata *dd, u64 data)
8859 {
8860         int ret;
8861
8862         dd_dev_info(dd, "%s: sending idle message 0x%llx\n", __func__, data);
8863         ret = do_8051_command(dd, HCMD_SEND_LCB_IDLE_MSG, data, NULL);
8864         if (ret != HCMD_SUCCESS) {
8865                 dd_dev_err(dd, "send idle message: data 0x%llx, err %d\n",
8866                            data, ret);
8867                 return -EINVAL;
8868         }
8869         return 0;
8870 }
8871
8872 /*
8873  * Send an idle SMA message.
8874  *
8875  * Returns 0 on success, -EINVAL on error
8876  */
8877 int send_idle_sma(struct hfi1_devdata *dd, u64 message)
8878 {
8879         u64 data;
8880
8881         data = ((message & IDLE_PAYLOAD_MASK) << IDLE_PAYLOAD_SHIFT) |
8882                 ((u64)IDLE_SMA << IDLE_MSG_TYPE_SHIFT);
8883         return send_idle_message(dd, data);
8884 }
8885
8886 /*
8887  * Initialize the LCB then do a quick link up.  This may or may not be
8888  * in loopback.
8889  *
8890  * return 0 on success, -errno on error
8891  */
8892 static int do_quick_linkup(struct hfi1_devdata *dd)
8893 {
8894         u64 reg;
8895         unsigned long timeout;
8896         int ret;
8897
8898         lcb_shutdown(dd, 0);
8899
8900         if (loopback) {
8901                 /* LCB_CFG_LOOPBACK.VAL = 2 */
8902                 /* LCB_CFG_LANE_WIDTH.VAL = 0 */
8903                 write_csr(dd, DC_LCB_CFG_LOOPBACK,
8904                           IB_PACKET_TYPE << DC_LCB_CFG_LOOPBACK_VAL_SHIFT);
8905                 write_csr(dd, DC_LCB_CFG_LANE_WIDTH, 0);
8906         }
8907
8908         /* start the LCBs */
8909         /* LCB_CFG_TX_FIFOS_RESET.VAL = 0 */
8910         write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET, 0);
8911
8912         /* simulator only loopback steps */
8913         if (loopback && dd->icode == ICODE_FUNCTIONAL_SIMULATOR) {
8914                 /* LCB_CFG_RUN.EN = 1 */
8915                 write_csr(dd, DC_LCB_CFG_RUN,
8916                           1ull << DC_LCB_CFG_RUN_EN_SHIFT);
8917
8918                 /* watch LCB_STS_LINK_TRANSFER_ACTIVE */
8919                 timeout = jiffies + msecs_to_jiffies(10);
8920                 while (1) {
8921                         reg = read_csr(dd, DC_LCB_STS_LINK_TRANSFER_ACTIVE);
8922                         if (reg)
8923                                 break;
8924                         if (time_after(jiffies, timeout)) {
8925                                 dd_dev_err(dd,
8926                                            "timeout waiting for LINK_TRANSFER_ACTIVE\n");
8927                                 return -ETIMEDOUT;
8928                         }
8929                         udelay(2);
8930                 }
8931
8932                 write_csr(dd, DC_LCB_CFG_ALLOW_LINK_UP,
8933                           1ull << DC_LCB_CFG_ALLOW_LINK_UP_VAL_SHIFT);
8934         }
8935
8936         if (!loopback) {
8937                 /*
8938                  * When doing quick linkup and not in loopback, both
8939                  * sides must be done with LCB set-up before either
8940                  * starts the quick linkup.  Put a delay here so that
8941                  * both sides can be started and have a chance to be
8942                  * done with LCB set up before resuming.
8943                  */
8944                 dd_dev_err(dd,
8945                            "Pausing for peer to be finished with LCB set up\n");
8946                 msleep(5000);
8947                 dd_dev_err(dd, "Continuing with quick linkup\n");
8948         }
8949
8950         write_csr(dd, DC_LCB_ERR_EN, 0); /* mask LCB errors */
8951         set_8051_lcb_access(dd);
8952
8953         /*
8954          * State "quick" LinkUp request sets the physical link state to
8955          * LinkUp without a verify capability sequence.
8956          * This state is in simulator v37 and later.
8957          */
8958         ret = set_physical_link_state(dd, PLS_QUICK_LINKUP);
8959         if (ret != HCMD_SUCCESS) {
8960                 dd_dev_err(dd,
8961                            "%s: set physical link state to quick LinkUp failed with return %d\n",
8962                            __func__, ret);
8963
8964                 set_host_lcb_access(dd);
8965                 write_csr(dd, DC_LCB_ERR_EN, ~0ull); /* watch LCB errors */
8966
8967                 if (ret >= 0)
8968                         ret = -EINVAL;
8969                 return ret;
8970         }
8971
8972         return 0; /* success */
8973 }
8974
8975 /*
8976  * Set the SerDes to internal loopback mode.
8977  * Returns 0 on success, -errno on error.
8978  */
8979 static int set_serdes_loopback_mode(struct hfi1_devdata *dd)
8980 {
8981         int ret;
8982
8983         ret = set_physical_link_state(dd, PLS_INTERNAL_SERDES_LOOPBACK);
8984         if (ret == HCMD_SUCCESS)
8985                 return 0;
8986         dd_dev_err(dd,
8987                    "Set physical link state to SerDes Loopback failed with return %d\n",
8988                    ret);
8989         if (ret >= 0)
8990                 ret = -EINVAL;
8991         return ret;
8992 }
8993
8994 /*
8995  * Do all special steps to set up loopback.
8996  */
8997 static int init_loopback(struct hfi1_devdata *dd)
8998 {
8999         dd_dev_info(dd, "Entering loopback mode\n");
9000
9001         /* all loopbacks should disable self GUID check */
9002         write_csr(dd, DC_DC8051_CFG_MODE,
9003                   (read_csr(dd, DC_DC8051_CFG_MODE) | DISABLE_SELF_GUID_CHECK));
9004
9005         /*
9006          * The simulator has only one loopback option - LCB.  Switch
9007          * to that option, which includes quick link up.
9008          *
9009          * Accept all valid loopback values.
9010          */
9011         if ((dd->icode == ICODE_FUNCTIONAL_SIMULATOR) &&
9012             (loopback == LOOPBACK_SERDES || loopback == LOOPBACK_LCB ||
9013              loopback == LOOPBACK_CABLE)) {
9014                 loopback = LOOPBACK_LCB;
9015                 quick_linkup = 1;
9016                 return 0;
9017         }
9018
9019         /* handle serdes loopback */
9020         if (loopback == LOOPBACK_SERDES) {
9021                 /* internal serdes loopack needs quick linkup on RTL */
9022                 if (dd->icode == ICODE_RTL_SILICON)
9023                         quick_linkup = 1;
9024                 return set_serdes_loopback_mode(dd);
9025         }
9026
9027         /* LCB loopback - handled at poll time */
9028         if (loopback == LOOPBACK_LCB) {
9029                 quick_linkup = 1; /* LCB is always quick linkup */
9030
9031                 /* not supported in emulation due to emulation RTL changes */
9032                 if (dd->icode == ICODE_FPGA_EMULATION) {
9033                         dd_dev_err(dd,
9034                                    "LCB loopback not supported in emulation\n");
9035                         return -EINVAL;
9036                 }
9037                 return 0;
9038         }
9039
9040         /* external cable loopback requires no extra steps */
9041         if (loopback == LOOPBACK_CABLE)
9042                 return 0;
9043
9044         dd_dev_err(dd, "Invalid loopback mode %d\n", loopback);
9045         return -EINVAL;
9046 }
9047
9048 /*
9049  * Translate from the OPA_LINK_WIDTH handed to us by the FM to bits
9050  * used in the Verify Capability link width attribute.
9051  */
9052 static u16 opa_to_vc_link_widths(u16 opa_widths)
9053 {
9054         int i;
9055         u16 result = 0;
9056
9057         static const struct link_bits {
9058                 u16 from;
9059                 u16 to;
9060         } opa_link_xlate[] = {
9061                 { OPA_LINK_WIDTH_1X, 1 << (1 - 1)  },
9062                 { OPA_LINK_WIDTH_2X, 1 << (2 - 1)  },
9063                 { OPA_LINK_WIDTH_3X, 1 << (3 - 1)  },
9064                 { OPA_LINK_WIDTH_4X, 1 << (4 - 1)  },
9065         };
9066
9067         for (i = 0; i < ARRAY_SIZE(opa_link_xlate); i++) {
9068                 if (opa_widths & opa_link_xlate[i].from)
9069                         result |= opa_link_xlate[i].to;
9070         }
9071         return result;
9072 }
9073
9074 /*
9075  * Set link attributes before moving to polling.
9076  */
9077 static int set_local_link_attributes(struct hfi1_pportdata *ppd)
9078 {
9079         struct hfi1_devdata *dd = ppd->dd;
9080         u8 enable_lane_tx;
9081         u8 tx_polarity_inversion;
9082         u8 rx_polarity_inversion;
9083         int ret;
9084
9085         /* reset our fabric serdes to clear any lingering problems */
9086         fabric_serdes_reset(dd);
9087
9088         /* set the local tx rate - need to read-modify-write */
9089         ret = read_tx_settings(dd, &enable_lane_tx, &tx_polarity_inversion,
9090                                &rx_polarity_inversion, &ppd->local_tx_rate);
9091         if (ret)
9092                 goto set_local_link_attributes_fail;
9093
9094         if (dd->dc8051_ver < dc8051_ver(0, 20)) {
9095                 /* set the tx rate to the fastest enabled */
9096                 if (ppd->link_speed_enabled & OPA_LINK_SPEED_25G)
9097                         ppd->local_tx_rate = 1;
9098                 else
9099                         ppd->local_tx_rate = 0;
9100         } else {
9101                 /* set the tx rate to all enabled */
9102                 ppd->local_tx_rate = 0;
9103                 if (ppd->link_speed_enabled & OPA_LINK_SPEED_25G)
9104                         ppd->local_tx_rate |= 2;
9105                 if (ppd->link_speed_enabled & OPA_LINK_SPEED_12_5G)
9106                         ppd->local_tx_rate |= 1;
9107         }
9108
9109         enable_lane_tx = 0xF; /* enable all four lanes */
9110         ret = write_tx_settings(dd, enable_lane_tx, tx_polarity_inversion,
9111                                 rx_polarity_inversion, ppd->local_tx_rate);
9112         if (ret != HCMD_SUCCESS)
9113                 goto set_local_link_attributes_fail;
9114
9115         /*
9116          * DC supports continuous updates.
9117          */
9118         ret = write_vc_local_phy(dd,
9119                                  0 /* no power management */,
9120                                  1 /* continuous updates */);
9121         if (ret != HCMD_SUCCESS)
9122                 goto set_local_link_attributes_fail;
9123
9124         /* z=1 in the next call: AU of 0 is not supported by the hardware */
9125         ret = write_vc_local_fabric(dd, dd->vau, 1, dd->vcu, dd->vl15_init,
9126                                     ppd->port_crc_mode_enabled);
9127         if (ret != HCMD_SUCCESS)
9128                 goto set_local_link_attributes_fail;
9129
9130         ret = write_vc_local_link_width(dd, 0, 0,
9131                                         opa_to_vc_link_widths(
9132                                                 ppd->link_width_enabled));
9133         if (ret != HCMD_SUCCESS)
9134                 goto set_local_link_attributes_fail;
9135
9136         /* let peer know who we are */
9137         ret = write_local_device_id(dd, dd->pcidev->device, dd->minrev);
9138         if (ret == HCMD_SUCCESS)
9139                 return 0;
9140
9141 set_local_link_attributes_fail:
9142         dd_dev_err(dd,
9143                    "Failed to set local link attributes, return 0x%x\n",
9144                    ret);
9145         return ret;
9146 }
9147
9148 /*
9149  * Call this to start the link.
9150  * Do not do anything if the link is disabled.
9151  * Returns 0 if link is disabled, moved to polling, or the driver is not ready.
9152  */
9153 int start_link(struct hfi1_pportdata *ppd)
9154 {
9155         /*
9156          * Tune the SerDes to a ballpark setting for optimal signal and bit
9157          * error rate.  Needs to be done before starting the link.
9158          */
9159         tune_serdes(ppd);
9160
9161         if (!ppd->link_enabled) {
9162                 dd_dev_info(ppd->dd,
9163                             "%s: stopping link start because link is disabled\n",
9164                             __func__);
9165                 return 0;
9166         }
9167         if (!ppd->driver_link_ready) {
9168                 dd_dev_info(ppd->dd,
9169                             "%s: stopping link start because driver is not ready\n",
9170                             __func__);
9171                 return 0;
9172         }
9173
9174         /*
9175          * FULL_MGMT_P_KEY is cleared from the pkey table, so that the
9176          * pkey table can be configured properly if the HFI unit is connected
9177          * to switch port with MgmtAllowed=NO
9178          */
9179         clear_full_mgmt_pkey(ppd);
9180
9181         return set_link_state(ppd, HLS_DN_POLL);
9182 }
9183
9184 static void wait_for_qsfp_init(struct hfi1_pportdata *ppd)
9185 {
9186         struct hfi1_devdata *dd = ppd->dd;
9187         u64 mask;
9188         unsigned long timeout;
9189
9190         /*
9191          * Some QSFP cables have a quirk that asserts the IntN line as a side
9192          * effect of power up on plug-in. We ignore this false positive
9193          * interrupt until the module has finished powering up by waiting for
9194          * a minimum timeout of the module inrush initialization time of
9195          * 500 ms (SFF 8679 Table 5-6) to ensure the voltage rails in the
9196          * module have stabilized.
9197          */
9198         msleep(500);
9199
9200         /*
9201          * Check for QSFP interrupt for t_init (SFF 8679 Table 8-1)
9202          */
9203         timeout = jiffies + msecs_to_jiffies(2000);
9204         while (1) {
9205                 mask = read_csr(dd, dd->hfi1_id ?
9206                                 ASIC_QSFP2_IN : ASIC_QSFP1_IN);
9207                 if (!(mask & QSFP_HFI0_INT_N))
9208                         break;
9209                 if (time_after(jiffies, timeout)) {
9210                         dd_dev_info(dd, "%s: No IntN detected, reset complete\n",
9211                                     __func__);
9212                         break;
9213                 }
9214                 udelay(2);
9215         }
9216 }
9217
9218 static void set_qsfp_int_n(struct hfi1_pportdata *ppd, u8 enable)
9219 {
9220         struct hfi1_devdata *dd = ppd->dd;
9221         u64 mask;
9222
9223         mask = read_csr(dd, dd->hfi1_id ? ASIC_QSFP2_MASK : ASIC_QSFP1_MASK);
9224         if (enable) {
9225                 /*
9226                  * Clear the status register to avoid an immediate interrupt
9227                  * when we re-enable the IntN pin
9228                  */
9229                 write_csr(dd, dd->hfi1_id ? ASIC_QSFP2_CLEAR : ASIC_QSFP1_CLEAR,
9230                           QSFP_HFI0_INT_N);
9231                 mask |= (u64)QSFP_HFI0_INT_N;
9232         } else {
9233                 mask &= ~(u64)QSFP_HFI0_INT_N;
9234         }
9235         write_csr(dd, dd->hfi1_id ? ASIC_QSFP2_MASK : ASIC_QSFP1_MASK, mask);
9236 }
9237
9238 void reset_qsfp(struct hfi1_pportdata *ppd)
9239 {
9240         struct hfi1_devdata *dd = ppd->dd;
9241         u64 mask, qsfp_mask;
9242
9243         /* Disable INT_N from triggering QSFP interrupts */
9244         set_qsfp_int_n(ppd, 0);
9245
9246         /* Reset the QSFP */
9247         mask = (u64)QSFP_HFI0_RESET_N;
9248
9249         qsfp_mask = read_csr(dd,
9250                              dd->hfi1_id ? ASIC_QSFP2_OUT : ASIC_QSFP1_OUT);
9251         qsfp_mask &= ~mask;
9252         write_csr(dd,
9253                   dd->hfi1_id ? ASIC_QSFP2_OUT : ASIC_QSFP1_OUT, qsfp_mask);
9254
9255         udelay(10);
9256
9257         qsfp_mask |= mask;
9258         write_csr(dd,
9259                   dd->hfi1_id ? ASIC_QSFP2_OUT : ASIC_QSFP1_OUT, qsfp_mask);
9260
9261         wait_for_qsfp_init(ppd);
9262
9263         /*
9264          * Allow INT_N to trigger the QSFP interrupt to watch
9265          * for alarms and warnings
9266          */
9267         set_qsfp_int_n(ppd, 1);
9268 }
9269
9270 static int handle_qsfp_error_conditions(struct hfi1_pportdata *ppd,
9271                                         u8 *qsfp_interrupt_status)
9272 {
9273         struct hfi1_devdata *dd = ppd->dd;
9274
9275         if ((qsfp_interrupt_status[0] & QSFP_HIGH_TEMP_ALARM) ||
9276             (qsfp_interrupt_status[0] & QSFP_HIGH_TEMP_WARNING))
9277                 dd_dev_info(dd, "%s: QSFP cable on fire\n",
9278                             __func__);
9279
9280         if ((qsfp_interrupt_status[0] & QSFP_LOW_TEMP_ALARM) ||
9281             (qsfp_interrupt_status[0] & QSFP_LOW_TEMP_WARNING))
9282                 dd_dev_info(dd, "%s: QSFP cable temperature too low\n",
9283                             __func__);
9284
9285         /*
9286          * The remaining alarms/warnings don't matter if the link is down.
9287          */
9288         if (ppd->host_link_state & HLS_DOWN)
9289                 return 0;
9290
9291         if ((qsfp_interrupt_status[1] & QSFP_HIGH_VCC_ALARM) ||
9292             (qsfp_interrupt_status[1] & QSFP_HIGH_VCC_WARNING))
9293                 dd_dev_info(dd, "%s: QSFP supply voltage too high\n",
9294                             __func__);
9295
9296         if ((qsfp_interrupt_status[1] & QSFP_LOW_VCC_ALARM) ||
9297             (qsfp_interrupt_status[1] & QSFP_LOW_VCC_WARNING))
9298                 dd_dev_info(dd, "%s: QSFP supply voltage too low\n",
9299                             __func__);
9300
9301         /* Byte 2 is vendor specific */
9302
9303         if ((qsfp_interrupt_status[3] & QSFP_HIGH_POWER_ALARM) ||
9304             (qsfp_interrupt_status[3] & QSFP_HIGH_POWER_WARNING))
9305                 dd_dev_info(dd, "%s: Cable RX channel 1/2 power too high\n",
9306                             __func__);
9307
9308         if ((qsfp_interrupt_status[3] & QSFP_LOW_POWER_ALARM) ||
9309             (qsfp_interrupt_status[3] & QSFP_LOW_POWER_WARNING))
9310                 dd_dev_info(dd, "%s: Cable RX channel 1/2 power too low\n",
9311                             __func__);
9312
9313         if ((qsfp_interrupt_status[4] & QSFP_HIGH_POWER_ALARM) ||
9314             (qsfp_interrupt_status[4] & QSFP_HIGH_POWER_WARNING))
9315                 dd_dev_info(dd, "%s: Cable RX channel 3/4 power too high\n",
9316                             __func__);
9317
9318         if ((qsfp_interrupt_status[4] & QSFP_LOW_POWER_ALARM) ||
9319             (qsfp_interrupt_status[4] & QSFP_LOW_POWER_WARNING))
9320                 dd_dev_info(dd, "%s: Cable RX channel 3/4 power too low\n",
9321                             __func__);
9322
9323         if ((qsfp_interrupt_status[5] & QSFP_HIGH_BIAS_ALARM) ||
9324             (qsfp_interrupt_status[5] & QSFP_HIGH_BIAS_WARNING))
9325                 dd_dev_info(dd, "%s: Cable TX channel 1/2 bias too high\n",
9326                             __func__);
9327
9328         if ((qsfp_interrupt_status[5] & QSFP_LOW_BIAS_ALARM) ||
9329             (qsfp_interrupt_status[5] & QSFP_LOW_BIAS_WARNING))
9330                 dd_dev_info(dd, "%s: Cable TX channel 1/2 bias too low\n",
9331                             __func__);
9332
9333         if ((qsfp_interrupt_status[6] & QSFP_HIGH_BIAS_ALARM) ||
9334             (qsfp_interrupt_status[6] & QSFP_HIGH_BIAS_WARNING))
9335                 dd_dev_info(dd, "%s: Cable TX channel 3/4 bias too high\n",
9336                             __func__);
9337
9338         if ((qsfp_interrupt_status[6] & QSFP_LOW_BIAS_ALARM) ||
9339             (qsfp_interrupt_status[6] & QSFP_LOW_BIAS_WARNING))
9340                 dd_dev_info(dd, "%s: Cable TX channel 3/4 bias too low\n",
9341                             __func__);
9342
9343         if ((qsfp_interrupt_status[7] & QSFP_HIGH_POWER_ALARM) ||
9344             (qsfp_interrupt_status[7] & QSFP_HIGH_POWER_WARNING))
9345                 dd_dev_info(dd, "%s: Cable TX channel 1/2 power too high\n",
9346                             __func__);
9347
9348         if ((qsfp_interrupt_status[7] & QSFP_LOW_POWER_ALARM) ||
9349             (qsfp_interrupt_status[7] & QSFP_LOW_POWER_WARNING))
9350                 dd_dev_info(dd, "%s: Cable TX channel 1/2 power too low\n",
9351                             __func__);
9352
9353         if ((qsfp_interrupt_status[8] & QSFP_HIGH_POWER_ALARM) ||
9354             (qsfp_interrupt_status[8] & QSFP_HIGH_POWER_WARNING))
9355                 dd_dev_info(dd, "%s: Cable TX channel 3/4 power too high\n",
9356                             __func__);
9357
9358         if ((qsfp_interrupt_status[8] & QSFP_LOW_POWER_ALARM) ||
9359             (qsfp_interrupt_status[8] & QSFP_LOW_POWER_WARNING))
9360                 dd_dev_info(dd, "%s: Cable TX channel 3/4 power too low\n",
9361                             __func__);
9362
9363         /* Bytes 9-10 and 11-12 are reserved */
9364         /* Bytes 13-15 are vendor specific */
9365
9366         return 0;
9367 }
9368
9369 /* This routine will only be scheduled if the QSFP module present is asserted */
9370 void qsfp_event(struct work_struct *work)
9371 {
9372         struct qsfp_data *qd;
9373         struct hfi1_pportdata *ppd;
9374         struct hfi1_devdata *dd;
9375
9376         qd = container_of(work, struct qsfp_data, qsfp_work);
9377         ppd = qd->ppd;
9378         dd = ppd->dd;
9379
9380         /* Sanity check */
9381         if (!qsfp_mod_present(ppd))
9382                 return;
9383
9384         /*
9385          * Turn DC back on after cable has been re-inserted. Up until
9386          * now, the DC has been in reset to save power.
9387          */
9388         dc_start(dd);
9389
9390         if (qd->cache_refresh_required) {
9391                 set_qsfp_int_n(ppd, 0);
9392
9393                 wait_for_qsfp_init(ppd);
9394
9395                 /*
9396                  * Allow INT_N to trigger the QSFP interrupt to watch
9397                  * for alarms and warnings
9398                  */
9399                 set_qsfp_int_n(ppd, 1);
9400
9401                 start_link(ppd);
9402         }
9403
9404         if (qd->check_interrupt_flags) {
9405                 u8 qsfp_interrupt_status[16] = {0,};
9406
9407                 if (one_qsfp_read(ppd, dd->hfi1_id, 6,
9408                                   &qsfp_interrupt_status[0], 16) != 16) {
9409                         dd_dev_info(dd,
9410                                     "%s: Failed to read status of QSFP module\n",
9411                                     __func__);
9412                 } else {
9413                         unsigned long flags;
9414
9415                         handle_qsfp_error_conditions(
9416                                         ppd, qsfp_interrupt_status);
9417                         spin_lock_irqsave(&ppd->qsfp_info.qsfp_lock, flags);
9418                         ppd->qsfp_info.check_interrupt_flags = 0;
9419                         spin_unlock_irqrestore(&ppd->qsfp_info.qsfp_lock,
9420                                                flags);
9421                 }
9422         }
9423 }
9424
9425 static void init_qsfp_int(struct hfi1_devdata *dd)
9426 {
9427         struct hfi1_pportdata *ppd = dd->pport;
9428         u64 qsfp_mask, cce_int_mask;
9429         const int qsfp1_int_smask = QSFP1_INT % 64;
9430         const int qsfp2_int_smask = QSFP2_INT % 64;
9431
9432         /*
9433          * disable QSFP1 interrupts for HFI1, QSFP2 interrupts for HFI0
9434          * Qsfp1Int and Qsfp2Int are adjacent bits in the same CSR,
9435          * therefore just one of QSFP1_INT/QSFP2_INT can be used to find
9436          * the index of the appropriate CSR in the CCEIntMask CSR array
9437          */
9438         cce_int_mask = read_csr(dd, CCE_INT_MASK +
9439                                 (8 * (QSFP1_INT / 64)));
9440         if (dd->hfi1_id) {
9441                 cce_int_mask &= ~((u64)1 << qsfp1_int_smask);
9442                 write_csr(dd, CCE_INT_MASK + (8 * (QSFP1_INT / 64)),
9443                           cce_int_mask);
9444         } else {
9445                 cce_int_mask &= ~((u64)1 << qsfp2_int_smask);
9446                 write_csr(dd, CCE_INT_MASK + (8 * (QSFP2_INT / 64)),
9447                           cce_int_mask);
9448         }
9449
9450         qsfp_mask = (u64)(QSFP_HFI0_INT_N | QSFP_HFI0_MODPRST_N);
9451         /* Clear current status to avoid spurious interrupts */
9452         write_csr(dd, dd->hfi1_id ? ASIC_QSFP2_CLEAR : ASIC_QSFP1_CLEAR,
9453                   qsfp_mask);
9454         write_csr(dd, dd->hfi1_id ? ASIC_QSFP2_MASK : ASIC_QSFP1_MASK,
9455                   qsfp_mask);
9456
9457         set_qsfp_int_n(ppd, 0);
9458
9459         /* Handle active low nature of INT_N and MODPRST_N pins */
9460         if (qsfp_mod_present(ppd))
9461                 qsfp_mask &= ~(u64)QSFP_HFI0_MODPRST_N;
9462         write_csr(dd,
9463                   dd->hfi1_id ? ASIC_QSFP2_INVERT : ASIC_QSFP1_INVERT,
9464                   qsfp_mask);
9465 }
9466
9467 /*
9468  * Do a one-time initialize of the LCB block.
9469  */
9470 static void init_lcb(struct hfi1_devdata *dd)
9471 {
9472         /* simulator does not correctly handle LCB cclk loopback, skip */
9473         if (dd->icode == ICODE_FUNCTIONAL_SIMULATOR)
9474                 return;
9475
9476         /* the DC has been reset earlier in the driver load */
9477
9478         /* set LCB for cclk loopback on the port */
9479         write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET, 0x01);
9480         write_csr(dd, DC_LCB_CFG_LANE_WIDTH, 0x00);
9481         write_csr(dd, DC_LCB_CFG_REINIT_AS_SLAVE, 0x00);
9482         write_csr(dd, DC_LCB_CFG_CNT_FOR_SKIP_STALL, 0x110);
9483         write_csr(dd, DC_LCB_CFG_CLK_CNTR, 0x08);
9484         write_csr(dd, DC_LCB_CFG_LOOPBACK, 0x02);
9485         write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET, 0x00);
9486 }
9487
9488 /*
9489  * Perform a test read on the QSFP.  Return 0 on success, -ERRNO
9490  * on error.
9491  */
9492 static int test_qsfp_read(struct hfi1_pportdata *ppd)
9493 {
9494         int ret;
9495         u8 status;
9496
9497         /*
9498          * Report success if not a QSFP or, if it is a QSFP, but the cable is
9499          * not present
9500          */
9501         if (ppd->port_type != PORT_TYPE_QSFP || !qsfp_mod_present(ppd))
9502                 return 0;
9503
9504         /* read byte 2, the status byte */
9505         ret = one_qsfp_read(ppd, ppd->dd->hfi1_id, 2, &status, 1);
9506         if (ret < 0)
9507                 return ret;
9508         if (ret != 1)
9509                 return -EIO;
9510
9511         return 0; /* success */
9512 }
9513
9514 /*
9515  * Values for QSFP retry.
9516  *
9517  * Give up after 10s (20 x 500ms).  The overall timeout was empirically
9518  * arrived at from experience on a large cluster.
9519  */
9520 #define MAX_QSFP_RETRIES 20
9521 #define QSFP_RETRY_WAIT 500 /* msec */
9522
9523 /*
9524  * Try a QSFP read.  If it fails, schedule a retry for later.
9525  * Called on first link activation after driver load.
9526  */
9527 static void try_start_link(struct hfi1_pportdata *ppd)
9528 {
9529         if (test_qsfp_read(ppd)) {
9530                 /* read failed */
9531                 if (ppd->qsfp_retry_count >= MAX_QSFP_RETRIES) {
9532                         dd_dev_err(ppd->dd, "QSFP not responding, giving up\n");
9533                         return;
9534                 }
9535                 dd_dev_info(ppd->dd,
9536                             "QSFP not responding, waiting and retrying %d\n",
9537                             (int)ppd->qsfp_retry_count);
9538                 ppd->qsfp_retry_count++;
9539                 queue_delayed_work(ppd->hfi1_wq, &ppd->start_link_work,
9540                                    msecs_to_jiffies(QSFP_RETRY_WAIT));
9541                 return;
9542         }
9543         ppd->qsfp_retry_count = 0;
9544
9545         start_link(ppd);
9546 }
9547
9548 /*
9549  * Workqueue function to start the link after a delay.
9550  */
9551 void handle_start_link(struct work_struct *work)
9552 {
9553         struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata,
9554                                                   start_link_work.work);
9555         try_start_link(ppd);
9556 }
9557
9558 int bringup_serdes(struct hfi1_pportdata *ppd)
9559 {
9560         struct hfi1_devdata *dd = ppd->dd;
9561         u64 guid;
9562         int ret;
9563
9564         if (HFI1_CAP_IS_KSET(EXTENDED_PSN))
9565                 add_rcvctrl(dd, RCV_CTRL_RCV_EXTENDED_PSN_ENABLE_SMASK);
9566
9567         guid = ppd->guid;
9568         if (!guid) {
9569                 if (dd->base_guid)
9570                         guid = dd->base_guid + ppd->port - 1;
9571                 ppd->guid = guid;
9572         }
9573
9574         /* Set linkinit_reason on power up per OPA spec */
9575         ppd->linkinit_reason = OPA_LINKINIT_REASON_LINKUP;
9576
9577         /* one-time init of the LCB */
9578         init_lcb(dd);
9579
9580         if (loopback) {
9581                 ret = init_loopback(dd);
9582                 if (ret < 0)
9583                         return ret;
9584         }
9585
9586         get_port_type(ppd);
9587         if (ppd->port_type == PORT_TYPE_QSFP) {
9588                 set_qsfp_int_n(ppd, 0);
9589                 wait_for_qsfp_init(ppd);
9590                 set_qsfp_int_n(ppd, 1);
9591         }
9592
9593         try_start_link(ppd);
9594         return 0;
9595 }
9596
9597 void hfi1_quiet_serdes(struct hfi1_pportdata *ppd)
9598 {
9599         struct hfi1_devdata *dd = ppd->dd;
9600
9601         /*
9602          * Shut down the link and keep it down.   First turn off that the
9603          * driver wants to allow the link to be up (driver_link_ready).
9604          * Then make sure the link is not automatically restarted
9605          * (link_enabled).  Cancel any pending restart.  And finally
9606          * go offline.
9607          */
9608         ppd->driver_link_ready = 0;
9609         ppd->link_enabled = 0;
9610
9611         ppd->qsfp_retry_count = MAX_QSFP_RETRIES; /* prevent more retries */
9612         flush_delayed_work(&ppd->start_link_work);
9613         cancel_delayed_work_sync(&ppd->start_link_work);
9614
9615         ppd->offline_disabled_reason =
9616                         HFI1_ODR_MASK(OPA_LINKDOWN_REASON_SMA_DISABLED);
9617         set_link_down_reason(ppd, OPA_LINKDOWN_REASON_SMA_DISABLED, 0,
9618                              OPA_LINKDOWN_REASON_SMA_DISABLED);
9619         set_link_state(ppd, HLS_DN_OFFLINE);
9620
9621         /* disable the port */
9622         clear_rcvctrl(dd, RCV_CTRL_RCV_PORT_ENABLE_SMASK);
9623         cancel_work_sync(&ppd->freeze_work);
9624 }
9625
9626 static inline int init_cpu_counters(struct hfi1_devdata *dd)
9627 {
9628         struct hfi1_pportdata *ppd;
9629         int i;
9630
9631         ppd = (struct hfi1_pportdata *)(dd + 1);
9632         for (i = 0; i < dd->num_pports; i++, ppd++) {
9633                 ppd->ibport_data.rvp.rc_acks = NULL;
9634                 ppd->ibport_data.rvp.rc_qacks = NULL;
9635                 ppd->ibport_data.rvp.rc_acks = alloc_percpu(u64);
9636                 ppd->ibport_data.rvp.rc_qacks = alloc_percpu(u64);
9637                 ppd->ibport_data.rvp.rc_delayed_comp = alloc_percpu(u64);
9638                 if (!ppd->ibport_data.rvp.rc_acks ||
9639                     !ppd->ibport_data.rvp.rc_delayed_comp ||
9640                     !ppd->ibport_data.rvp.rc_qacks)
9641                         return -ENOMEM;
9642         }
9643
9644         return 0;
9645 }
9646
9647 static const char * const pt_names[] = {
9648         "expected",
9649         "eager",
9650         "invalid"
9651 };
9652
9653 static const char *pt_name(u32 type)
9654 {
9655         return type >= ARRAY_SIZE(pt_names) ? "unknown" : pt_names[type];
9656 }
9657
9658 /*
9659  * index is the index into the receive array
9660  */
9661 void hfi1_put_tid(struct hfi1_devdata *dd, u32 index,
9662                   u32 type, unsigned long pa, u16 order)
9663 {
9664         u64 reg;
9665         void __iomem *base = (dd->rcvarray_wc ? dd->rcvarray_wc :
9666                               (dd->kregbase + RCV_ARRAY));
9667
9668         if (!(dd->flags & HFI1_PRESENT))
9669                 goto done;
9670
9671         if (type == PT_INVALID) {
9672                 pa = 0;
9673         } else if (type > PT_INVALID) {
9674                 dd_dev_err(dd,
9675                            "unexpected receive array type %u for index %u, not handled\n",
9676                            type, index);
9677                 goto done;
9678         }
9679
9680         hfi1_cdbg(TID, "type %s, index 0x%x, pa 0x%lx, bsize 0x%lx",
9681                   pt_name(type), index, pa, (unsigned long)order);
9682
9683 #define RT_ADDR_SHIFT 12        /* 4KB kernel address boundary */
9684         reg = RCV_ARRAY_RT_WRITE_ENABLE_SMASK
9685                 | (u64)order << RCV_ARRAY_RT_BUF_SIZE_SHIFT
9686                 | ((pa >> RT_ADDR_SHIFT) & RCV_ARRAY_RT_ADDR_MASK)
9687                                         << RCV_ARRAY_RT_ADDR_SHIFT;
9688         writeq(reg, base + (index * 8));
9689
9690         if (type == PT_EAGER)
9691                 /*
9692                  * Eager entries are written one-by-one so we have to push them
9693                  * after we write the entry.
9694                  */
9695                 flush_wc();
9696 done:
9697         return;
9698 }
9699
9700 void hfi1_clear_tids(struct hfi1_ctxtdata *rcd)
9701 {
9702         struct hfi1_devdata *dd = rcd->dd;
9703         u32 i;
9704
9705         /* this could be optimized */
9706         for (i = rcd->eager_base; i < rcd->eager_base +
9707                      rcd->egrbufs.alloced; i++)
9708                 hfi1_put_tid(dd, i, PT_INVALID, 0, 0);
9709
9710         for (i = rcd->expected_base;
9711                         i < rcd->expected_base + rcd->expected_count; i++)
9712                 hfi1_put_tid(dd, i, PT_INVALID, 0, 0);
9713 }
9714
9715 struct ib_header *hfi1_get_msgheader(
9716         struct hfi1_devdata *dd, __le32 *rhf_addr)
9717 {
9718         u32 offset = rhf_hdrq_offset(rhf_to_cpu(rhf_addr));
9719
9720         return (struct ib_header *)
9721                 (rhf_addr - dd->rhf_offset + offset);
9722 }
9723
9724 static const char * const ib_cfg_name_strings[] = {
9725         "HFI1_IB_CFG_LIDLMC",
9726         "HFI1_IB_CFG_LWID_DG_ENB",
9727         "HFI1_IB_CFG_LWID_ENB",
9728         "HFI1_IB_CFG_LWID",
9729         "HFI1_IB_CFG_SPD_ENB",
9730         "HFI1_IB_CFG_SPD",
9731         "HFI1_IB_CFG_RXPOL_ENB",
9732         "HFI1_IB_CFG_LREV_ENB",
9733         "HFI1_IB_CFG_LINKLATENCY",
9734         "HFI1_IB_CFG_HRTBT",
9735         "HFI1_IB_CFG_OP_VLS",
9736         "HFI1_IB_CFG_VL_HIGH_CAP",
9737         "HFI1_IB_CFG_VL_LOW_CAP",
9738         "HFI1_IB_CFG_OVERRUN_THRESH",
9739         "HFI1_IB_CFG_PHYERR_THRESH",
9740         "HFI1_IB_CFG_LINKDEFAULT",
9741         "HFI1_IB_CFG_PKEYS",
9742         "HFI1_IB_CFG_MTU",
9743         "HFI1_IB_CFG_LSTATE",
9744         "HFI1_IB_CFG_VL_HIGH_LIMIT",
9745         "HFI1_IB_CFG_PMA_TICKS",
9746         "HFI1_IB_CFG_PORT"
9747 };
9748
9749 static const char *ib_cfg_name(int which)
9750 {
9751         if (which < 0 || which >= ARRAY_SIZE(ib_cfg_name_strings))
9752                 return "invalid";
9753         return ib_cfg_name_strings[which];
9754 }
9755
9756 int hfi1_get_ib_cfg(struct hfi1_pportdata *ppd, int which)
9757 {
9758         struct hfi1_devdata *dd = ppd->dd;
9759         int val = 0;
9760
9761         switch (which) {
9762         case HFI1_IB_CFG_LWID_ENB: /* allowed Link-width */
9763                 val = ppd->link_width_enabled;
9764                 break;
9765         case HFI1_IB_CFG_LWID: /* currently active Link-width */
9766                 val = ppd->link_width_active;
9767                 break;
9768         case HFI1_IB_CFG_SPD_ENB: /* allowed Link speeds */
9769                 val = ppd->link_speed_enabled;
9770                 break;
9771         case HFI1_IB_CFG_SPD: /* current Link speed */
9772                 val = ppd->link_speed_active;
9773                 break;
9774
9775         case HFI1_IB_CFG_RXPOL_ENB: /* Auto-RX-polarity enable */
9776         case HFI1_IB_CFG_LREV_ENB: /* Auto-Lane-reversal enable */
9777         case HFI1_IB_CFG_LINKLATENCY:
9778                 goto unimplemented;
9779
9780         case HFI1_IB_CFG_OP_VLS:
9781                 val = ppd->actual_vls_operational;
9782                 break;
9783         case HFI1_IB_CFG_VL_HIGH_CAP: /* VL arb high priority table size */
9784                 val = VL_ARB_HIGH_PRIO_TABLE_SIZE;
9785                 break;
9786         case HFI1_IB_CFG_VL_LOW_CAP: /* VL arb low priority table size */
9787                 val = VL_ARB_LOW_PRIO_TABLE_SIZE;
9788                 break;
9789         case HFI1_IB_CFG_OVERRUN_THRESH: /* IB overrun threshold */
9790                 val = ppd->overrun_threshold;
9791                 break;
9792         case HFI1_IB_CFG_PHYERR_THRESH: /* IB PHY error threshold */
9793                 val = ppd->phy_error_threshold;
9794                 break;
9795         case HFI1_IB_CFG_LINKDEFAULT: /* IB link default (sleep/poll) */
9796                 val = dd->link_default;
9797                 break;
9798
9799         case HFI1_IB_CFG_HRTBT: /* Heartbeat off/enable/auto */
9800         case HFI1_IB_CFG_PMA_TICKS:
9801         default:
9802 unimplemented:
9803                 if (HFI1_CAP_IS_KSET(PRINT_UNIMPL))
9804                         dd_dev_info(
9805                                 dd,
9806                                 "%s: which %s: not implemented\n",
9807                                 __func__,
9808                                 ib_cfg_name(which));
9809                 break;
9810         }
9811
9812         return val;
9813 }
9814
9815 /*
9816  * The largest MAD packet size.
9817  */
9818 #define MAX_MAD_PACKET 2048
9819
9820 /*
9821  * Return the maximum header bytes that can go on the _wire_
9822  * for this device. This count includes the ICRC which is
9823  * not part of the packet held in memory but it is appended
9824  * by the HW.
9825  * This is dependent on the device's receive header entry size.
9826  * HFI allows this to be set per-receive context, but the
9827  * driver presently enforces a global value.
9828  */
9829 u32 lrh_max_header_bytes(struct hfi1_devdata *dd)
9830 {
9831         /*
9832          * The maximum non-payload (MTU) bytes in LRH.PktLen are
9833          * the Receive Header Entry Size minus the PBC (or RHF) size
9834          * plus one DW for the ICRC appended by HW.
9835          *
9836          * dd->rcd[0].rcvhdrqentsize is in DW.
9837          * We use rcd[0] as all context will have the same value. Also,
9838          * the first kernel context would have been allocated by now so
9839          * we are guaranteed a valid value.
9840          */
9841         return (dd->rcd[0]->rcvhdrqentsize - 2/*PBC/RHF*/ + 1/*ICRC*/) << 2;
9842 }
9843
9844 /*
9845  * Set Send Length
9846  * @ppd - per port data
9847  *
9848  * Set the MTU by limiting how many DWs may be sent.  The SendLenCheck*
9849  * registers compare against LRH.PktLen, so use the max bytes included
9850  * in the LRH.
9851  *
9852  * This routine changes all VL values except VL15, which it maintains at
9853  * the same value.
9854  */
9855 static void set_send_length(struct hfi1_pportdata *ppd)
9856 {
9857         struct hfi1_devdata *dd = ppd->dd;
9858         u32 max_hb = lrh_max_header_bytes(dd), dcmtu;
9859         u32 maxvlmtu = dd->vld[15].mtu;
9860         u64 len1 = 0, len2 = (((dd->vld[15].mtu + max_hb) >> 2)
9861                               & SEND_LEN_CHECK1_LEN_VL15_MASK) <<
9862                 SEND_LEN_CHECK1_LEN_VL15_SHIFT;
9863         int i, j;
9864         u32 thres;
9865
9866         for (i = 0; i < ppd->vls_supported; i++) {
9867                 if (dd->vld[i].mtu > maxvlmtu)
9868                         maxvlmtu = dd->vld[i].mtu;
9869                 if (i <= 3)
9870                         len1 |= (((dd->vld[i].mtu + max_hb) >> 2)
9871                                  & SEND_LEN_CHECK0_LEN_VL0_MASK) <<
9872                                 ((i % 4) * SEND_LEN_CHECK0_LEN_VL1_SHIFT);
9873                 else
9874                         len2 |= (((dd->vld[i].mtu + max_hb) >> 2)
9875                                  & SEND_LEN_CHECK1_LEN_VL4_MASK) <<
9876                                 ((i % 4) * SEND_LEN_CHECK1_LEN_VL5_SHIFT);
9877         }
9878         write_csr(dd, SEND_LEN_CHECK0, len1);
9879         write_csr(dd, SEND_LEN_CHECK1, len2);
9880         /* adjust kernel credit return thresholds based on new MTUs */
9881         /* all kernel receive contexts have the same hdrqentsize */
9882         for (i = 0; i < ppd->vls_supported; i++) {
9883                 thres = min(sc_percent_to_threshold(dd->vld[i].sc, 50),
9884                             sc_mtu_to_threshold(dd->vld[i].sc,
9885                                                 dd->vld[i].mtu,
9886                                                 dd->rcd[0]->rcvhdrqentsize));
9887                 for (j = 0; j < INIT_SC_PER_VL; j++)
9888                         sc_set_cr_threshold(
9889                                         pio_select_send_context_vl(dd, j, i),
9890                                             thres);
9891         }
9892         thres = min(sc_percent_to_threshold(dd->vld[15].sc, 50),
9893                     sc_mtu_to_threshold(dd->vld[15].sc,
9894                                         dd->vld[15].mtu,
9895                                         dd->rcd[0]->rcvhdrqentsize));
9896         sc_set_cr_threshold(dd->vld[15].sc, thres);
9897
9898         /* Adjust maximum MTU for the port in DC */
9899         dcmtu = maxvlmtu == 10240 ? DCC_CFG_PORT_MTU_CAP_10240 :
9900                 (ilog2(maxvlmtu >> 8) + 1);
9901         len1 = read_csr(ppd->dd, DCC_CFG_PORT_CONFIG);
9902         len1 &= ~DCC_CFG_PORT_CONFIG_MTU_CAP_SMASK;
9903         len1 |= ((u64)dcmtu & DCC_CFG_PORT_CONFIG_MTU_CAP_MASK) <<
9904                 DCC_CFG_PORT_CONFIG_MTU_CAP_SHIFT;
9905         write_csr(ppd->dd, DCC_CFG_PORT_CONFIG, len1);
9906 }
9907
9908 static void set_lidlmc(struct hfi1_pportdata *ppd)
9909 {
9910         int i;
9911         u64 sreg = 0;
9912         struct hfi1_devdata *dd = ppd->dd;
9913         u32 mask = ~((1U << ppd->lmc) - 1);
9914         u64 c1 = read_csr(ppd->dd, DCC_CFG_PORT_CONFIG1);
9915
9916         c1 &= ~(DCC_CFG_PORT_CONFIG1_TARGET_DLID_SMASK
9917                 | DCC_CFG_PORT_CONFIG1_DLID_MASK_SMASK);
9918         c1 |= ((ppd->lid & DCC_CFG_PORT_CONFIG1_TARGET_DLID_MASK)
9919                         << DCC_CFG_PORT_CONFIG1_TARGET_DLID_SHIFT) |
9920               ((mask & DCC_CFG_PORT_CONFIG1_DLID_MASK_MASK)
9921                         << DCC_CFG_PORT_CONFIG1_DLID_MASK_SHIFT);
9922         write_csr(ppd->dd, DCC_CFG_PORT_CONFIG1, c1);
9923
9924         /*
9925          * Iterate over all the send contexts and set their SLID check
9926          */
9927         sreg = ((mask & SEND_CTXT_CHECK_SLID_MASK_MASK) <<
9928                         SEND_CTXT_CHECK_SLID_MASK_SHIFT) |
9929                (((ppd->lid & mask) & SEND_CTXT_CHECK_SLID_VALUE_MASK) <<
9930                         SEND_CTXT_CHECK_SLID_VALUE_SHIFT);
9931
9932         for (i = 0; i < dd->chip_send_contexts; i++) {
9933                 hfi1_cdbg(LINKVERB, "SendContext[%d].SLID_CHECK = 0x%x",
9934                           i, (u32)sreg);
9935                 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_SLID, sreg);
9936         }
9937
9938         /* Now we have to do the same thing for the sdma engines */
9939         sdma_update_lmc(dd, mask, ppd->lid);
9940 }
9941
9942 static int wait_phy_linkstate(struct hfi1_devdata *dd, u32 state, u32 msecs)
9943 {
9944         unsigned long timeout;
9945         u32 curr_state;
9946
9947         timeout = jiffies + msecs_to_jiffies(msecs);
9948         while (1) {
9949                 curr_state = read_physical_state(dd);
9950                 if (curr_state == state)
9951                         break;
9952                 if (time_after(jiffies, timeout)) {
9953                         dd_dev_err(dd,
9954                                    "timeout waiting for phy link state 0x%x, current state is 0x%x\n",
9955                                    state, curr_state);
9956                         return -ETIMEDOUT;
9957                 }
9958                 usleep_range(1950, 2050); /* sleep 2ms-ish */
9959         }
9960
9961         return 0;
9962 }
9963
9964 static const char *state_completed_string(u32 completed)
9965 {
9966         static const char * const state_completed[] = {
9967                 "EstablishComm",
9968                 "OptimizeEQ",
9969                 "VerifyCap"
9970         };
9971
9972         if (completed < ARRAY_SIZE(state_completed))
9973                 return state_completed[completed];
9974
9975         return "unknown";
9976 }
9977
9978 static const char all_lanes_dead_timeout_expired[] =
9979         "All lanes were inactive – was the interconnect media removed?";
9980 static const char tx_out_of_policy[] =
9981         "Passing lanes on local port do not meet the local link width policy";
9982 static const char no_state_complete[] =
9983         "State timeout occurred before link partner completed the state";
9984 static const char * const state_complete_reasons[] = {
9985         [0x00] = "Reason unknown",
9986         [0x01] = "Link was halted by driver, refer to LinkDownReason",
9987         [0x02] = "Link partner reported failure",
9988         [0x10] = "Unable to achieve frame sync on any lane",
9989         [0x11] =
9990           "Unable to find a common bit rate with the link partner",
9991         [0x12] =
9992           "Unable to achieve frame sync on sufficient lanes to meet the local link width policy",
9993         [0x13] =
9994           "Unable to identify preset equalization on sufficient lanes to meet the local link width policy",
9995         [0x14] = no_state_complete,
9996         [0x15] =
9997           "State timeout occurred before link partner identified equalization presets",
9998         [0x16] =
9999           "Link partner completed the EstablishComm state, but the passing lanes do not meet the local link width policy",
10000         [0x17] = tx_out_of_policy,
10001         [0x20] = all_lanes_dead_timeout_expired,
10002         [0x21] =
10003           "Unable to achieve acceptable BER on sufficient lanes to meet the local link width policy",
10004         [0x22] = no_state_complete,
10005         [0x23] =
10006           "Link partner completed the OptimizeEq state, but the passing lanes do not meet the local link width policy",
10007         [0x24] = tx_out_of_policy,
10008         [0x30] = all_lanes_dead_timeout_expired,
10009         [0x31] =
10010           "State timeout occurred waiting for host to process received frames",
10011         [0x32] = no_state_complete,
10012         [0x33] =
10013           "Link partner completed the VerifyCap state, but the passing lanes do not meet the local link width policy",
10014         [0x34] = tx_out_of_policy,
10015 };
10016
10017 static const char *state_complete_reason_code_string(struct hfi1_pportdata *ppd,
10018                                                      u32 code)
10019 {
10020         const char *str = NULL;
10021
10022         if (code < ARRAY_SIZE(state_complete_reasons))
10023                 str = state_complete_reasons[code];
10024
10025         if (str)
10026                 return str;
10027         return "Reserved";
10028 }
10029
10030 /* describe the given last state complete frame */
10031 static void decode_state_complete(struct hfi1_pportdata *ppd, u32 frame,
10032                                   const char *prefix)
10033 {
10034         struct hfi1_devdata *dd = ppd->dd;
10035         u32 success;
10036         u32 state;
10037         u32 reason;
10038         u32 lanes;
10039
10040         /*
10041          * Decode frame:
10042          *  [ 0: 0] - success
10043          *  [ 3: 1] - state
10044          *  [ 7: 4] - next state timeout
10045          *  [15: 8] - reason code
10046          *  [31:16] - lanes
10047          */
10048         success = frame & 0x1;
10049         state = (frame >> 1) & 0x7;
10050         reason = (frame >> 8) & 0xff;
10051         lanes = (frame >> 16) & 0xffff;
10052
10053         dd_dev_err(dd, "Last %s LNI state complete frame 0x%08x:\n",
10054                    prefix, frame);
10055         dd_dev_err(dd, "    last reported state state: %s (0x%x)\n",
10056                    state_completed_string(state), state);
10057         dd_dev_err(dd, "    state successfully completed: %s\n",
10058                    success ? "yes" : "no");
10059         dd_dev_err(dd, "    fail reason 0x%x: %s\n",
10060                    reason, state_complete_reason_code_string(ppd, reason));
10061         dd_dev_err(dd, "    passing lane mask: 0x%x", lanes);
10062 }
10063
10064 /*
10065  * Read the last state complete frames and explain them.  This routine
10066  * expects to be called if the link went down during link negotiation
10067  * and initialization (LNI).  That is, anywhere between polling and link up.
10068  */
10069 static void check_lni_states(struct hfi1_pportdata *ppd)
10070 {
10071         u32 last_local_state;
10072         u32 last_remote_state;
10073
10074         read_last_local_state(ppd->dd, &last_local_state);
10075         read_last_remote_state(ppd->dd, &last_remote_state);
10076
10077         /*
10078          * Don't report anything if there is nothing to report.  A value of
10079          * 0 means the link was taken down while polling and there was no
10080          * training in-process.
10081          */
10082         if (last_local_state == 0 && last_remote_state == 0)
10083                 return;
10084
10085         decode_state_complete(ppd, last_local_state, "transmitted");
10086         decode_state_complete(ppd, last_remote_state, "received");
10087 }
10088
10089 /*
10090  * Helper for set_link_state().  Do not call except from that routine.
10091  * Expects ppd->hls_mutex to be held.
10092  *
10093  * @rem_reason value to be sent to the neighbor
10094  *
10095  * LinkDownReasons only set if transition succeeds.
10096  */
10097 static int goto_offline(struct hfi1_pportdata *ppd, u8 rem_reason)
10098 {
10099         struct hfi1_devdata *dd = ppd->dd;
10100         u32 pstate, previous_state;
10101         int ret;
10102         int do_transition;
10103         int do_wait;
10104
10105         previous_state = ppd->host_link_state;
10106         ppd->host_link_state = HLS_GOING_OFFLINE;
10107         pstate = read_physical_state(dd);
10108         if (pstate == PLS_OFFLINE) {
10109                 do_transition = 0;      /* in right state */
10110                 do_wait = 0;            /* ...no need to wait */
10111         } else if ((pstate & 0xff) == PLS_OFFLINE) {
10112                 do_transition = 0;      /* in an offline transient state */
10113                 do_wait = 1;            /* ...wait for it to settle */
10114         } else {
10115                 do_transition = 1;      /* need to move to offline */
10116                 do_wait = 1;            /* ...will need to wait */
10117         }
10118
10119         if (do_transition) {
10120                 ret = set_physical_link_state(dd,
10121                                               (rem_reason << 8) | PLS_OFFLINE);
10122
10123                 if (ret != HCMD_SUCCESS) {
10124                         dd_dev_err(dd,
10125                                    "Failed to transition to Offline link state, return %d\n",
10126                                    ret);
10127                         return -EINVAL;
10128                 }
10129                 if (ppd->offline_disabled_reason ==
10130                                 HFI1_ODR_MASK(OPA_LINKDOWN_REASON_NONE))
10131                         ppd->offline_disabled_reason =
10132                         HFI1_ODR_MASK(OPA_LINKDOWN_REASON_TRANSIENT);
10133         }
10134
10135         if (do_wait) {
10136                 /* it can take a while for the link to go down */
10137                 ret = wait_phy_linkstate(dd, PLS_OFFLINE, 10000);
10138                 if (ret < 0)
10139                         return ret;
10140         }
10141
10142         /* make sure the logical state is also down */
10143         wait_logical_linkstate(ppd, IB_PORT_DOWN, 1000);
10144
10145         /*
10146          * Now in charge of LCB - must be after the physical state is
10147          * offline.quiet and before host_link_state is changed.
10148          */
10149         set_host_lcb_access(dd);
10150         write_csr(dd, DC_LCB_ERR_EN, ~0ull); /* watch LCB errors */
10151         ppd->host_link_state = HLS_LINK_COOLDOWN; /* LCB access allowed */
10152
10153         if (ppd->port_type == PORT_TYPE_QSFP &&
10154             ppd->qsfp_info.limiting_active &&
10155             qsfp_mod_present(ppd)) {
10156                 int ret;
10157
10158                 ret = acquire_chip_resource(dd, qsfp_resource(dd), QSFP_WAIT);
10159                 if (ret == 0) {
10160                         set_qsfp_tx(ppd, 0);
10161                         release_chip_resource(dd, qsfp_resource(dd));
10162                 } else {
10163                         /* not fatal, but should warn */
10164                         dd_dev_err(dd,
10165                                    "Unable to acquire lock to turn off QSFP TX\n");
10166                 }
10167         }
10168
10169         /*
10170          * The LNI has a mandatory wait time after the physical state
10171          * moves to Offline.Quiet.  The wait time may be different
10172          * depending on how the link went down.  The 8051 firmware
10173          * will observe the needed wait time and only move to ready
10174          * when that is completed.  The largest of the quiet timeouts
10175          * is 6s, so wait that long and then at least 0.5s more for
10176          * other transitions, and another 0.5s for a buffer.
10177          */
10178         ret = wait_fm_ready(dd, 7000);
10179         if (ret) {
10180                 dd_dev_err(dd,
10181                            "After going offline, timed out waiting for the 8051 to become ready to accept host requests\n");
10182                 /* state is really offline, so make it so */
10183                 ppd->host_link_state = HLS_DN_OFFLINE;
10184                 return ret;
10185         }
10186
10187         /*
10188          * The state is now offline and the 8051 is ready to accept host
10189          * requests.
10190          *      - change our state
10191          *      - notify others if we were previously in a linkup state
10192          */
10193         ppd->host_link_state = HLS_DN_OFFLINE;
10194         if (previous_state & HLS_UP) {
10195                 /* went down while link was up */
10196                 handle_linkup_change(dd, 0);
10197         } else if (previous_state
10198                         & (HLS_DN_POLL | HLS_VERIFY_CAP | HLS_GOING_UP)) {
10199                 /* went down while attempting link up */
10200                 check_lni_states(ppd);
10201         }
10202
10203         /* the active link width (downgrade) is 0 on link down */
10204         ppd->link_width_active = 0;
10205         ppd->link_width_downgrade_tx_active = 0;
10206         ppd->link_width_downgrade_rx_active = 0;
10207         ppd->current_egress_rate = 0;
10208         return 0;
10209 }
10210
10211 /* return the link state name */
10212 static const char *link_state_name(u32 state)
10213 {
10214         const char *name;
10215         int n = ilog2(state);
10216         static const char * const names[] = {
10217                 [__HLS_UP_INIT_BP]       = "INIT",
10218                 [__HLS_UP_ARMED_BP]      = "ARMED",
10219                 [__HLS_UP_ACTIVE_BP]     = "ACTIVE",
10220                 [__HLS_DN_DOWNDEF_BP]    = "DOWNDEF",
10221                 [__HLS_DN_POLL_BP]       = "POLL",
10222                 [__HLS_DN_DISABLE_BP]    = "DISABLE",
10223                 [__HLS_DN_OFFLINE_BP]    = "OFFLINE",
10224                 [__HLS_VERIFY_CAP_BP]    = "VERIFY_CAP",
10225                 [__HLS_GOING_UP_BP]      = "GOING_UP",
10226                 [__HLS_GOING_OFFLINE_BP] = "GOING_OFFLINE",
10227                 [__HLS_LINK_COOLDOWN_BP] = "LINK_COOLDOWN"
10228         };
10229
10230         name = n < ARRAY_SIZE(names) ? names[n] : NULL;
10231         return name ? name : "unknown";
10232 }
10233
10234 /* return the link state reason name */
10235 static const char *link_state_reason_name(struct hfi1_pportdata *ppd, u32 state)
10236 {
10237         if (state == HLS_UP_INIT) {
10238                 switch (ppd->linkinit_reason) {
10239                 case OPA_LINKINIT_REASON_LINKUP:
10240                         return "(LINKUP)";
10241                 case OPA_LINKINIT_REASON_FLAPPING:
10242                         return "(FLAPPING)";
10243                 case OPA_LINKINIT_OUTSIDE_POLICY:
10244                         return "(OUTSIDE_POLICY)";
10245                 case OPA_LINKINIT_QUARANTINED:
10246                         return "(QUARANTINED)";
10247                 case OPA_LINKINIT_INSUFIC_CAPABILITY:
10248                         return "(INSUFIC_CAPABILITY)";
10249                 default:
10250                         break;
10251                 }
10252         }
10253         return "";
10254 }
10255
10256 /*
10257  * driver_physical_state - convert the driver's notion of a port's
10258  * state (an HLS_*) into a physical state (a {IB,OPA}_PORTPHYSSTATE_*).
10259  * Return -1 (converted to a u32) to indicate error.
10260  */
10261 u32 driver_physical_state(struct hfi1_pportdata *ppd)
10262 {
10263         switch (ppd->host_link_state) {
10264         case HLS_UP_INIT:
10265         case HLS_UP_ARMED:
10266         case HLS_UP_ACTIVE:
10267                 return IB_PORTPHYSSTATE_LINKUP;
10268         case HLS_DN_POLL:
10269                 return IB_PORTPHYSSTATE_POLLING;
10270         case HLS_DN_DISABLE:
10271                 return IB_PORTPHYSSTATE_DISABLED;
10272         case HLS_DN_OFFLINE:
10273                 return OPA_PORTPHYSSTATE_OFFLINE;
10274         case HLS_VERIFY_CAP:
10275                 return IB_PORTPHYSSTATE_POLLING;
10276         case HLS_GOING_UP:
10277                 return IB_PORTPHYSSTATE_POLLING;
10278         case HLS_GOING_OFFLINE:
10279                 return OPA_PORTPHYSSTATE_OFFLINE;
10280         case HLS_LINK_COOLDOWN:
10281                 return OPA_PORTPHYSSTATE_OFFLINE;
10282         case HLS_DN_DOWNDEF:
10283         default:
10284                 dd_dev_err(ppd->dd, "invalid host_link_state 0x%x\n",
10285                            ppd->host_link_state);
10286                 return  -1;
10287         }
10288 }
10289
10290 /*
10291  * driver_logical_state - convert the driver's notion of a port's
10292  * state (an HLS_*) into a logical state (a IB_PORT_*). Return -1
10293  * (converted to a u32) to indicate error.
10294  */
10295 u32 driver_logical_state(struct hfi1_pportdata *ppd)
10296 {
10297         if (ppd->host_link_state && (ppd->host_link_state & HLS_DOWN))
10298                 return IB_PORT_DOWN;
10299
10300         switch (ppd->host_link_state & HLS_UP) {
10301         case HLS_UP_INIT:
10302                 return IB_PORT_INIT;
10303         case HLS_UP_ARMED:
10304                 return IB_PORT_ARMED;
10305         case HLS_UP_ACTIVE:
10306                 return IB_PORT_ACTIVE;
10307         default:
10308                 dd_dev_err(ppd->dd, "invalid host_link_state 0x%x\n",
10309                            ppd->host_link_state);
10310         return -1;
10311         }
10312 }
10313
10314 void set_link_down_reason(struct hfi1_pportdata *ppd, u8 lcl_reason,
10315                           u8 neigh_reason, u8 rem_reason)
10316 {
10317         if (ppd->local_link_down_reason.latest == 0 &&
10318             ppd->neigh_link_down_reason.latest == 0) {
10319                 ppd->local_link_down_reason.latest = lcl_reason;
10320                 ppd->neigh_link_down_reason.latest = neigh_reason;
10321                 ppd->remote_link_down_reason = rem_reason;
10322         }
10323 }
10324
10325 /*
10326  * Change the physical and/or logical link state.
10327  *
10328  * Do not call this routine while inside an interrupt.  It contains
10329  * calls to routines that can take multiple seconds to finish.
10330  *
10331  * Returns 0 on success, -errno on failure.
10332  */
10333 int set_link_state(struct hfi1_pportdata *ppd, u32 state)
10334 {
10335         struct hfi1_devdata *dd = ppd->dd;
10336         struct ib_event event = {.device = NULL};
10337         int ret1, ret = 0;
10338         int orig_new_state, poll_bounce;
10339
10340         mutex_lock(&ppd->hls_lock);
10341
10342         orig_new_state = state;
10343         if (state == HLS_DN_DOWNDEF)
10344                 state = dd->link_default;
10345
10346         /* interpret poll -> poll as a link bounce */
10347         poll_bounce = ppd->host_link_state == HLS_DN_POLL &&
10348                       state == HLS_DN_POLL;
10349
10350         dd_dev_info(dd, "%s: current %s, new %s %s%s\n", __func__,
10351                     link_state_name(ppd->host_link_state),
10352                     link_state_name(orig_new_state),
10353                     poll_bounce ? "(bounce) " : "",
10354                     link_state_reason_name(ppd, state));
10355
10356         /*
10357          * If we're going to a (HLS_*) link state that implies the logical
10358          * link state is neither of (IB_PORT_ARMED, IB_PORT_ACTIVE), then
10359          * reset is_sm_config_started to 0.
10360          */
10361         if (!(state & (HLS_UP_ARMED | HLS_UP_ACTIVE)))
10362                 ppd->is_sm_config_started = 0;
10363
10364         /*
10365          * Do nothing if the states match.  Let a poll to poll link bounce
10366          * go through.
10367          */
10368         if (ppd->host_link_state == state && !poll_bounce)
10369                 goto done;
10370
10371         switch (state) {
10372         case HLS_UP_INIT:
10373                 if (ppd->host_link_state == HLS_DN_POLL &&
10374                     (quick_linkup || dd->icode == ICODE_FUNCTIONAL_SIMULATOR)) {
10375                         /*
10376                          * Quick link up jumps from polling to here.
10377                          *
10378                          * Whether in normal or loopback mode, the
10379                          * simulator jumps from polling to link up.
10380                          * Accept that here.
10381                          */
10382                         /* OK */
10383                 } else if (ppd->host_link_state != HLS_GOING_UP) {
10384                         goto unexpected;
10385                 }
10386
10387                 ppd->host_link_state = HLS_UP_INIT;
10388                 ret = wait_logical_linkstate(ppd, IB_PORT_INIT, 1000);
10389                 if (ret) {
10390                         /* logical state didn't change, stay at going_up */
10391                         ppd->host_link_state = HLS_GOING_UP;
10392                         dd_dev_err(dd,
10393                                    "%s: logical state did not change to INIT\n",
10394                                    __func__);
10395                 } else {
10396                         /* clear old transient LINKINIT_REASON code */
10397                         if (ppd->linkinit_reason >= OPA_LINKINIT_REASON_CLEAR)
10398                                 ppd->linkinit_reason =
10399                                         OPA_LINKINIT_REASON_LINKUP;
10400
10401                         /* enable the port */
10402                         add_rcvctrl(dd, RCV_CTRL_RCV_PORT_ENABLE_SMASK);
10403
10404                         handle_linkup_change(dd, 1);
10405                 }
10406                 break;
10407         case HLS_UP_ARMED:
10408                 if (ppd->host_link_state != HLS_UP_INIT)
10409                         goto unexpected;
10410
10411                 ppd->host_link_state = HLS_UP_ARMED;
10412                 set_logical_state(dd, LSTATE_ARMED);
10413                 ret = wait_logical_linkstate(ppd, IB_PORT_ARMED, 1000);
10414                 if (ret) {
10415                         /* logical state didn't change, stay at init */
10416                         ppd->host_link_state = HLS_UP_INIT;
10417                         dd_dev_err(dd,
10418                                    "%s: logical state did not change to ARMED\n",
10419                                    __func__);
10420                 }
10421                 /*
10422                  * The simulator does not currently implement SMA messages,
10423                  * so neighbor_normal is not set.  Set it here when we first
10424                  * move to Armed.
10425                  */
10426                 if (dd->icode == ICODE_FUNCTIONAL_SIMULATOR)
10427                         ppd->neighbor_normal = 1;
10428                 break;
10429         case HLS_UP_ACTIVE:
10430                 if (ppd->host_link_state != HLS_UP_ARMED)
10431                         goto unexpected;
10432
10433                 ppd->host_link_state = HLS_UP_ACTIVE;
10434                 set_logical_state(dd, LSTATE_ACTIVE);
10435                 ret = wait_logical_linkstate(ppd, IB_PORT_ACTIVE, 1000);
10436                 if (ret) {
10437                         /* logical state didn't change, stay at armed */
10438                         ppd->host_link_state = HLS_UP_ARMED;
10439                         dd_dev_err(dd,
10440                                    "%s: logical state did not change to ACTIVE\n",
10441                                    __func__);
10442                 } else {
10443                         /* tell all engines to go running */
10444                         sdma_all_running(dd);
10445
10446                         /* Signal the IB layer that the port has went active */
10447                         event.device = &dd->verbs_dev.rdi.ibdev;
10448                         event.element.port_num = ppd->port;
10449                         event.event = IB_EVENT_PORT_ACTIVE;
10450                 }
10451                 break;
10452         case HLS_DN_POLL:
10453                 if ((ppd->host_link_state == HLS_DN_DISABLE ||
10454                      ppd->host_link_state == HLS_DN_OFFLINE) &&
10455                     dd->dc_shutdown)
10456                         dc_start(dd);
10457                 /* Hand LED control to the DC */
10458                 write_csr(dd, DCC_CFG_LED_CNTRL, 0);
10459
10460                 if (ppd->host_link_state != HLS_DN_OFFLINE) {
10461                         u8 tmp = ppd->link_enabled;
10462
10463                         ret = goto_offline(ppd, ppd->remote_link_down_reason);
10464                         if (ret) {
10465                                 ppd->link_enabled = tmp;
10466                                 break;
10467                         }
10468                         ppd->remote_link_down_reason = 0;
10469
10470                         if (ppd->driver_link_ready)
10471                                 ppd->link_enabled = 1;
10472                 }
10473
10474                 set_all_slowpath(ppd->dd);
10475                 ret = set_local_link_attributes(ppd);
10476                 if (ret)
10477                         break;
10478
10479                 ppd->port_error_action = 0;
10480                 ppd->host_link_state = HLS_DN_POLL;
10481
10482                 if (quick_linkup) {
10483                         /* quick linkup does not go into polling */
10484                         ret = do_quick_linkup(dd);
10485                 } else {
10486                         ret1 = set_physical_link_state(dd, PLS_POLLING);
10487                         if (ret1 != HCMD_SUCCESS) {
10488                                 dd_dev_err(dd,
10489                                            "Failed to transition to Polling link state, return 0x%x\n",
10490                                            ret1);
10491                                 ret = -EINVAL;
10492                         }
10493                 }
10494                 ppd->offline_disabled_reason =
10495                         HFI1_ODR_MASK(OPA_LINKDOWN_REASON_NONE);
10496                 /*
10497                  * If an error occurred above, go back to offline.  The
10498                  * caller may reschedule another attempt.
10499                  */
10500                 if (ret)
10501                         goto_offline(ppd, 0);
10502                 break;
10503         case HLS_DN_DISABLE:
10504                 /* link is disabled */
10505                 ppd->link_enabled = 0;
10506
10507                 /* allow any state to transition to disabled */
10508
10509                 /* must transition to offline first */
10510                 if (ppd->host_link_state != HLS_DN_OFFLINE) {
10511                         ret = goto_offline(ppd, ppd->remote_link_down_reason);
10512                         if (ret)
10513                                 break;
10514                         ppd->remote_link_down_reason = 0;
10515                 }
10516
10517                 ret1 = set_physical_link_state(dd, PLS_DISABLED);
10518                 if (ret1 != HCMD_SUCCESS) {
10519                         dd_dev_err(dd,
10520                                    "Failed to transition to Disabled link state, return 0x%x\n",
10521                                    ret1);
10522                         ret = -EINVAL;
10523                         break;
10524                 }
10525                 ppd->host_link_state = HLS_DN_DISABLE;
10526                 dc_shutdown(dd);
10527                 break;
10528         case HLS_DN_OFFLINE:
10529                 if (ppd->host_link_state == HLS_DN_DISABLE)
10530                         dc_start(dd);
10531
10532                 /* allow any state to transition to offline */
10533                 ret = goto_offline(ppd, ppd->remote_link_down_reason);
10534                 if (!ret)
10535                         ppd->remote_link_down_reason = 0;
10536                 break;
10537         case HLS_VERIFY_CAP:
10538                 if (ppd->host_link_state != HLS_DN_POLL)
10539                         goto unexpected;
10540                 ppd->host_link_state = HLS_VERIFY_CAP;
10541                 break;
10542         case HLS_GOING_UP:
10543                 if (ppd->host_link_state != HLS_VERIFY_CAP)
10544                         goto unexpected;
10545
10546                 ret1 = set_physical_link_state(dd, PLS_LINKUP);
10547                 if (ret1 != HCMD_SUCCESS) {
10548                         dd_dev_err(dd,
10549                                    "Failed to transition to link up state, return 0x%x\n",
10550                                    ret1);
10551                         ret = -EINVAL;
10552                         break;
10553                 }
10554                 ppd->host_link_state = HLS_GOING_UP;
10555                 break;
10556
10557         case HLS_GOING_OFFLINE:         /* transient within goto_offline() */
10558         case HLS_LINK_COOLDOWN:         /* transient within goto_offline() */
10559         default:
10560                 dd_dev_info(dd, "%s: state 0x%x: not supported\n",
10561                             __func__, state);
10562                 ret = -EINVAL;
10563                 break;
10564         }
10565
10566         goto done;
10567
10568 unexpected:
10569         dd_dev_err(dd, "%s: unexpected state transition from %s to %s\n",
10570                    __func__, link_state_name(ppd->host_link_state),
10571                    link_state_name(state));
10572         ret = -EINVAL;
10573
10574 done:
10575         mutex_unlock(&ppd->hls_lock);
10576
10577         if (event.device)
10578                 ib_dispatch_event(&event);
10579
10580         return ret;
10581 }
10582
10583 int hfi1_set_ib_cfg(struct hfi1_pportdata *ppd, int which, u32 val)
10584 {
10585         u64 reg;
10586         int ret = 0;
10587
10588         switch (which) {
10589         case HFI1_IB_CFG_LIDLMC:
10590                 set_lidlmc(ppd);
10591                 break;
10592         case HFI1_IB_CFG_VL_HIGH_LIMIT:
10593                 /*
10594                  * The VL Arbitrator high limit is sent in units of 4k
10595                  * bytes, while HFI stores it in units of 64 bytes.
10596                  */
10597                 val *= 4096 / 64;
10598                 reg = ((u64)val & SEND_HIGH_PRIORITY_LIMIT_LIMIT_MASK)
10599                         << SEND_HIGH_PRIORITY_LIMIT_LIMIT_SHIFT;
10600                 write_csr(ppd->dd, SEND_HIGH_PRIORITY_LIMIT, reg);
10601                 break;
10602         case HFI1_IB_CFG_LINKDEFAULT: /* IB link default (sleep/poll) */
10603                 /* HFI only supports POLL as the default link down state */
10604                 if (val != HLS_DN_POLL)
10605                         ret = -EINVAL;
10606                 break;
10607         case HFI1_IB_CFG_OP_VLS:
10608                 if (ppd->vls_operational != val) {
10609                         ppd->vls_operational = val;
10610                         if (!ppd->port)
10611                                 ret = -EINVAL;
10612                 }
10613                 break;
10614         /*
10615          * For link width, link width downgrade, and speed enable, always AND
10616          * the setting with what is actually supported.  This has two benefits.
10617          * First, enabled can't have unsupported values, no matter what the
10618          * SM or FM might want.  Second, the ALL_SUPPORTED wildcards that mean
10619          * "fill in with your supported value" have all the bits in the
10620          * field set, so simply ANDing with supported has the desired result.
10621          */
10622         case HFI1_IB_CFG_LWID_ENB: /* set allowed Link-width */
10623                 ppd->link_width_enabled = val & ppd->link_width_supported;
10624                 break;
10625         case HFI1_IB_CFG_LWID_DG_ENB: /* set allowed link width downgrade */
10626                 ppd->link_width_downgrade_enabled =
10627                                 val & ppd->link_width_downgrade_supported;
10628                 break;
10629         case HFI1_IB_CFG_SPD_ENB: /* allowed Link speeds */
10630                 ppd->link_speed_enabled = val & ppd->link_speed_supported;
10631                 break;
10632         case HFI1_IB_CFG_OVERRUN_THRESH: /* IB overrun threshold */
10633                 /*
10634                  * HFI does not follow IB specs, save this value
10635                  * so we can report it, if asked.
10636                  */
10637                 ppd->overrun_threshold = val;
10638                 break;
10639         case HFI1_IB_CFG_PHYERR_THRESH: /* IB PHY error threshold */
10640                 /*
10641                  * HFI does not follow IB specs, save this value
10642                  * so we can report it, if asked.
10643                  */
10644                 ppd->phy_error_threshold = val;
10645                 break;
10646
10647         case HFI1_IB_CFG_MTU:
10648                 set_send_length(ppd);
10649                 break;
10650
10651         case HFI1_IB_CFG_PKEYS:
10652                 if (HFI1_CAP_IS_KSET(PKEY_CHECK))
10653                         set_partition_keys(ppd);
10654                 break;
10655
10656         default:
10657                 if (HFI1_CAP_IS_KSET(PRINT_UNIMPL))
10658                         dd_dev_info(ppd->dd,
10659                                     "%s: which %s, val 0x%x: not implemented\n",
10660                                     __func__, ib_cfg_name(which), val);
10661                 break;
10662         }
10663         return ret;
10664 }
10665
10666 /* begin functions related to vl arbitration table caching */
10667 static void init_vl_arb_caches(struct hfi1_pportdata *ppd)
10668 {
10669         int i;
10670
10671         BUILD_BUG_ON(VL_ARB_TABLE_SIZE !=
10672                         VL_ARB_LOW_PRIO_TABLE_SIZE);
10673         BUILD_BUG_ON(VL_ARB_TABLE_SIZE !=
10674                         VL_ARB_HIGH_PRIO_TABLE_SIZE);
10675
10676         /*
10677          * Note that we always return values directly from the
10678          * 'vl_arb_cache' (and do no CSR reads) in response to a
10679          * 'Get(VLArbTable)'. This is obviously correct after a
10680          * 'Set(VLArbTable)', since the cache will then be up to
10681          * date. But it's also correct prior to any 'Set(VLArbTable)'
10682          * since then both the cache, and the relevant h/w registers
10683          * will be zeroed.
10684          */
10685
10686         for (i = 0; i < MAX_PRIO_TABLE; i++)
10687                 spin_lock_init(&ppd->vl_arb_cache[i].lock);
10688 }
10689
10690 /*
10691  * vl_arb_lock_cache
10692  *
10693  * All other vl_arb_* functions should be called only after locking
10694  * the cache.
10695  */
10696 static inline struct vl_arb_cache *
10697 vl_arb_lock_cache(struct hfi1_pportdata *ppd, int idx)
10698 {
10699         if (idx != LO_PRIO_TABLE && idx != HI_PRIO_TABLE)
10700                 return NULL;
10701         spin_lock(&ppd->vl_arb_cache[idx].lock);
10702         return &ppd->vl_arb_cache[idx];
10703 }
10704
10705 static inline void vl_arb_unlock_cache(struct hfi1_pportdata *ppd, int idx)
10706 {
10707         spin_unlock(&ppd->vl_arb_cache[idx].lock);
10708 }
10709
10710 static void vl_arb_get_cache(struct vl_arb_cache *cache,
10711                              struct ib_vl_weight_elem *vl)
10712 {
10713         memcpy(vl, cache->table, VL_ARB_TABLE_SIZE * sizeof(*vl));
10714 }
10715
10716 static void vl_arb_set_cache(struct vl_arb_cache *cache,
10717                              struct ib_vl_weight_elem *vl)
10718 {
10719         memcpy(cache->table, vl, VL_ARB_TABLE_SIZE * sizeof(*vl));
10720 }
10721
10722 static int vl_arb_match_cache(struct vl_arb_cache *cache,
10723                               struct ib_vl_weight_elem *vl)
10724 {
10725         return !memcmp(cache->table, vl, VL_ARB_TABLE_SIZE * sizeof(*vl));
10726 }
10727
10728 /* end functions related to vl arbitration table caching */
10729
10730 static int set_vl_weights(struct hfi1_pportdata *ppd, u32 target,
10731                           u32 size, struct ib_vl_weight_elem *vl)
10732 {
10733         struct hfi1_devdata *dd = ppd->dd;
10734         u64 reg;
10735         unsigned int i, is_up = 0;
10736         int drain, ret = 0;
10737
10738         mutex_lock(&ppd->hls_lock);
10739
10740         if (ppd->host_link_state & HLS_UP)
10741                 is_up = 1;
10742
10743         drain = !is_ax(dd) && is_up;
10744
10745         if (drain)
10746                 /*
10747                  * Before adjusting VL arbitration weights, empty per-VL
10748                  * FIFOs, otherwise a packet whose VL weight is being
10749                  * set to 0 could get stuck in a FIFO with no chance to
10750                  * egress.
10751                  */
10752                 ret = stop_drain_data_vls(dd);
10753
10754         if (ret) {
10755                 dd_dev_err(
10756                         dd,
10757                         "%s: cannot stop/drain VLs - refusing to change VL arbitration weights\n",
10758                         __func__);
10759                 goto err;
10760         }
10761
10762         for (i = 0; i < size; i++, vl++) {
10763                 /*
10764                  * NOTE: The low priority shift and mask are used here, but
10765                  * they are the same for both the low and high registers.
10766                  */
10767                 reg = (((u64)vl->vl & SEND_LOW_PRIORITY_LIST_VL_MASK)
10768                                 << SEND_LOW_PRIORITY_LIST_VL_SHIFT)
10769                       | (((u64)vl->weight
10770                                 & SEND_LOW_PRIORITY_LIST_WEIGHT_MASK)
10771                                 << SEND_LOW_PRIORITY_LIST_WEIGHT_SHIFT);
10772                 write_csr(dd, target + (i * 8), reg);
10773         }
10774         pio_send_control(dd, PSC_GLOBAL_VLARB_ENABLE);
10775
10776         if (drain)
10777                 open_fill_data_vls(dd); /* reopen all VLs */
10778
10779 err:
10780         mutex_unlock(&ppd->hls_lock);
10781
10782         return ret;
10783 }
10784
10785 /*
10786  * Read one credit merge VL register.
10787  */
10788 static void read_one_cm_vl(struct hfi1_devdata *dd, u32 csr,
10789                            struct vl_limit *vll)
10790 {
10791         u64 reg = read_csr(dd, csr);
10792
10793         vll->dedicated = cpu_to_be16(
10794                 (reg >> SEND_CM_CREDIT_VL_DEDICATED_LIMIT_VL_SHIFT)
10795                 & SEND_CM_CREDIT_VL_DEDICATED_LIMIT_VL_MASK);
10796         vll->shared = cpu_to_be16(
10797                 (reg >> SEND_CM_CREDIT_VL_SHARED_LIMIT_VL_SHIFT)
10798                 & SEND_CM_CREDIT_VL_SHARED_LIMIT_VL_MASK);
10799 }
10800
10801 /*
10802  * Read the current credit merge limits.
10803  */
10804 static int get_buffer_control(struct hfi1_devdata *dd,
10805                               struct buffer_control *bc, u16 *overall_limit)
10806 {
10807         u64 reg;
10808         int i;
10809
10810         /* not all entries are filled in */
10811         memset(bc, 0, sizeof(*bc));
10812
10813         /* OPA and HFI have a 1-1 mapping */
10814         for (i = 0; i < TXE_NUM_DATA_VL; i++)
10815                 read_one_cm_vl(dd, SEND_CM_CREDIT_VL + (8 * i), &bc->vl[i]);
10816
10817         /* NOTE: assumes that VL* and VL15 CSRs are bit-wise identical */
10818         read_one_cm_vl(dd, SEND_CM_CREDIT_VL15, &bc->vl[15]);
10819
10820         reg = read_csr(dd, SEND_CM_GLOBAL_CREDIT);
10821         bc->overall_shared_limit = cpu_to_be16(
10822                 (reg >> SEND_CM_GLOBAL_CREDIT_SHARED_LIMIT_SHIFT)
10823                 & SEND_CM_GLOBAL_CREDIT_SHARED_LIMIT_MASK);
10824         if (overall_limit)
10825                 *overall_limit = (reg
10826                         >> SEND_CM_GLOBAL_CREDIT_TOTAL_CREDIT_LIMIT_SHIFT)
10827                         & SEND_CM_GLOBAL_CREDIT_TOTAL_CREDIT_LIMIT_MASK;
10828         return sizeof(struct buffer_control);
10829 }
10830
10831 static int get_sc2vlnt(struct hfi1_devdata *dd, struct sc2vlnt *dp)
10832 {
10833         u64 reg;
10834         int i;
10835
10836         /* each register contains 16 SC->VLnt mappings, 4 bits each */
10837         reg = read_csr(dd, DCC_CFG_SC_VL_TABLE_15_0);
10838         for (i = 0; i < sizeof(u64); i++) {
10839                 u8 byte = *(((u8 *)&reg) + i);
10840
10841                 dp->vlnt[2 * i] = byte & 0xf;
10842                 dp->vlnt[(2 * i) + 1] = (byte & 0xf0) >> 4;
10843         }
10844
10845         reg = read_csr(dd, DCC_CFG_SC_VL_TABLE_31_16);
10846         for (i = 0; i < sizeof(u64); i++) {
10847                 u8 byte = *(((u8 *)&reg) + i);
10848
10849                 dp->vlnt[16 + (2 * i)] = byte & 0xf;
10850                 dp->vlnt[16 + (2 * i) + 1] = (byte & 0xf0) >> 4;
10851         }
10852         return sizeof(struct sc2vlnt);
10853 }
10854
10855 static void get_vlarb_preempt(struct hfi1_devdata *dd, u32 nelems,
10856                               struct ib_vl_weight_elem *vl)
10857 {
10858         unsigned int i;
10859
10860         for (i = 0; i < nelems; i++, vl++) {
10861                 vl->vl = 0xf;
10862                 vl->weight = 0;
10863         }
10864 }
10865
10866 static void set_sc2vlnt(struct hfi1_devdata *dd, struct sc2vlnt *dp)
10867 {
10868         write_csr(dd, DCC_CFG_SC_VL_TABLE_15_0,
10869                   DC_SC_VL_VAL(15_0,
10870                                0, dp->vlnt[0] & 0xf,
10871                                1, dp->vlnt[1] & 0xf,
10872                                2, dp->vlnt[2] & 0xf,
10873                                3, dp->vlnt[3] & 0xf,
10874                                4, dp->vlnt[4] & 0xf,
10875                                5, dp->vlnt[5] & 0xf,
10876                                6, dp->vlnt[6] & 0xf,
10877                                7, dp->vlnt[7] & 0xf,
10878                                8, dp->vlnt[8] & 0xf,
10879                                9, dp->vlnt[9] & 0xf,
10880                                10, dp->vlnt[10] & 0xf,
10881                                11, dp->vlnt[11] & 0xf,
10882                                12, dp->vlnt[12] & 0xf,
10883                                13, dp->vlnt[13] & 0xf,
10884                                14, dp->vlnt[14] & 0xf,
10885                                15, dp->vlnt[15] & 0xf));
10886         write_csr(dd, DCC_CFG_SC_VL_TABLE_31_16,
10887                   DC_SC_VL_VAL(31_16,
10888                                16, dp->vlnt[16] & 0xf,
10889                                17, dp->vlnt[17] & 0xf,
10890                                18, dp->vlnt[18] & 0xf,
10891                                19, dp->vlnt[19] & 0xf,
10892                                20, dp->vlnt[20] & 0xf,
10893                                21, dp->vlnt[21] & 0xf,
10894                                22, dp->vlnt[22] & 0xf,
10895                                23, dp->vlnt[23] & 0xf,
10896                                24, dp->vlnt[24] & 0xf,
10897                                25, dp->vlnt[25] & 0xf,
10898                                26, dp->vlnt[26] & 0xf,
10899                                27, dp->vlnt[27] & 0xf,
10900                                28, dp->vlnt[28] & 0xf,
10901                                29, dp->vlnt[29] & 0xf,
10902                                30, dp->vlnt[30] & 0xf,
10903                                31, dp->vlnt[31] & 0xf));
10904 }
10905
10906 static void nonzero_msg(struct hfi1_devdata *dd, int idx, const char *what,
10907                         u16 limit)
10908 {
10909         if (limit != 0)
10910                 dd_dev_info(dd, "Invalid %s limit %d on VL %d, ignoring\n",
10911                             what, (int)limit, idx);
10912 }
10913
10914 /* change only the shared limit portion of SendCmGLobalCredit */
10915 static void set_global_shared(struct hfi1_devdata *dd, u16 limit)
10916 {
10917         u64 reg;
10918
10919         reg = read_csr(dd, SEND_CM_GLOBAL_CREDIT);
10920         reg &= ~SEND_CM_GLOBAL_CREDIT_SHARED_LIMIT_SMASK;
10921         reg |= (u64)limit << SEND_CM_GLOBAL_CREDIT_SHARED_LIMIT_SHIFT;
10922         write_csr(dd, SEND_CM_GLOBAL_CREDIT, reg);
10923 }
10924
10925 /* change only the total credit limit portion of SendCmGLobalCredit */
10926 static void set_global_limit(struct hfi1_devdata *dd, u16 limit)
10927 {
10928         u64 reg;
10929
10930         reg = read_csr(dd, SEND_CM_GLOBAL_CREDIT);
10931         reg &= ~SEND_CM_GLOBAL_CREDIT_TOTAL_CREDIT_LIMIT_SMASK;
10932         reg |= (u64)limit << SEND_CM_GLOBAL_CREDIT_TOTAL_CREDIT_LIMIT_SHIFT;
10933         write_csr(dd, SEND_CM_GLOBAL_CREDIT, reg);
10934 }
10935
10936 /* set the given per-VL shared limit */
10937 static void set_vl_shared(struct hfi1_devdata *dd, int vl, u16 limit)
10938 {
10939         u64 reg;
10940         u32 addr;
10941
10942         if (vl < TXE_NUM_DATA_VL)
10943                 addr = SEND_CM_CREDIT_VL + (8 * vl);
10944         else
10945                 addr = SEND_CM_CREDIT_VL15;
10946
10947         reg = read_csr(dd, addr);
10948         reg &= ~SEND_CM_CREDIT_VL_SHARED_LIMIT_VL_SMASK;
10949         reg |= (u64)limit << SEND_CM_CREDIT_VL_SHARED_LIMIT_VL_SHIFT;
10950         write_csr(dd, addr, reg);
10951 }
10952
10953 /* set the given per-VL dedicated limit */
10954 static void set_vl_dedicated(struct hfi1_devdata *dd, int vl, u16 limit)
10955 {
10956         u64 reg;
10957         u32 addr;
10958
10959         if (vl < TXE_NUM_DATA_VL)
10960                 addr = SEND_CM_CREDIT_VL + (8 * vl);
10961         else
10962                 addr = SEND_CM_CREDIT_VL15;
10963
10964         reg = read_csr(dd, addr);
10965         reg &= ~SEND_CM_CREDIT_VL_DEDICATED_LIMIT_VL_SMASK;
10966         reg |= (u64)limit << SEND_CM_CREDIT_VL_DEDICATED_LIMIT_VL_SHIFT;
10967         write_csr(dd, addr, reg);
10968 }
10969
10970 /* spin until the given per-VL status mask bits clear */
10971 static void wait_for_vl_status_clear(struct hfi1_devdata *dd, u64 mask,
10972                                      const char *which)
10973 {
10974         unsigned long timeout;
10975         u64 reg;
10976
10977         timeout = jiffies + msecs_to_jiffies(VL_STATUS_CLEAR_TIMEOUT);
10978         while (1) {
10979                 reg = read_csr(dd, SEND_CM_CREDIT_USED_STATUS) & mask;
10980
10981                 if (reg == 0)
10982                         return; /* success */
10983                 if (time_after(jiffies, timeout))
10984                         break;          /* timed out */
10985                 udelay(1);
10986         }
10987
10988         dd_dev_err(dd,
10989                    "%s credit change status not clearing after %dms, mask 0x%llx, not clear 0x%llx\n",
10990                    which, VL_STATUS_CLEAR_TIMEOUT, mask, reg);
10991         /*
10992          * If this occurs, it is likely there was a credit loss on the link.
10993          * The only recovery from that is a link bounce.
10994          */
10995         dd_dev_err(dd,
10996                    "Continuing anyway.  A credit loss may occur.  Suggest a link bounce\n");
10997 }
10998
10999 /*
11000  * The number of credits on the VLs may be changed while everything
11001  * is "live", but the following algorithm must be followed due to
11002  * how the hardware is actually implemented.  In particular,
11003  * Return_Credit_Status[] is the only correct status check.
11004  *
11005  * if (reducing Global_Shared_Credit_Limit or any shared limit changing)
11006  *     set Global_Shared_Credit_Limit = 0
11007  *     use_all_vl = 1
11008  * mask0 = all VLs that are changing either dedicated or shared limits
11009  * set Shared_Limit[mask0] = 0
11010  * spin until Return_Credit_Status[use_all_vl ? all VL : mask0] == 0
11011  * if (changing any dedicated limit)
11012  *     mask1 = all VLs that are lowering dedicated limits
11013  *     lower Dedicated_Limit[mask1]
11014  *     spin until Return_Credit_Status[mask1] == 0
11015  *     raise Dedicated_Limits
11016  * raise Shared_Limits
11017  * raise Global_Shared_Credit_Limit
11018  *
11019  * lower = if the new limit is lower, set the limit to the new value
11020  * raise = if the new limit is higher than the current value (may be changed
11021  *      earlier in the algorithm), set the new limit to the new value
11022  */
11023 int set_buffer_control(struct hfi1_pportdata *ppd,
11024                        struct buffer_control *new_bc)
11025 {
11026         struct hfi1_devdata *dd = ppd->dd;
11027         u64 changing_mask, ld_mask, stat_mask;
11028         int change_count;
11029         int i, use_all_mask;
11030         int this_shared_changing;
11031         int vl_count = 0, ret;
11032         /*
11033          * A0: add the variable any_shared_limit_changing below and in the
11034          * algorithm above.  If removing A0 support, it can be removed.
11035          */
11036         int any_shared_limit_changing;
11037         struct buffer_control cur_bc;
11038         u8 changing[OPA_MAX_VLS];
11039         u8 lowering_dedicated[OPA_MAX_VLS];
11040         u16 cur_total;
11041         u32 new_total = 0;
11042         const u64 all_mask =
11043         SEND_CM_CREDIT_USED_STATUS_VL0_RETURN_CREDIT_STATUS_SMASK
11044          | SEND_CM_CREDIT_USED_STATUS_VL1_RETURN_CREDIT_STATUS_SMASK
11045          | SEND_CM_CREDIT_USED_STATUS_VL2_RETURN_CREDIT_STATUS_SMASK
11046          | SEND_CM_CREDIT_USED_STATUS_VL3_RETURN_CREDIT_STATUS_SMASK
11047          | SEND_CM_CREDIT_USED_STATUS_VL4_RETURN_CREDIT_STATUS_SMASK
11048          | SEND_CM_CREDIT_USED_STATUS_VL5_RETURN_CREDIT_STATUS_SMASK
11049          | SEND_CM_CREDIT_USED_STATUS_VL6_RETURN_CREDIT_STATUS_SMASK
11050          | SEND_CM_CREDIT_USED_STATUS_VL7_RETURN_CREDIT_STATUS_SMASK
11051          | SEND_CM_CREDIT_USED_STATUS_VL15_RETURN_CREDIT_STATUS_SMASK;
11052
11053 #define valid_vl(idx) ((idx) < TXE_NUM_DATA_VL || (idx) == 15)
11054 #define NUM_USABLE_VLS 16       /* look at VL15 and less */
11055
11056         /* find the new total credits, do sanity check on unused VLs */
11057         for (i = 0; i < OPA_MAX_VLS; i++) {
11058                 if (valid_vl(i)) {
11059                         new_total += be16_to_cpu(new_bc->vl[i].dedicated);
11060                         continue;
11061                 }
11062                 nonzero_msg(dd, i, "dedicated",
11063                             be16_to_cpu(new_bc->vl[i].dedicated));
11064                 nonzero_msg(dd, i, "shared",
11065                             be16_to_cpu(new_bc->vl[i].shared));
11066                 new_bc->vl[i].dedicated = 0;
11067                 new_bc->vl[i].shared = 0;
11068         }
11069         new_total += be16_to_cpu(new_bc->overall_shared_limit);
11070
11071         /* fetch the current values */
11072         get_buffer_control(dd, &cur_bc, &cur_total);
11073
11074         /*
11075          * Create the masks we will use.
11076          */
11077         memset(changing, 0, sizeof(changing));
11078         memset(lowering_dedicated, 0, sizeof(lowering_dedicated));
11079         /*
11080          * NOTE: Assumes that the individual VL bits are adjacent and in
11081          * increasing order
11082          */
11083         stat_mask =
11084                 SEND_CM_CREDIT_USED_STATUS_VL0_RETURN_CREDIT_STATUS_SMASK;
11085         changing_mask = 0;
11086         ld_mask = 0;
11087         change_count = 0;
11088         any_shared_limit_changing = 0;
11089         for (i = 0; i < NUM_USABLE_VLS; i++, stat_mask <<= 1) {
11090                 if (!valid_vl(i))
11091                         continue;
11092                 this_shared_changing = new_bc->vl[i].shared
11093                                                 != cur_bc.vl[i].shared;
11094                 if (this_shared_changing)
11095                         any_shared_limit_changing = 1;
11096                 if (new_bc->vl[i].dedicated != cur_bc.vl[i].dedicated ||
11097                     this_shared_changing) {
11098                         changing[i] = 1;
11099                         changing_mask |= stat_mask;
11100                         change_count++;
11101                 }
11102                 if (be16_to_cpu(new_bc->vl[i].dedicated) <
11103                                         be16_to_cpu(cur_bc.vl[i].dedicated)) {
11104                         lowering_dedicated[i] = 1;
11105                         ld_mask |= stat_mask;
11106                 }
11107         }
11108
11109         /* bracket the credit change with a total adjustment */
11110         if (new_total > cur_total)
11111                 set_global_limit(dd, new_total);
11112
11113         /*
11114          * Start the credit change algorithm.
11115          */
11116         use_all_mask = 0;
11117         if ((be16_to_cpu(new_bc->overall_shared_limit) <
11118              be16_to_cpu(cur_bc.overall_shared_limit)) ||
11119             (is_ax(dd) && any_shared_limit_changing)) {
11120                 set_global_shared(dd, 0);
11121                 cur_bc.overall_shared_limit = 0;
11122                 use_all_mask = 1;
11123         }
11124
11125         for (i = 0; i < NUM_USABLE_VLS; i++) {
11126                 if (!valid_vl(i))
11127                         continue;
11128
11129                 if (changing[i]) {
11130                         set_vl_shared(dd, i, 0);
11131                         cur_bc.vl[i].shared = 0;
11132                 }
11133         }
11134
11135         wait_for_vl_status_clear(dd, use_all_mask ? all_mask : changing_mask,
11136                                  "shared");
11137
11138         if (change_count > 0) {
11139                 for (i = 0; i < NUM_USABLE_VLS; i++) {
11140                         if (!valid_vl(i))
11141                                 continue;
11142
11143                         if (lowering_dedicated[i]) {
11144                                 set_vl_dedicated(dd, i,
11145                                                  be16_to_cpu(new_bc->
11146                                                              vl[i].dedicated));
11147                                 cur_bc.vl[i].dedicated =
11148                                                 new_bc->vl[i].dedicated;
11149                         }
11150                 }
11151
11152                 wait_for_vl_status_clear(dd, ld_mask, "dedicated");
11153
11154                 /* now raise all dedicated that are going up */
11155                 for (i = 0; i < NUM_USABLE_VLS; i++) {
11156                         if (!valid_vl(i))
11157                                 continue;
11158
11159                         if (be16_to_cpu(new_bc->vl[i].dedicated) >
11160                                         be16_to_cpu(cur_bc.vl[i].dedicated))
11161                                 set_vl_dedicated(dd, i,
11162                                                  be16_to_cpu(new_bc->
11163                                                              vl[i].dedicated));
11164                 }
11165         }
11166
11167         /* next raise all shared that are going up */
11168         for (i = 0; i < NUM_USABLE_VLS; i++) {
11169                 if (!valid_vl(i))
11170                         continue;
11171
11172                 if (be16_to_cpu(new_bc->vl[i].shared) >
11173                                 be16_to_cpu(cur_bc.vl[i].shared))
11174                         set_vl_shared(dd, i, be16_to_cpu(new_bc->vl[i].shared));
11175         }
11176
11177         /* finally raise the global shared */
11178         if (be16_to_cpu(new_bc->overall_shared_limit) >
11179             be16_to_cpu(cur_bc.overall_shared_limit))
11180                 set_global_shared(dd,
11181                                   be16_to_cpu(new_bc->overall_shared_limit));
11182
11183         /* bracket the credit change with a total adjustment */
11184         if (new_total < cur_total)
11185                 set_global_limit(dd, new_total);
11186
11187         /*
11188          * Determine the actual number of operational VLS using the number of
11189          * dedicated and shared credits for each VL.
11190          */
11191         if (change_count > 0) {
11192                 for (i = 0; i < TXE_NUM_DATA_VL; i++)
11193                         if (be16_to_cpu(new_bc->vl[i].dedicated) > 0 ||
11194                             be16_to_cpu(new_bc->vl[i].shared) > 0)
11195                                 vl_count++;
11196                 ppd->actual_vls_operational = vl_count;
11197                 ret = sdma_map_init(dd, ppd->port - 1, vl_count ?
11198                                     ppd->actual_vls_operational :
11199                                     ppd->vls_operational,
11200                                     NULL);
11201                 if (ret == 0)
11202                         ret = pio_map_init(dd, ppd->port - 1, vl_count ?
11203                                            ppd->actual_vls_operational :
11204                                            ppd->vls_operational, NULL);
11205                 if (ret)
11206                         return ret;
11207         }
11208         return 0;
11209 }
11210
11211 /*
11212  * Read the given fabric manager table. Return the size of the
11213  * table (in bytes) on success, and a negative error code on
11214  * failure.
11215  */
11216 int fm_get_table(struct hfi1_pportdata *ppd, int which, void *t)
11217
11218 {
11219         int size;
11220         struct vl_arb_cache *vlc;
11221
11222         switch (which) {
11223         case FM_TBL_VL_HIGH_ARB:
11224                 size = 256;
11225                 /*
11226                  * OPA specifies 128 elements (of 2 bytes each), though
11227                  * HFI supports only 16 elements in h/w.
11228                  */
11229                 vlc = vl_arb_lock_cache(ppd, HI_PRIO_TABLE);
11230                 vl_arb_get_cache(vlc, t);
11231                 vl_arb_unlock_cache(ppd, HI_PRIO_TABLE);
11232                 break;
11233         case FM_TBL_VL_LOW_ARB:
11234                 size = 256;
11235                 /*
11236                  * OPA specifies 128 elements (of 2 bytes each), though
11237                  * HFI supports only 16 elements in h/w.
11238                  */
11239                 vlc = vl_arb_lock_cache(ppd, LO_PRIO_TABLE);
11240                 vl_arb_get_cache(vlc, t);
11241                 vl_arb_unlock_cache(ppd, LO_PRIO_TABLE);
11242                 break;
11243         case FM_TBL_BUFFER_CONTROL:
11244                 size = get_buffer_control(ppd->dd, t, NULL);
11245                 break;
11246         case FM_TBL_SC2VLNT:
11247                 size = get_sc2vlnt(ppd->dd, t);
11248                 break;
11249         case FM_TBL_VL_PREEMPT_ELEMS:
11250                 size = 256;
11251                 /* OPA specifies 128 elements, of 2 bytes each */
11252                 get_vlarb_preempt(ppd->dd, OPA_MAX_VLS, t);
11253                 break;
11254         case FM_TBL_VL_PREEMPT_MATRIX:
11255                 size = 256;
11256                 /*
11257                  * OPA specifies that this is the same size as the VL
11258                  * arbitration tables (i.e., 256 bytes).
11259                  */
11260                 break;
11261         default:
11262                 return -EINVAL;
11263         }
11264         return size;
11265 }
11266
11267 /*
11268  * Write the given fabric manager table.
11269  */
11270 int fm_set_table(struct hfi1_pportdata *ppd, int which, void *t)
11271 {
11272         int ret = 0;
11273         struct vl_arb_cache *vlc;
11274
11275         switch (which) {
11276         case FM_TBL_VL_HIGH_ARB:
11277                 vlc = vl_arb_lock_cache(ppd, HI_PRIO_TABLE);
11278                 if (vl_arb_match_cache(vlc, t)) {
11279                         vl_arb_unlock_cache(ppd, HI_PRIO_TABLE);
11280                         break;
11281                 }
11282                 vl_arb_set_cache(vlc, t);
11283                 vl_arb_unlock_cache(ppd, HI_PRIO_TABLE);
11284                 ret = set_vl_weights(ppd, SEND_HIGH_PRIORITY_LIST,
11285                                      VL_ARB_HIGH_PRIO_TABLE_SIZE, t);
11286                 break;
11287         case FM_TBL_VL_LOW_ARB:
11288                 vlc = vl_arb_lock_cache(ppd, LO_PRIO_TABLE);
11289                 if (vl_arb_match_cache(vlc, t)) {
11290                         vl_arb_unlock_cache(ppd, LO_PRIO_TABLE);
11291                         break;
11292                 }
11293                 vl_arb_set_cache(vlc, t);
11294                 vl_arb_unlock_cache(ppd, LO_PRIO_TABLE);
11295                 ret = set_vl_weights(ppd, SEND_LOW_PRIORITY_LIST,
11296                                      VL_ARB_LOW_PRIO_TABLE_SIZE, t);
11297                 break;
11298         case FM_TBL_BUFFER_CONTROL:
11299                 ret = set_buffer_control(ppd, t);
11300                 break;
11301         case FM_TBL_SC2VLNT:
11302                 set_sc2vlnt(ppd->dd, t);
11303                 break;
11304         default:
11305                 ret = -EINVAL;
11306         }
11307         return ret;
11308 }
11309
11310 /*
11311  * Disable all data VLs.
11312  *
11313  * Return 0 if disabled, non-zero if the VLs cannot be disabled.
11314  */
11315 static int disable_data_vls(struct hfi1_devdata *dd)
11316 {
11317         if (is_ax(dd))
11318                 return 1;
11319
11320         pio_send_control(dd, PSC_DATA_VL_DISABLE);
11321
11322         return 0;
11323 }
11324
11325 /*
11326  * open_fill_data_vls() - the counterpart to stop_drain_data_vls().
11327  * Just re-enables all data VLs (the "fill" part happens
11328  * automatically - the name was chosen for symmetry with
11329  * stop_drain_data_vls()).
11330  *
11331  * Return 0 if successful, non-zero if the VLs cannot be enabled.
11332  */
11333 int open_fill_data_vls(struct hfi1_devdata *dd)
11334 {
11335         if (is_ax(dd))
11336                 return 1;
11337
11338         pio_send_control(dd, PSC_DATA_VL_ENABLE);
11339
11340         return 0;
11341 }
11342
11343 /*
11344  * drain_data_vls() - assumes that disable_data_vls() has been called,
11345  * wait for occupancy (of per-VL FIFOs) for all contexts, and SDMA
11346  * engines to drop to 0.
11347  */
11348 static void drain_data_vls(struct hfi1_devdata *dd)
11349 {
11350         sc_wait(dd);
11351         sdma_wait(dd);
11352         pause_for_credit_return(dd);
11353 }
11354
11355 /*
11356  * stop_drain_data_vls() - disable, then drain all per-VL fifos.
11357  *
11358  * Use open_fill_data_vls() to resume using data VLs.  This pair is
11359  * meant to be used like this:
11360  *
11361  * stop_drain_data_vls(dd);
11362  * // do things with per-VL resources
11363  * open_fill_data_vls(dd);
11364  */
11365 int stop_drain_data_vls(struct hfi1_devdata *dd)
11366 {
11367         int ret;
11368
11369         ret = disable_data_vls(dd);
11370         if (ret == 0)
11371                 drain_data_vls(dd);
11372
11373         return ret;
11374 }
11375
11376 /*
11377  * Convert a nanosecond time to a cclock count.  No matter how slow
11378  * the cclock, a non-zero ns will always have a non-zero result.
11379  */
11380 u32 ns_to_cclock(struct hfi1_devdata *dd, u32 ns)
11381 {
11382         u32 cclocks;
11383
11384         if (dd->icode == ICODE_FPGA_EMULATION)
11385                 cclocks = (ns * 1000) / FPGA_CCLOCK_PS;
11386         else  /* simulation pretends to be ASIC */
11387                 cclocks = (ns * 1000) / ASIC_CCLOCK_PS;
11388         if (ns && !cclocks)     /* if ns nonzero, must be at least 1 */
11389                 cclocks = 1;
11390         return cclocks;
11391 }
11392
11393 /*
11394  * Convert a cclock count to nanoseconds. Not matter how slow
11395  * the cclock, a non-zero cclocks will always have a non-zero result.
11396  */
11397 u32 cclock_to_ns(struct hfi1_devdata *dd, u32 cclocks)
11398 {
11399         u32 ns;
11400
11401         if (dd->icode == ICODE_FPGA_EMULATION)
11402                 ns = (cclocks * FPGA_CCLOCK_PS) / 1000;
11403         else  /* simulation pretends to be ASIC */
11404                 ns = (cclocks * ASIC_CCLOCK_PS) / 1000;
11405         if (cclocks && !ns)
11406                 ns = 1;
11407         return ns;
11408 }
11409
11410 /*
11411  * Dynamically adjust the receive interrupt timeout for a context based on
11412  * incoming packet rate.
11413  *
11414  * NOTE: Dynamic adjustment does not allow rcv_intr_count to be zero.
11415  */
11416 static void adjust_rcv_timeout(struct hfi1_ctxtdata *rcd, u32 npkts)
11417 {
11418         struct hfi1_devdata *dd = rcd->dd;
11419         u32 timeout = rcd->rcvavail_timeout;
11420
11421         /*
11422          * This algorithm doubles or halves the timeout depending on whether
11423          * the number of packets received in this interrupt were less than or
11424          * greater equal the interrupt count.
11425          *
11426          * The calculations below do not allow a steady state to be achieved.
11427          * Only at the endpoints it is possible to have an unchanging
11428          * timeout.
11429          */
11430         if (npkts < rcv_intr_count) {
11431                 /*
11432                  * Not enough packets arrived before the timeout, adjust
11433                  * timeout downward.
11434                  */
11435                 if (timeout < 2) /* already at minimum? */
11436                         return;
11437                 timeout >>= 1;
11438         } else {
11439                 /*
11440                  * More than enough packets arrived before the timeout, adjust
11441                  * timeout upward.
11442                  */
11443                 if (timeout >= dd->rcv_intr_timeout_csr) /* already at max? */
11444                         return;
11445                 timeout = min(timeout << 1, dd->rcv_intr_timeout_csr);
11446         }
11447
11448         rcd->rcvavail_timeout = timeout;
11449         /*
11450          * timeout cannot be larger than rcv_intr_timeout_csr which has already
11451          * been verified to be in range
11452          */
11453         write_kctxt_csr(dd, rcd->ctxt, RCV_AVAIL_TIME_OUT,
11454                         (u64)timeout <<
11455                         RCV_AVAIL_TIME_OUT_TIME_OUT_RELOAD_SHIFT);
11456 }
11457
11458 void update_usrhead(struct hfi1_ctxtdata *rcd, u32 hd, u32 updegr, u32 egrhd,
11459                     u32 intr_adjust, u32 npkts)
11460 {
11461         struct hfi1_devdata *dd = rcd->dd;
11462         u64 reg;
11463         u32 ctxt = rcd->ctxt;
11464
11465         /*
11466          * Need to write timeout register before updating RcvHdrHead to ensure
11467          * that a new value is used when the HW decides to restart counting.
11468          */
11469         if (intr_adjust)
11470                 adjust_rcv_timeout(rcd, npkts);
11471         if (updegr) {
11472                 reg = (egrhd & RCV_EGR_INDEX_HEAD_HEAD_MASK)
11473                         << RCV_EGR_INDEX_HEAD_HEAD_SHIFT;
11474                 write_uctxt_csr(dd, ctxt, RCV_EGR_INDEX_HEAD, reg);
11475         }
11476         mmiowb();
11477         reg = ((u64)rcv_intr_count << RCV_HDR_HEAD_COUNTER_SHIFT) |
11478                 (((u64)hd & RCV_HDR_HEAD_HEAD_MASK)
11479                         << RCV_HDR_HEAD_HEAD_SHIFT);
11480         write_uctxt_csr(dd, ctxt, RCV_HDR_HEAD, reg);
11481         mmiowb();
11482 }
11483
11484 u32 hdrqempty(struct hfi1_ctxtdata *rcd)
11485 {
11486         u32 head, tail;
11487
11488         head = (read_uctxt_csr(rcd->dd, rcd->ctxt, RCV_HDR_HEAD)
11489                 & RCV_HDR_HEAD_HEAD_SMASK) >> RCV_HDR_HEAD_HEAD_SHIFT;
11490
11491         if (rcd->rcvhdrtail_kvaddr)
11492                 tail = get_rcvhdrtail(rcd);
11493         else
11494                 tail = read_uctxt_csr(rcd->dd, rcd->ctxt, RCV_HDR_TAIL);
11495
11496         return head == tail;
11497 }
11498
11499 /*
11500  * Context Control and Receive Array encoding for buffer size:
11501  *      0x0 invalid
11502  *      0x1   4 KB
11503  *      0x2   8 KB
11504  *      0x3  16 KB
11505  *      0x4  32 KB
11506  *      0x5  64 KB
11507  *      0x6 128 KB
11508  *      0x7 256 KB
11509  *      0x8 512 KB (Receive Array only)
11510  *      0x9   1 MB (Receive Array only)
11511  *      0xa   2 MB (Receive Array only)
11512  *
11513  *      0xB-0xF - reserved (Receive Array only)
11514  *
11515  *
11516  * This routine assumes that the value has already been sanity checked.
11517  */
11518 static u32 encoded_size(u32 size)
11519 {
11520         switch (size) {
11521         case   4 * 1024: return 0x1;
11522         case   8 * 1024: return 0x2;
11523         case  16 * 1024: return 0x3;
11524         case  32 * 1024: return 0x4;
11525         case  64 * 1024: return 0x5;
11526         case 128 * 1024: return 0x6;
11527         case 256 * 1024: return 0x7;
11528         case 512 * 1024: return 0x8;
11529         case   1 * 1024 * 1024: return 0x9;
11530         case   2 * 1024 * 1024: return 0xa;
11531         }
11532         return 0x1;     /* if invalid, go with the minimum size */
11533 }
11534
11535 void hfi1_rcvctrl(struct hfi1_devdata *dd, unsigned int op, int ctxt)
11536 {
11537         struct hfi1_ctxtdata *rcd;
11538         u64 rcvctrl, reg;
11539         int did_enable = 0;
11540
11541         rcd = dd->rcd[ctxt];
11542         if (!rcd)
11543                 return;
11544
11545         hfi1_cdbg(RCVCTRL, "ctxt %d op 0x%x", ctxt, op);
11546
11547         rcvctrl = read_kctxt_csr(dd, ctxt, RCV_CTXT_CTRL);
11548         /* if the context already enabled, don't do the extra steps */
11549         if ((op & HFI1_RCVCTRL_CTXT_ENB) &&
11550             !(rcvctrl & RCV_CTXT_CTRL_ENABLE_SMASK)) {
11551                 /* reset the tail and hdr addresses, and sequence count */
11552                 write_kctxt_csr(dd, ctxt, RCV_HDR_ADDR,
11553                                 rcd->rcvhdrq_dma);
11554                 if (rcd->rcvhdrtail_kvaddr)
11555                         write_kctxt_csr(dd, ctxt, RCV_HDR_TAIL_ADDR,
11556                                         rcd->rcvhdrqtailaddr_dma);
11557                 rcd->seq_cnt = 1;
11558
11559                 /* reset the cached receive header queue head value */
11560                 rcd->head = 0;
11561
11562                 /*
11563                  * Zero the receive header queue so we don't get false
11564                  * positives when checking the sequence number.  The
11565                  * sequence numbers could land exactly on the same spot.
11566                  * E.g. a rcd restart before the receive header wrapped.
11567                  */
11568                 memset(rcd->rcvhdrq, 0, rcd->rcvhdrq_size);
11569
11570                 /* starting timeout */
11571                 rcd->rcvavail_timeout = dd->rcv_intr_timeout_csr;
11572
11573                 /* enable the context */
11574                 rcvctrl |= RCV_CTXT_CTRL_ENABLE_SMASK;
11575
11576                 /* clean the egr buffer size first */
11577                 rcvctrl &= ~RCV_CTXT_CTRL_EGR_BUF_SIZE_SMASK;
11578                 rcvctrl |= ((u64)encoded_size(rcd->egrbufs.rcvtid_size)
11579                                 & RCV_CTXT_CTRL_EGR_BUF_SIZE_MASK)
11580                                         << RCV_CTXT_CTRL_EGR_BUF_SIZE_SHIFT;
11581
11582                 /* zero RcvHdrHead - set RcvHdrHead.Counter after enable */
11583                 write_uctxt_csr(dd, ctxt, RCV_HDR_HEAD, 0);
11584                 did_enable = 1;
11585
11586                 /* zero RcvEgrIndexHead */
11587                 write_uctxt_csr(dd, ctxt, RCV_EGR_INDEX_HEAD, 0);
11588
11589                 /* set eager count and base index */
11590                 reg = (((u64)(rcd->egrbufs.alloced >> RCV_SHIFT)
11591                         & RCV_EGR_CTRL_EGR_CNT_MASK)
11592                        << RCV_EGR_CTRL_EGR_CNT_SHIFT) |
11593                         (((rcd->eager_base >> RCV_SHIFT)
11594                           & RCV_EGR_CTRL_EGR_BASE_INDEX_MASK)
11595                          << RCV_EGR_CTRL_EGR_BASE_INDEX_SHIFT);
11596                 write_kctxt_csr(dd, ctxt, RCV_EGR_CTRL, reg);
11597
11598                 /*
11599                  * Set TID (expected) count and base index.
11600                  * rcd->expected_count is set to individual RcvArray entries,
11601                  * not pairs, and the CSR takes a pair-count in groups of
11602                  * four, so divide by 8.
11603                  */
11604                 reg = (((rcd->expected_count >> RCV_SHIFT)
11605                                         & RCV_TID_CTRL_TID_PAIR_CNT_MASK)
11606                                 << RCV_TID_CTRL_TID_PAIR_CNT_SHIFT) |
11607                       (((rcd->expected_base >> RCV_SHIFT)
11608                                         & RCV_TID_CTRL_TID_BASE_INDEX_MASK)
11609                                 << RCV_TID_CTRL_TID_BASE_INDEX_SHIFT);
11610                 write_kctxt_csr(dd, ctxt, RCV_TID_CTRL, reg);
11611                 if (ctxt == HFI1_CTRL_CTXT)
11612                         write_csr(dd, RCV_VL15, HFI1_CTRL_CTXT);
11613         }
11614         if (op & HFI1_RCVCTRL_CTXT_DIS) {
11615                 write_csr(dd, RCV_VL15, 0);
11616                 /*
11617                  * When receive context is being disabled turn on tail
11618                  * update with a dummy tail address and then disable
11619                  * receive context.
11620                  */
11621                 if (dd->rcvhdrtail_dummy_dma) {
11622                         write_kctxt_csr(dd, ctxt, RCV_HDR_TAIL_ADDR,
11623                                         dd->rcvhdrtail_dummy_dma);
11624                         /* Enabling RcvCtxtCtrl.TailUpd is intentional. */
11625                         rcvctrl |= RCV_CTXT_CTRL_TAIL_UPD_SMASK;
11626                 }
11627
11628                 rcvctrl &= ~RCV_CTXT_CTRL_ENABLE_SMASK;
11629         }
11630         if (op & HFI1_RCVCTRL_INTRAVAIL_ENB)
11631                 rcvctrl |= RCV_CTXT_CTRL_INTR_AVAIL_SMASK;
11632         if (op & HFI1_RCVCTRL_INTRAVAIL_DIS)
11633                 rcvctrl &= ~RCV_CTXT_CTRL_INTR_AVAIL_SMASK;
11634         if ((op & HFI1_RCVCTRL_TAILUPD_ENB) && rcd->rcvhdrtail_kvaddr)
11635                 rcvctrl |= RCV_CTXT_CTRL_TAIL_UPD_SMASK;
11636         if (op & HFI1_RCVCTRL_TAILUPD_DIS) {
11637                 /* See comment on RcvCtxtCtrl.TailUpd above */
11638                 if (!(op & HFI1_RCVCTRL_CTXT_DIS))
11639                         rcvctrl &= ~RCV_CTXT_CTRL_TAIL_UPD_SMASK;
11640         }
11641         if (op & HFI1_RCVCTRL_TIDFLOW_ENB)
11642                 rcvctrl |= RCV_CTXT_CTRL_TID_FLOW_ENABLE_SMASK;
11643         if (op & HFI1_RCVCTRL_TIDFLOW_DIS)
11644                 rcvctrl &= ~RCV_CTXT_CTRL_TID_FLOW_ENABLE_SMASK;
11645         if (op & HFI1_RCVCTRL_ONE_PKT_EGR_ENB) {
11646                 /*
11647                  * In one-packet-per-eager mode, the size comes from
11648                  * the RcvArray entry.
11649                  */
11650                 rcvctrl &= ~RCV_CTXT_CTRL_EGR_BUF_SIZE_SMASK;
11651                 rcvctrl |= RCV_CTXT_CTRL_ONE_PACKET_PER_EGR_BUFFER_SMASK;
11652         }
11653         if (op & HFI1_RCVCTRL_ONE_PKT_EGR_DIS)
11654                 rcvctrl &= ~RCV_CTXT_CTRL_ONE_PACKET_PER_EGR_BUFFER_SMASK;
11655         if (op & HFI1_RCVCTRL_NO_RHQ_DROP_ENB)
11656                 rcvctrl |= RCV_CTXT_CTRL_DONT_DROP_RHQ_FULL_SMASK;
11657         if (op & HFI1_RCVCTRL_NO_RHQ_DROP_DIS)
11658                 rcvctrl &= ~RCV_CTXT_CTRL_DONT_DROP_RHQ_FULL_SMASK;
11659         if (op & HFI1_RCVCTRL_NO_EGR_DROP_ENB)
11660                 rcvctrl |= RCV_CTXT_CTRL_DONT_DROP_EGR_FULL_SMASK;
11661         if (op & HFI1_RCVCTRL_NO_EGR_DROP_DIS)
11662                 rcvctrl &= ~RCV_CTXT_CTRL_DONT_DROP_EGR_FULL_SMASK;
11663         rcd->rcvctrl = rcvctrl;
11664         hfi1_cdbg(RCVCTRL, "ctxt %d rcvctrl 0x%llx\n", ctxt, rcvctrl);
11665         write_kctxt_csr(dd, ctxt, RCV_CTXT_CTRL, rcd->rcvctrl);
11666
11667         /* work around sticky RcvCtxtStatus.BlockedRHQFull */
11668         if (did_enable &&
11669             (rcvctrl & RCV_CTXT_CTRL_DONT_DROP_RHQ_FULL_SMASK)) {
11670                 reg = read_kctxt_csr(dd, ctxt, RCV_CTXT_STATUS);
11671                 if (reg != 0) {
11672                         dd_dev_info(dd, "ctxt %d status %lld (blocked)\n",
11673                                     ctxt, reg);
11674                         read_uctxt_csr(dd, ctxt, RCV_HDR_HEAD);
11675                         write_uctxt_csr(dd, ctxt, RCV_HDR_HEAD, 0x10);
11676                         write_uctxt_csr(dd, ctxt, RCV_HDR_HEAD, 0x00);
11677                         read_uctxt_csr(dd, ctxt, RCV_HDR_HEAD);
11678                         reg = read_kctxt_csr(dd, ctxt, RCV_CTXT_STATUS);
11679                         dd_dev_info(dd, "ctxt %d status %lld (%s blocked)\n",
11680                                     ctxt, reg, reg == 0 ? "not" : "still");
11681                 }
11682         }
11683
11684         if (did_enable) {
11685                 /*
11686                  * The interrupt timeout and count must be set after
11687                  * the context is enabled to take effect.
11688                  */
11689                 /* set interrupt timeout */
11690                 write_kctxt_csr(dd, ctxt, RCV_AVAIL_TIME_OUT,
11691                                 (u64)rcd->rcvavail_timeout <<
11692                                 RCV_AVAIL_TIME_OUT_TIME_OUT_RELOAD_SHIFT);
11693
11694                 /* set RcvHdrHead.Counter, zero RcvHdrHead.Head (again) */
11695                 reg = (u64)rcv_intr_count << RCV_HDR_HEAD_COUNTER_SHIFT;
11696                 write_uctxt_csr(dd, ctxt, RCV_HDR_HEAD, reg);
11697         }
11698
11699         if (op & (HFI1_RCVCTRL_TAILUPD_DIS | HFI1_RCVCTRL_CTXT_DIS))
11700                 /*
11701                  * If the context has been disabled and the Tail Update has
11702                  * been cleared, set the RCV_HDR_TAIL_ADDR CSR to dummy address
11703                  * so it doesn't contain an address that is invalid.
11704                  */
11705                 write_kctxt_csr(dd, ctxt, RCV_HDR_TAIL_ADDR,
11706                                 dd->rcvhdrtail_dummy_dma);
11707 }
11708
11709 u32 hfi1_read_cntrs(struct hfi1_devdata *dd, char **namep, u64 **cntrp)
11710 {
11711         int ret;
11712         u64 val = 0;
11713
11714         if (namep) {
11715                 ret = dd->cntrnameslen;
11716                 *namep = dd->cntrnames;
11717         } else {
11718                 const struct cntr_entry *entry;
11719                 int i, j;
11720
11721                 ret = (dd->ndevcntrs) * sizeof(u64);
11722
11723                 /* Get the start of the block of counters */
11724                 *cntrp = dd->cntrs;
11725
11726                 /*
11727                  * Now go and fill in each counter in the block.
11728                  */
11729                 for (i = 0; i < DEV_CNTR_LAST; i++) {
11730                         entry = &dev_cntrs[i];
11731                         hfi1_cdbg(CNTR, "reading %s", entry->name);
11732                         if (entry->flags & CNTR_DISABLED) {
11733                                 /* Nothing */
11734                                 hfi1_cdbg(CNTR, "\tDisabled\n");
11735                         } else {
11736                                 if (entry->flags & CNTR_VL) {
11737                                         hfi1_cdbg(CNTR, "\tPer VL\n");
11738                                         for (j = 0; j < C_VL_COUNT; j++) {
11739                                                 val = entry->rw_cntr(entry,
11740                                                                   dd, j,
11741                                                                   CNTR_MODE_R,
11742                                                                   0);
11743                                                 hfi1_cdbg(
11744                                                    CNTR,
11745                                                    "\t\tRead 0x%llx for %d\n",
11746                                                    val, j);
11747                                                 dd->cntrs[entry->offset + j] =
11748                                                                             val;
11749                                         }
11750                                 } else if (entry->flags & CNTR_SDMA) {
11751                                         hfi1_cdbg(CNTR,
11752                                                   "\t Per SDMA Engine\n");
11753                                         for (j = 0; j < dd->chip_sdma_engines;
11754                                              j++) {
11755                                                 val =
11756                                                 entry->rw_cntr(entry, dd, j,
11757                                                                CNTR_MODE_R, 0);
11758                                                 hfi1_cdbg(CNTR,
11759                                                           "\t\tRead 0x%llx for %d\n",
11760                                                           val, j);
11761                                                 dd->cntrs[entry->offset + j] =
11762                                                                         val;
11763                                         }
11764                                 } else {
11765                                         val = entry->rw_cntr(entry, dd,
11766                                                         CNTR_INVALID_VL,
11767                                                         CNTR_MODE_R, 0);
11768                                         dd->cntrs[entry->offset] = val;
11769                                         hfi1_cdbg(CNTR, "\tRead 0x%llx", val);
11770                                 }
11771                         }
11772                 }
11773         }
11774         return ret;
11775 }
11776
11777 /*
11778  * Used by sysfs to create files for hfi stats to read
11779  */
11780 u32 hfi1_read_portcntrs(struct hfi1_pportdata *ppd, char **namep, u64 **cntrp)
11781 {
11782         int ret;
11783         u64 val = 0;
11784
11785         if (namep) {
11786                 ret = ppd->dd->portcntrnameslen;
11787                 *namep = ppd->dd->portcntrnames;
11788         } else {
11789                 const struct cntr_entry *entry;
11790                 int i, j;
11791
11792                 ret = ppd->dd->nportcntrs * sizeof(u64);
11793                 *cntrp = ppd->cntrs;
11794
11795                 for (i = 0; i < PORT_CNTR_LAST; i++) {
11796                         entry = &port_cntrs[i];
11797                         hfi1_cdbg(CNTR, "reading %s", entry->name);
11798                         if (entry->flags & CNTR_DISABLED) {
11799                                 /* Nothing */
11800                                 hfi1_cdbg(CNTR, "\tDisabled\n");
11801                                 continue;
11802                         }
11803
11804                         if (entry->flags & CNTR_VL) {
11805                                 hfi1_cdbg(CNTR, "\tPer VL");
11806                                 for (j = 0; j < C_VL_COUNT; j++) {
11807                                         val = entry->rw_cntr(entry, ppd, j,
11808                                                                CNTR_MODE_R,
11809                                                                0);
11810                                         hfi1_cdbg(
11811                                            CNTR,
11812                                            "\t\tRead 0x%llx for %d",
11813                                            val, j);
11814                                         ppd->cntrs[entry->offset + j] = val;
11815                                 }
11816                         } else {
11817                                 val = entry->rw_cntr(entry, ppd,
11818                                                        CNTR_INVALID_VL,
11819                                                        CNTR_MODE_R,
11820                                                        0);
11821                                 ppd->cntrs[entry->offset] = val;
11822                                 hfi1_cdbg(CNTR, "\tRead 0x%llx", val);
11823                         }
11824                 }
11825         }
11826         return ret;
11827 }
11828
11829 static void free_cntrs(struct hfi1_devdata *dd)
11830 {
11831         struct hfi1_pportdata *ppd;
11832         int i;
11833
11834         if (dd->synth_stats_timer.data)
11835                 del_timer_sync(&dd->synth_stats_timer);
11836         dd->synth_stats_timer.data = 0;
11837         ppd = (struct hfi1_pportdata *)(dd + 1);
11838         for (i = 0; i < dd->num_pports; i++, ppd++) {
11839                 kfree(ppd->cntrs);
11840                 kfree(ppd->scntrs);
11841                 free_percpu(ppd->ibport_data.rvp.rc_acks);
11842                 free_percpu(ppd->ibport_data.rvp.rc_qacks);
11843                 free_percpu(ppd->ibport_data.rvp.rc_delayed_comp);
11844                 ppd->cntrs = NULL;
11845                 ppd->scntrs = NULL;
11846                 ppd->ibport_data.rvp.rc_acks = NULL;
11847                 ppd->ibport_data.rvp.rc_qacks = NULL;
11848                 ppd->ibport_data.rvp.rc_delayed_comp = NULL;
11849         }
11850         kfree(dd->portcntrnames);
11851         dd->portcntrnames = NULL;
11852         kfree(dd->cntrs);
11853         dd->cntrs = NULL;
11854         kfree(dd->scntrs);
11855         dd->scntrs = NULL;
11856         kfree(dd->cntrnames);
11857         dd->cntrnames = NULL;
11858         if (dd->update_cntr_wq) {
11859                 destroy_workqueue(dd->update_cntr_wq);
11860                 dd->update_cntr_wq = NULL;
11861         }
11862 }
11863
11864 static u64 read_dev_port_cntr(struct hfi1_devdata *dd, struct cntr_entry *entry,
11865                               u64 *psval, void *context, int vl)
11866 {
11867         u64 val;
11868         u64 sval = *psval;
11869
11870         if (entry->flags & CNTR_DISABLED) {
11871                 dd_dev_err(dd, "Counter %s not enabled", entry->name);
11872                 return 0;
11873         }
11874
11875         hfi1_cdbg(CNTR, "cntr: %s vl %d psval 0x%llx", entry->name, vl, *psval);
11876
11877         val = entry->rw_cntr(entry, context, vl, CNTR_MODE_R, 0);
11878
11879         /* If its a synthetic counter there is more work we need to do */
11880         if (entry->flags & CNTR_SYNTH) {
11881                 if (sval == CNTR_MAX) {
11882                         /* No need to read already saturated */
11883                         return CNTR_MAX;
11884                 }
11885
11886                 if (entry->flags & CNTR_32BIT) {
11887                         /* 32bit counters can wrap multiple times */
11888                         u64 upper = sval >> 32;
11889                         u64 lower = (sval << 32) >> 32;
11890
11891                         if (lower > val) { /* hw wrapped */
11892                                 if (upper == CNTR_32BIT_MAX)
11893                                         val = CNTR_MAX;
11894                                 else
11895                                         upper++;
11896                         }
11897
11898                         if (val != CNTR_MAX)
11899                                 val = (upper << 32) | val;
11900
11901                 } else {
11902                         /* If we rolled we are saturated */
11903                         if ((val < sval) || (val > CNTR_MAX))
11904                                 val = CNTR_MAX;
11905                 }
11906         }
11907
11908         *psval = val;
11909
11910         hfi1_cdbg(CNTR, "\tNew val=0x%llx", val);
11911
11912         return val;
11913 }
11914
11915 static u64 write_dev_port_cntr(struct hfi1_devdata *dd,
11916                                struct cntr_entry *entry,
11917                                u64 *psval, void *context, int vl, u64 data)
11918 {
11919         u64 val;
11920
11921         if (entry->flags & CNTR_DISABLED) {
11922                 dd_dev_err(dd, "Counter %s not enabled", entry->name);
11923                 return 0;
11924         }
11925
11926         hfi1_cdbg(CNTR, "cntr: %s vl %d psval 0x%llx", entry->name, vl, *psval);
11927
11928         if (entry->flags & CNTR_SYNTH) {
11929                 *psval = data;
11930                 if (entry->flags & CNTR_32BIT) {
11931                         val = entry->rw_cntr(entry, context, vl, CNTR_MODE_W,
11932                                              (data << 32) >> 32);
11933                         val = data; /* return the full 64bit value */
11934                 } else {
11935                         val = entry->rw_cntr(entry, context, vl, CNTR_MODE_W,
11936                                              data);
11937                 }
11938         } else {
11939                 val = entry->rw_cntr(entry, context, vl, CNTR_MODE_W, data);
11940         }
11941
11942         *psval = val;
11943
11944         hfi1_cdbg(CNTR, "\tNew val=0x%llx", val);
11945
11946         return val;
11947 }
11948
11949 u64 read_dev_cntr(struct hfi1_devdata *dd, int index, int vl)
11950 {
11951         struct cntr_entry *entry;
11952         u64 *sval;
11953
11954         entry = &dev_cntrs[index];
11955         sval = dd->scntrs + entry->offset;
11956
11957         if (vl != CNTR_INVALID_VL)
11958                 sval += vl;
11959
11960         return read_dev_port_cntr(dd, entry, sval, dd, vl);
11961 }
11962
11963 u64 write_dev_cntr(struct hfi1_devdata *dd, int index, int vl, u64 data)
11964 {
11965         struct cntr_entry *entry;
11966         u64 *sval;
11967
11968         entry = &dev_cntrs[index];
11969         sval = dd->scntrs + entry->offset;
11970
11971         if (vl != CNTR_INVALID_VL)
11972                 sval += vl;
11973
11974         return write_dev_port_cntr(dd, entry, sval, dd, vl, data);
11975 }
11976
11977 u64 read_port_cntr(struct hfi1_pportdata *ppd, int index, int vl)
11978 {
11979         struct cntr_entry *entry;
11980         u64 *sval;
11981
11982         entry = &port_cntrs[index];
11983         sval = ppd->scntrs + entry->offset;
11984
11985         if (vl != CNTR_INVALID_VL)
11986                 sval += vl;
11987
11988         if ((index >= C_RCV_HDR_OVF_FIRST + ppd->dd->num_rcv_contexts) &&
11989             (index <= C_RCV_HDR_OVF_LAST)) {
11990                 /* We do not want to bother for disabled contexts */
11991                 return 0;
11992         }
11993
11994         return read_dev_port_cntr(ppd->dd, entry, sval, ppd, vl);
11995 }
11996
11997 u64 write_port_cntr(struct hfi1_pportdata *ppd, int index, int vl, u64 data)
11998 {
11999         struct cntr_entry *entry;
12000         u64 *sval;
12001
12002         entry = &port_cntrs[index];
12003         sval = ppd->scntrs + entry->offset;
12004
12005         if (vl != CNTR_INVALID_VL)
12006                 sval += vl;
12007
12008         if ((index >= C_RCV_HDR_OVF_FIRST + ppd->dd->num_rcv_contexts) &&
12009             (index <= C_RCV_HDR_OVF_LAST)) {
12010                 /* We do not want to bother for disabled contexts */
12011                 return 0;
12012         }
12013
12014         return write_dev_port_cntr(ppd->dd, entry, sval, ppd, vl, data);
12015 }
12016
12017 static void do_update_synth_timer(struct work_struct *work)
12018 {
12019         u64 cur_tx;
12020         u64 cur_rx;
12021         u64 total_flits;
12022         u8 update = 0;
12023         int i, j, vl;
12024         struct hfi1_pportdata *ppd;
12025         struct cntr_entry *entry;
12026         struct hfi1_devdata *dd = container_of(work, struct hfi1_devdata,
12027                                                update_cntr_work);
12028
12029         /*
12030          * Rather than keep beating on the CSRs pick a minimal set that we can
12031          * check to watch for potential roll over. We can do this by looking at
12032          * the number of flits sent/recv. If the total flits exceeds 32bits then
12033          * we have to iterate all the counters and update.
12034          */
12035         entry = &dev_cntrs[C_DC_RCV_FLITS];
12036         cur_rx = entry->rw_cntr(entry, dd, CNTR_INVALID_VL, CNTR_MODE_R, 0);
12037
12038         entry = &dev_cntrs[C_DC_XMIT_FLITS];
12039         cur_tx = entry->rw_cntr(entry, dd, CNTR_INVALID_VL, CNTR_MODE_R, 0);
12040
12041         hfi1_cdbg(
12042             CNTR,
12043             "[%d] curr tx=0x%llx rx=0x%llx :: last tx=0x%llx rx=0x%llx\n",
12044             dd->unit, cur_tx, cur_rx, dd->last_tx, dd->last_rx);
12045
12046         if ((cur_tx < dd->last_tx) || (cur_rx < dd->last_rx)) {
12047                 /*
12048                  * May not be strictly necessary to update but it won't hurt and
12049                  * simplifies the logic here.
12050                  */
12051                 update = 1;
12052                 hfi1_cdbg(CNTR, "[%d] Tripwire counter rolled, updating",
12053                           dd->unit);
12054         } else {
12055                 total_flits = (cur_tx - dd->last_tx) + (cur_rx - dd->last_rx);
12056                 hfi1_cdbg(CNTR,
12057                           "[%d] total flits 0x%llx limit 0x%llx\n", dd->unit,
12058                           total_flits, (u64)CNTR_32BIT_MAX);
12059                 if (total_flits >= CNTR_32BIT_MAX) {
12060                         hfi1_cdbg(CNTR, "[%d] 32bit limit hit, updating",
12061                                   dd->unit);
12062                         update = 1;
12063                 }
12064         }
12065
12066         if (update) {
12067                 hfi1_cdbg(CNTR, "[%d] Updating dd and ppd counters", dd->unit);
12068                 for (i = 0; i < DEV_CNTR_LAST; i++) {
12069                         entry = &dev_cntrs[i];
12070                         if (entry->flags & CNTR_VL) {
12071                                 for (vl = 0; vl < C_VL_COUNT; vl++)
12072                                         read_dev_cntr(dd, i, vl);
12073                         } else {
12074                                 read_dev_cntr(dd, i, CNTR_INVALID_VL);
12075                         }
12076                 }
12077                 ppd = (struct hfi1_pportdata *)(dd + 1);
12078                 for (i = 0; i < dd->num_pports; i++, ppd++) {
12079                         for (j = 0; j < PORT_CNTR_LAST; j++) {
12080                                 entry = &port_cntrs[j];
12081                                 if (entry->flags & CNTR_VL) {
12082                                         for (vl = 0; vl < C_VL_COUNT; vl++)
12083                                                 read_port_cntr(ppd, j, vl);
12084                                 } else {
12085                                         read_port_cntr(ppd, j, CNTR_INVALID_VL);
12086                                 }
12087                         }
12088                 }
12089
12090                 /*
12091                  * We want the value in the register. The goal is to keep track
12092                  * of the number of "ticks" not the counter value. In other
12093                  * words if the register rolls we want to notice it and go ahead
12094                  * and force an update.
12095                  */
12096                 entry = &dev_cntrs[C_DC_XMIT_FLITS];
12097                 dd->last_tx = entry->rw_cntr(entry, dd, CNTR_INVALID_VL,
12098                                                 CNTR_MODE_R, 0);
12099
12100                 entry = &dev_cntrs[C_DC_RCV_FLITS];
12101                 dd->last_rx = entry->rw_cntr(entry, dd, CNTR_INVALID_VL,
12102                                                 CNTR_MODE_R, 0);
12103
12104                 hfi1_cdbg(CNTR, "[%d] setting last tx/rx to 0x%llx 0x%llx",
12105                           dd->unit, dd->last_tx, dd->last_rx);
12106
12107         } else {
12108                 hfi1_cdbg(CNTR, "[%d] No update necessary", dd->unit);
12109         }
12110 }
12111
12112 static void update_synth_timer(unsigned long opaque)
12113 {
12114         struct hfi1_devdata *dd = (struct hfi1_devdata *)opaque;
12115
12116         queue_work(dd->update_cntr_wq, &dd->update_cntr_work);
12117         mod_timer(&dd->synth_stats_timer, jiffies + HZ * SYNTH_CNT_TIME);
12118 }
12119
12120 #define C_MAX_NAME 16 /* 15 chars + one for /0 */
12121 static int init_cntrs(struct hfi1_devdata *dd)
12122 {
12123         int i, rcv_ctxts, j;
12124         size_t sz;
12125         char *p;
12126         char name[C_MAX_NAME];
12127         struct hfi1_pportdata *ppd;
12128         const char *bit_type_32 = ",32";
12129         const int bit_type_32_sz = strlen(bit_type_32);
12130
12131         /* set up the stats timer; the add_timer is done at the end */
12132         setup_timer(&dd->synth_stats_timer, update_synth_timer,
12133                     (unsigned long)dd);
12134
12135         /***********************/
12136         /* per device counters */
12137         /***********************/
12138
12139         /* size names and determine how many we have*/
12140         dd->ndevcntrs = 0;
12141         sz = 0;
12142
12143         for (i = 0; i < DEV_CNTR_LAST; i++) {
12144                 if (dev_cntrs[i].flags & CNTR_DISABLED) {
12145                         hfi1_dbg_early("\tSkipping %s\n", dev_cntrs[i].name);
12146                         continue;
12147                 }
12148
12149                 if (dev_cntrs[i].flags & CNTR_VL) {
12150                         dev_cntrs[i].offset = dd->ndevcntrs;
12151                         for (j = 0; j < C_VL_COUNT; j++) {
12152                                 snprintf(name, C_MAX_NAME, "%s%d",
12153                                          dev_cntrs[i].name, vl_from_idx(j));
12154                                 sz += strlen(name);
12155                                 /* Add ",32" for 32-bit counters */
12156                                 if (dev_cntrs[i].flags & CNTR_32BIT)
12157                                         sz += bit_type_32_sz;
12158                                 sz++;
12159                                 dd->ndevcntrs++;
12160                         }
12161                 } else if (dev_cntrs[i].flags & CNTR_SDMA) {
12162                         dev_cntrs[i].offset = dd->ndevcntrs;
12163                         for (j = 0; j < dd->chip_sdma_engines; j++) {
12164                                 snprintf(name, C_MAX_NAME, "%s%d",
12165                                          dev_cntrs[i].name, j);
12166                                 sz += strlen(name);
12167                                 /* Add ",32" for 32-bit counters */
12168                                 if (dev_cntrs[i].flags & CNTR_32BIT)
12169                                         sz += bit_type_32_sz;
12170                                 sz++;
12171                                 dd->ndevcntrs++;
12172                         }
12173                 } else {
12174                         /* +1 for newline. */
12175                         sz += strlen(dev_cntrs[i].name) + 1;
12176                         /* Add ",32" for 32-bit counters */
12177                         if (dev_cntrs[i].flags & CNTR_32BIT)
12178                                 sz += bit_type_32_sz;
12179                         dev_cntrs[i].offset = dd->ndevcntrs;
12180                         dd->ndevcntrs++;
12181                 }
12182         }
12183
12184         /* allocate space for the counter values */
12185         dd->cntrs = kcalloc(dd->ndevcntrs, sizeof(u64), GFP_KERNEL);
12186         if (!dd->cntrs)
12187                 goto bail;
12188
12189         dd->scntrs = kcalloc(dd->ndevcntrs, sizeof(u64), GFP_KERNEL);
12190         if (!dd->scntrs)
12191                 goto bail;
12192
12193         /* allocate space for the counter names */
12194         dd->cntrnameslen = sz;
12195         dd->cntrnames = kmalloc(sz, GFP_KERNEL);
12196         if (!dd->cntrnames)
12197                 goto bail;
12198
12199         /* fill in the names */
12200         for (p = dd->cntrnames, i = 0; i < DEV_CNTR_LAST; i++) {
12201                 if (dev_cntrs[i].flags & CNTR_DISABLED) {
12202                         /* Nothing */
12203                 } else if (dev_cntrs[i].flags & CNTR_VL) {
12204                         for (j = 0; j < C_VL_COUNT; j++) {
12205                                 snprintf(name, C_MAX_NAME, "%s%d",
12206                                          dev_cntrs[i].name,
12207                                          vl_from_idx(j));
12208                                 memcpy(p, name, strlen(name));
12209                                 p += strlen(name);
12210
12211                                 /* Counter is 32 bits */
12212                                 if (dev_cntrs[i].flags & CNTR_32BIT) {
12213                                         memcpy(p, bit_type_32, bit_type_32_sz);
12214                                         p += bit_type_32_sz;
12215                                 }
12216
12217                                 *p++ = '\n';
12218                         }
12219                 } else if (dev_cntrs[i].flags & CNTR_SDMA) {
12220                         for (j = 0; j < dd->chip_sdma_engines; j++) {
12221                                 snprintf(name, C_MAX_NAME, "%s%d",
12222                                          dev_cntrs[i].name, j);
12223                                 memcpy(p, name, strlen(name));
12224                                 p += strlen(name);
12225
12226                                 /* Counter is 32 bits */
12227                                 if (dev_cntrs[i].flags & CNTR_32BIT) {
12228                                         memcpy(p, bit_type_32, bit_type_32_sz);
12229                                         p += bit_type_32_sz;
12230                                 }
12231
12232                                 *p++ = '\n';
12233                         }
12234                 } else {
12235                         memcpy(p, dev_cntrs[i].name, strlen(dev_cntrs[i].name));
12236                         p += strlen(dev_cntrs[i].name);
12237
12238                         /* Counter is 32 bits */
12239                         if (dev_cntrs[i].flags & CNTR_32BIT) {
12240                                 memcpy(p, bit_type_32, bit_type_32_sz);
12241                                 p += bit_type_32_sz;
12242                         }
12243
12244                         *p++ = '\n';
12245                 }
12246         }
12247
12248         /*********************/
12249         /* per port counters */
12250         /*********************/
12251
12252         /*
12253          * Go through the counters for the overflows and disable the ones we
12254          * don't need. This varies based on platform so we need to do it
12255          * dynamically here.
12256          */
12257         rcv_ctxts = dd->num_rcv_contexts;
12258         for (i = C_RCV_HDR_OVF_FIRST + rcv_ctxts;
12259              i <= C_RCV_HDR_OVF_LAST; i++) {
12260                 port_cntrs[i].flags |= CNTR_DISABLED;
12261         }
12262
12263         /* size port counter names and determine how many we have*/
12264         sz = 0;
12265         dd->nportcntrs = 0;
12266         for (i = 0; i < PORT_CNTR_LAST; i++) {
12267                 if (port_cntrs[i].flags & CNTR_DISABLED) {
12268                         hfi1_dbg_early("\tSkipping %s\n", port_cntrs[i].name);
12269                         continue;
12270                 }
12271
12272                 if (port_cntrs[i].flags & CNTR_VL) {
12273                         port_cntrs[i].offset = dd->nportcntrs;
12274                         for (j = 0; j < C_VL_COUNT; j++) {
12275                                 snprintf(name, C_MAX_NAME, "%s%d",
12276                                          port_cntrs[i].name, vl_from_idx(j));
12277                                 sz += strlen(name);
12278                                 /* Add ",32" for 32-bit counters */
12279                                 if (port_cntrs[i].flags & CNTR_32BIT)
12280                                         sz += bit_type_32_sz;
12281                                 sz++;
12282                                 dd->nportcntrs++;
12283                         }
12284                 } else {
12285                         /* +1 for newline */
12286                         sz += strlen(port_cntrs[i].name) + 1;
12287                         /* Add ",32" for 32-bit counters */
12288                         if (port_cntrs[i].flags & CNTR_32BIT)
12289                                 sz += bit_type_32_sz;
12290                         port_cntrs[i].offset = dd->nportcntrs;
12291                         dd->nportcntrs++;
12292                 }
12293         }
12294
12295         /* allocate space for the counter names */
12296         dd->portcntrnameslen = sz;
12297         dd->portcntrnames = kmalloc(sz, GFP_KERNEL);
12298         if (!dd->portcntrnames)
12299                 goto bail;
12300
12301         /* fill in port cntr names */
12302         for (p = dd->portcntrnames, i = 0; i < PORT_CNTR_LAST; i++) {
12303                 if (port_cntrs[i].flags & CNTR_DISABLED)
12304                         continue;
12305
12306                 if (port_cntrs[i].flags & CNTR_VL) {
12307                         for (j = 0; j < C_VL_COUNT; j++) {
12308                                 snprintf(name, C_MAX_NAME, "%s%d",
12309                                          port_cntrs[i].name, vl_from_idx(j));
12310                                 memcpy(p, name, strlen(name));
12311                                 p += strlen(name);
12312
12313                                 /* Counter is 32 bits */
12314                                 if (port_cntrs[i].flags & CNTR_32BIT) {
12315                                         memcpy(p, bit_type_32, bit_type_32_sz);
12316                                         p += bit_type_32_sz;
12317                                 }
12318
12319                                 *p++ = '\n';
12320                         }
12321                 } else {
12322                         memcpy(p, port_cntrs[i].name,
12323                                strlen(port_cntrs[i].name));
12324                         p += strlen(port_cntrs[i].name);
12325
12326                         /* Counter is 32 bits */
12327                         if (port_cntrs[i].flags & CNTR_32BIT) {
12328                                 memcpy(p, bit_type_32, bit_type_32_sz);
12329                                 p += bit_type_32_sz;
12330                         }
12331
12332                         *p++ = '\n';
12333                 }
12334         }
12335
12336         /* allocate per port storage for counter values */
12337         ppd = (struct hfi1_pportdata *)(dd + 1);
12338         for (i = 0; i < dd->num_pports; i++, ppd++) {
12339                 ppd->cntrs = kcalloc(dd->nportcntrs, sizeof(u64), GFP_KERNEL);
12340                 if (!ppd->cntrs)
12341                         goto bail;
12342
12343                 ppd->scntrs = kcalloc(dd->nportcntrs, sizeof(u64), GFP_KERNEL);
12344                 if (!ppd->scntrs)
12345                         goto bail;
12346         }
12347
12348         /* CPU counters need to be allocated and zeroed */
12349         if (init_cpu_counters(dd))
12350                 goto bail;
12351
12352         dd->update_cntr_wq = alloc_ordered_workqueue("hfi1_update_cntr_%d",
12353                                                      WQ_MEM_RECLAIM, dd->unit);
12354         if (!dd->update_cntr_wq)
12355                 goto bail;
12356
12357         INIT_WORK(&dd->update_cntr_work, do_update_synth_timer);
12358
12359         mod_timer(&dd->synth_stats_timer, jiffies + HZ * SYNTH_CNT_TIME);
12360         return 0;
12361 bail:
12362         free_cntrs(dd);
12363         return -ENOMEM;
12364 }
12365
12366 static u32 chip_to_opa_lstate(struct hfi1_devdata *dd, u32 chip_lstate)
12367 {
12368         switch (chip_lstate) {
12369         default:
12370                 dd_dev_err(dd,
12371                            "Unknown logical state 0x%x, reporting IB_PORT_DOWN\n",
12372                            chip_lstate);
12373                 /* fall through */
12374         case LSTATE_DOWN:
12375                 return IB_PORT_DOWN;
12376         case LSTATE_INIT:
12377                 return IB_PORT_INIT;
12378         case LSTATE_ARMED:
12379                 return IB_PORT_ARMED;
12380         case LSTATE_ACTIVE:
12381                 return IB_PORT_ACTIVE;
12382         }
12383 }
12384
12385 u32 chip_to_opa_pstate(struct hfi1_devdata *dd, u32 chip_pstate)
12386 {
12387         /* look at the HFI meta-states only */
12388         switch (chip_pstate & 0xf0) {
12389         default:
12390                 dd_dev_err(dd, "Unexpected chip physical state of 0x%x\n",
12391                            chip_pstate);
12392                 /* fall through */
12393         case PLS_DISABLED:
12394                 return IB_PORTPHYSSTATE_DISABLED;
12395         case PLS_OFFLINE:
12396                 return OPA_PORTPHYSSTATE_OFFLINE;
12397         case PLS_POLLING:
12398                 return IB_PORTPHYSSTATE_POLLING;
12399         case PLS_CONFIGPHY:
12400                 return IB_PORTPHYSSTATE_TRAINING;
12401         case PLS_LINKUP:
12402                 return IB_PORTPHYSSTATE_LINKUP;
12403         case PLS_PHYTEST:
12404                 return IB_PORTPHYSSTATE_PHY_TEST;
12405         }
12406 }
12407
12408 /* return the OPA port logical state name */
12409 const char *opa_lstate_name(u32 lstate)
12410 {
12411         static const char * const port_logical_names[] = {
12412                 "PORT_NOP",
12413                 "PORT_DOWN",
12414                 "PORT_INIT",
12415                 "PORT_ARMED",
12416                 "PORT_ACTIVE",
12417                 "PORT_ACTIVE_DEFER",
12418         };
12419         if (lstate < ARRAY_SIZE(port_logical_names))
12420                 return port_logical_names[lstate];
12421         return "unknown";
12422 }
12423
12424 /* return the OPA port physical state name */
12425 const char *opa_pstate_name(u32 pstate)
12426 {
12427         static const char * const port_physical_names[] = {
12428                 "PHYS_NOP",
12429                 "reserved1",
12430                 "PHYS_POLL",
12431                 "PHYS_DISABLED",
12432                 "PHYS_TRAINING",
12433                 "PHYS_LINKUP",
12434                 "PHYS_LINK_ERR_RECOVER",
12435                 "PHYS_PHY_TEST",
12436                 "reserved8",
12437                 "PHYS_OFFLINE",
12438                 "PHYS_GANGED",
12439                 "PHYS_TEST",
12440         };
12441         if (pstate < ARRAY_SIZE(port_physical_names))
12442                 return port_physical_names[pstate];
12443         return "unknown";
12444 }
12445
12446 /*
12447  * Read the hardware link state and set the driver's cached value of it.
12448  * Return the (new) current value.
12449  */
12450 u32 get_logical_state(struct hfi1_pportdata *ppd)
12451 {
12452         u32 new_state;
12453
12454         new_state = chip_to_opa_lstate(ppd->dd, read_logical_state(ppd->dd));
12455         if (new_state != ppd->lstate) {
12456                 dd_dev_info(ppd->dd, "logical state changed to %s (0x%x)\n",
12457                             opa_lstate_name(new_state), new_state);
12458                 ppd->lstate = new_state;
12459         }
12460         /*
12461          * Set port status flags in the page mapped into userspace
12462          * memory. Do it here to ensure a reliable state - this is
12463          * the only function called by all state handling code.
12464          * Always set the flags due to the fact that the cache value
12465          * might have been changed explicitly outside of this
12466          * function.
12467          */
12468         if (ppd->statusp) {
12469                 switch (ppd->lstate) {
12470                 case IB_PORT_DOWN:
12471                 case IB_PORT_INIT:
12472                         *ppd->statusp &= ~(HFI1_STATUS_IB_CONF |
12473                                            HFI1_STATUS_IB_READY);
12474                         break;
12475                 case IB_PORT_ARMED:
12476                         *ppd->statusp |= HFI1_STATUS_IB_CONF;
12477                         break;
12478                 case IB_PORT_ACTIVE:
12479                         *ppd->statusp |= HFI1_STATUS_IB_READY;
12480                         break;
12481                 }
12482         }
12483         return ppd->lstate;
12484 }
12485
12486 /**
12487  * wait_logical_linkstate - wait for an IB link state change to occur
12488  * @ppd: port device
12489  * @state: the state to wait for
12490  * @msecs: the number of milliseconds to wait
12491  *
12492  * Wait up to msecs milliseconds for IB link state change to occur.
12493  * For now, take the easy polling route.
12494  * Returns 0 if state reached, otherwise -ETIMEDOUT.
12495  */
12496 static int wait_logical_linkstate(struct hfi1_pportdata *ppd, u32 state,
12497                                   int msecs)
12498 {
12499         unsigned long timeout;
12500
12501         timeout = jiffies + msecs_to_jiffies(msecs);
12502         while (1) {
12503                 if (get_logical_state(ppd) == state)
12504                         return 0;
12505                 if (time_after(jiffies, timeout))
12506                         break;
12507                 msleep(20);
12508         }
12509         dd_dev_err(ppd->dd, "timeout waiting for link state 0x%x\n", state);
12510
12511         return -ETIMEDOUT;
12512 }
12513
12514 u8 hfi1_ibphys_portstate(struct hfi1_pportdata *ppd)
12515 {
12516         u32 pstate;
12517         u32 ib_pstate;
12518
12519         pstate = read_physical_state(ppd->dd);
12520         ib_pstate = chip_to_opa_pstate(ppd->dd, pstate);
12521         if (ppd->last_pstate != ib_pstate) {
12522                 dd_dev_info(ppd->dd,
12523                             "%s: physical state changed to %s (0x%x), phy 0x%x\n",
12524                             __func__, opa_pstate_name(ib_pstate), ib_pstate,
12525                             pstate);
12526                 ppd->last_pstate = ib_pstate;
12527         }
12528         return ib_pstate;
12529 }
12530
12531 #define CLEAR_STATIC_RATE_CONTROL_SMASK(r) \
12532 (r &= ~SEND_CTXT_CHECK_ENABLE_DISALLOW_PBC_STATIC_RATE_CONTROL_SMASK)
12533
12534 #define SET_STATIC_RATE_CONTROL_SMASK(r) \
12535 (r |= SEND_CTXT_CHECK_ENABLE_DISALLOW_PBC_STATIC_RATE_CONTROL_SMASK)
12536
12537 int hfi1_init_ctxt(struct send_context *sc)
12538 {
12539         if (sc) {
12540                 struct hfi1_devdata *dd = sc->dd;
12541                 u64 reg;
12542                 u8 set = (sc->type == SC_USER ?
12543                           HFI1_CAP_IS_USET(STATIC_RATE_CTRL) :
12544                           HFI1_CAP_IS_KSET(STATIC_RATE_CTRL));
12545                 reg = read_kctxt_csr(dd, sc->hw_context,
12546                                      SEND_CTXT_CHECK_ENABLE);
12547                 if (set)
12548                         CLEAR_STATIC_RATE_CONTROL_SMASK(reg);
12549                 else
12550                         SET_STATIC_RATE_CONTROL_SMASK(reg);
12551                 write_kctxt_csr(dd, sc->hw_context,
12552                                 SEND_CTXT_CHECK_ENABLE, reg);
12553         }
12554         return 0;
12555 }
12556
12557 int hfi1_tempsense_rd(struct hfi1_devdata *dd, struct hfi1_temp *temp)
12558 {
12559         int ret = 0;
12560         u64 reg;
12561
12562         if (dd->icode != ICODE_RTL_SILICON) {
12563                 if (HFI1_CAP_IS_KSET(PRINT_UNIMPL))
12564                         dd_dev_info(dd, "%s: tempsense not supported by HW\n",
12565                                     __func__);
12566                 return -EINVAL;
12567         }
12568         reg = read_csr(dd, ASIC_STS_THERM);
12569         temp->curr = ((reg >> ASIC_STS_THERM_CURR_TEMP_SHIFT) &
12570                       ASIC_STS_THERM_CURR_TEMP_MASK);
12571         temp->lo_lim = ((reg >> ASIC_STS_THERM_LO_TEMP_SHIFT) &
12572                         ASIC_STS_THERM_LO_TEMP_MASK);
12573         temp->hi_lim = ((reg >> ASIC_STS_THERM_HI_TEMP_SHIFT) &
12574                         ASIC_STS_THERM_HI_TEMP_MASK);
12575         temp->crit_lim = ((reg >> ASIC_STS_THERM_CRIT_TEMP_SHIFT) &
12576                           ASIC_STS_THERM_CRIT_TEMP_MASK);
12577         /* triggers is a 3-bit value - 1 bit per trigger. */
12578         temp->triggers = (u8)((reg >> ASIC_STS_THERM_LOW_SHIFT) & 0x7);
12579
12580         return ret;
12581 }
12582
12583 /* ========================================================================= */
12584
12585 /*
12586  * Enable/disable chip from delivering interrupts.
12587  */
12588 void set_intr_state(struct hfi1_devdata *dd, u32 enable)
12589 {
12590         int i;
12591
12592         /*
12593          * In HFI, the mask needs to be 1 to allow interrupts.
12594          */
12595         if (enable) {
12596                 /* enable all interrupts */
12597                 for (i = 0; i < CCE_NUM_INT_CSRS; i++)
12598                         write_csr(dd, CCE_INT_MASK + (8 * i), ~(u64)0);
12599
12600                 init_qsfp_int(dd);
12601         } else {
12602                 for (i = 0; i < CCE_NUM_INT_CSRS; i++)
12603                         write_csr(dd, CCE_INT_MASK + (8 * i), 0ull);
12604         }
12605 }
12606
12607 /*
12608  * Clear all interrupt sources on the chip.
12609  */
12610 static void clear_all_interrupts(struct hfi1_devdata *dd)
12611 {
12612         int i;
12613
12614         for (i = 0; i < CCE_NUM_INT_CSRS; i++)
12615                 write_csr(dd, CCE_INT_CLEAR + (8 * i), ~(u64)0);
12616
12617         write_csr(dd, CCE_ERR_CLEAR, ~(u64)0);
12618         write_csr(dd, MISC_ERR_CLEAR, ~(u64)0);
12619         write_csr(dd, RCV_ERR_CLEAR, ~(u64)0);
12620         write_csr(dd, SEND_ERR_CLEAR, ~(u64)0);
12621         write_csr(dd, SEND_PIO_ERR_CLEAR, ~(u64)0);
12622         write_csr(dd, SEND_DMA_ERR_CLEAR, ~(u64)0);
12623         write_csr(dd, SEND_EGRESS_ERR_CLEAR, ~(u64)0);
12624         for (i = 0; i < dd->chip_send_contexts; i++)
12625                 write_kctxt_csr(dd, i, SEND_CTXT_ERR_CLEAR, ~(u64)0);
12626         for (i = 0; i < dd->chip_sdma_engines; i++)
12627                 write_kctxt_csr(dd, i, SEND_DMA_ENG_ERR_CLEAR, ~(u64)0);
12628
12629         write_csr(dd, DCC_ERR_FLG_CLR, ~(u64)0);
12630         write_csr(dd, DC_LCB_ERR_CLR, ~(u64)0);
12631         write_csr(dd, DC_DC8051_ERR_CLR, ~(u64)0);
12632 }
12633
12634 /* Move to pcie.c? */
12635 static void disable_intx(struct pci_dev *pdev)
12636 {
12637         pci_intx(pdev, 0);
12638 }
12639
12640 static void clean_up_interrupts(struct hfi1_devdata *dd)
12641 {
12642         int i;
12643
12644         /* remove irqs - must happen before disabling/turning off */
12645         if (dd->num_msix_entries) {
12646                 /* MSI-X */
12647                 struct hfi1_msix_entry *me = dd->msix_entries;
12648
12649                 for (i = 0; i < dd->num_msix_entries; i++, me++) {
12650                         if (!me->arg) /* => no irq, no affinity */
12651                                 continue;
12652                         hfi1_put_irq_affinity(dd, &dd->msix_entries[i]);
12653                         free_irq(me->msix.vector, me->arg);
12654                 }
12655         } else {
12656                 /* INTx */
12657                 if (dd->requested_intx_irq) {
12658                         free_irq(dd->pcidev->irq, dd);
12659                         dd->requested_intx_irq = 0;
12660                 }
12661         }
12662
12663         /* turn off interrupts */
12664         if (dd->num_msix_entries) {
12665                 /* MSI-X */
12666                 pci_disable_msix(dd->pcidev);
12667         } else {
12668                 /* INTx */
12669                 disable_intx(dd->pcidev);
12670         }
12671
12672         /* clean structures */
12673         kfree(dd->msix_entries);
12674         dd->msix_entries = NULL;
12675         dd->num_msix_entries = 0;
12676 }
12677
12678 /*
12679  * Remap the interrupt source from the general handler to the given MSI-X
12680  * interrupt.
12681  */
12682 static void remap_intr(struct hfi1_devdata *dd, int isrc, int msix_intr)
12683 {
12684         u64 reg;
12685         int m, n;
12686
12687         /* clear from the handled mask of the general interrupt */
12688         m = isrc / 64;
12689         n = isrc % 64;
12690         dd->gi_mask[m] &= ~((u64)1 << n);
12691
12692         /* direct the chip source to the given MSI-X interrupt */
12693         m = isrc / 8;
12694         n = isrc % 8;
12695         reg = read_csr(dd, CCE_INT_MAP + (8 * m));
12696         reg &= ~((u64)0xff << (8 * n));
12697         reg |= ((u64)msix_intr & 0xff) << (8 * n);
12698         write_csr(dd, CCE_INT_MAP + (8 * m), reg);
12699 }
12700
12701 static void remap_sdma_interrupts(struct hfi1_devdata *dd,
12702                                   int engine, int msix_intr)
12703 {
12704         /*
12705          * SDMA engine interrupt sources grouped by type, rather than
12706          * engine.  Per-engine interrupts are as follows:
12707          *      SDMA
12708          *      SDMAProgress
12709          *      SDMAIdle
12710          */
12711         remap_intr(dd, IS_SDMA_START + 0 * TXE_NUM_SDMA_ENGINES + engine,
12712                    msix_intr);
12713         remap_intr(dd, IS_SDMA_START + 1 * TXE_NUM_SDMA_ENGINES + engine,
12714                    msix_intr);
12715         remap_intr(dd, IS_SDMA_START + 2 * TXE_NUM_SDMA_ENGINES + engine,
12716                    msix_intr);
12717 }
12718
12719 static int request_intx_irq(struct hfi1_devdata *dd)
12720 {
12721         int ret;
12722
12723         snprintf(dd->intx_name, sizeof(dd->intx_name), DRIVER_NAME "_%d",
12724                  dd->unit);
12725         ret = request_irq(dd->pcidev->irq, general_interrupt,
12726                           IRQF_SHARED, dd->intx_name, dd);
12727         if (ret)
12728                 dd_dev_err(dd, "unable to request INTx interrupt, err %d\n",
12729                            ret);
12730         else
12731                 dd->requested_intx_irq = 1;
12732         return ret;
12733 }
12734
12735 static int request_msix_irqs(struct hfi1_devdata *dd)
12736 {
12737         int first_general, last_general;
12738         int first_sdma, last_sdma;
12739         int first_rx, last_rx;
12740         int i, ret = 0;
12741
12742         /* calculate the ranges we are going to use */
12743         first_general = 0;
12744         last_general = first_general + 1;
12745         first_sdma = last_general;
12746         last_sdma = first_sdma + dd->num_sdma;
12747         first_rx = last_sdma;
12748         last_rx = first_rx + dd->n_krcv_queues;
12749
12750         /*
12751          * Sanity check - the code expects all SDMA chip source
12752          * interrupts to be in the same CSR, starting at bit 0.  Verify
12753          * that this is true by checking the bit location of the start.
12754          */
12755         BUILD_BUG_ON(IS_SDMA_START % 64);
12756
12757         for (i = 0; i < dd->num_msix_entries; i++) {
12758                 struct hfi1_msix_entry *me = &dd->msix_entries[i];
12759                 const char *err_info;
12760                 irq_handler_t handler;
12761                 irq_handler_t thread = NULL;
12762                 void *arg;
12763                 int idx;
12764                 struct hfi1_ctxtdata *rcd = NULL;
12765                 struct sdma_engine *sde = NULL;
12766
12767                 /* obtain the arguments to request_irq */
12768                 if (first_general <= i && i < last_general) {
12769                         idx = i - first_general;
12770                         handler = general_interrupt;
12771                         arg = dd;
12772                         snprintf(me->name, sizeof(me->name),
12773                                  DRIVER_NAME "_%d", dd->unit);
12774                         err_info = "general";
12775                         me->type = IRQ_GENERAL;
12776                 } else if (first_sdma <= i && i < last_sdma) {
12777                         idx = i - first_sdma;
12778                         sde = &dd->per_sdma[idx];
12779                         handler = sdma_interrupt;
12780                         arg = sde;
12781                         snprintf(me->name, sizeof(me->name),
12782                                  DRIVER_NAME "_%d sdma%d", dd->unit, idx);
12783                         err_info = "sdma";
12784                         remap_sdma_interrupts(dd, idx, i);
12785                         me->type = IRQ_SDMA;
12786                 } else if (first_rx <= i && i < last_rx) {
12787                         idx = i - first_rx;
12788                         rcd = dd->rcd[idx];
12789                         /* no interrupt if no rcd */
12790                         if (!rcd)
12791                                 continue;
12792                         /*
12793                          * Set the interrupt register and mask for this
12794                          * context's interrupt.
12795                          */
12796                         rcd->ireg = (IS_RCVAVAIL_START + idx) / 64;
12797                         rcd->imask = ((u64)1) <<
12798                                         ((IS_RCVAVAIL_START + idx) % 64);
12799                         handler = receive_context_interrupt;
12800                         thread = receive_context_thread;
12801                         arg = rcd;
12802                         snprintf(me->name, sizeof(me->name),
12803                                  DRIVER_NAME "_%d kctxt%d", dd->unit, idx);
12804                         err_info = "receive context";
12805                         remap_intr(dd, IS_RCVAVAIL_START + idx, i);
12806                         me->type = IRQ_RCVCTXT;
12807                 } else {
12808                         /* not in our expected range - complain, then
12809                          * ignore it
12810                          */
12811                         dd_dev_err(dd,
12812                                    "Unexpected extra MSI-X interrupt %d\n", i);
12813                         continue;
12814                 }
12815                 /* no argument, no interrupt */
12816                 if (!arg)
12817                         continue;
12818                 /* make sure the name is terminated */
12819                 me->name[sizeof(me->name) - 1] = 0;
12820
12821                 ret = request_threaded_irq(me->msix.vector, handler, thread, 0,
12822                                            me->name, arg);
12823                 if (ret) {
12824                         dd_dev_err(dd,
12825                                    "unable to allocate %s interrupt, vector %d, index %d, err %d\n",
12826                                    err_info, me->msix.vector, idx, ret);
12827                         return ret;
12828                 }
12829                 /*
12830                  * assign arg after request_irq call, so it will be
12831                  * cleaned up
12832                  */
12833                 me->arg = arg;
12834
12835                 ret = hfi1_get_irq_affinity(dd, me);
12836                 if (ret)
12837                         dd_dev_err(dd,
12838                                    "unable to pin IRQ %d\n", ret);
12839         }
12840
12841         return ret;
12842 }
12843
12844 /*
12845  * Set the general handler to accept all interrupts, remap all
12846  * chip interrupts back to MSI-X 0.
12847  */
12848 static void reset_interrupts(struct hfi1_devdata *dd)
12849 {
12850         int i;
12851
12852         /* all interrupts handled by the general handler */
12853         for (i = 0; i < CCE_NUM_INT_CSRS; i++)
12854                 dd->gi_mask[i] = ~(u64)0;
12855
12856         /* all chip interrupts map to MSI-X 0 */
12857         for (i = 0; i < CCE_NUM_INT_MAP_CSRS; i++)
12858                 write_csr(dd, CCE_INT_MAP + (8 * i), 0);
12859 }
12860
12861 static int set_up_interrupts(struct hfi1_devdata *dd)
12862 {
12863         struct hfi1_msix_entry *entries;
12864         u32 total, request;
12865         int i, ret;
12866         int single_interrupt = 0; /* we expect to have all the interrupts */
12867
12868         /*
12869          * Interrupt count:
12870          *      1 general, "slow path" interrupt (includes the SDMA engines
12871          *              slow source, SDMACleanupDone)
12872          *      N interrupts - one per used SDMA engine
12873          *      M interrupt - one per kernel receive context
12874          */
12875         total = 1 + dd->num_sdma + dd->n_krcv_queues;
12876
12877         entries = kcalloc(total, sizeof(*entries), GFP_KERNEL);
12878         if (!entries) {
12879                 ret = -ENOMEM;
12880                 goto fail;
12881         }
12882         /* 1-1 MSI-X entry assignment */
12883         for (i = 0; i < total; i++)
12884                 entries[i].msix.entry = i;
12885
12886         /* ask for MSI-X interrupts */
12887         request = total;
12888         request_msix(dd, &request, entries);
12889
12890         if (request == 0) {
12891                 /* using INTx */
12892                 /* dd->num_msix_entries already zero */
12893                 kfree(entries);
12894                 single_interrupt = 1;
12895                 dd_dev_err(dd, "MSI-X failed, using INTx interrupts\n");
12896         } else {
12897                 /* using MSI-X */
12898                 dd->num_msix_entries = request;
12899                 dd->msix_entries = entries;
12900
12901                 if (request != total) {
12902                         /* using MSI-X, with reduced interrupts */
12903                         dd_dev_err(
12904                                 dd,
12905                                 "cannot handle reduced interrupt case, want %u, got %u\n",
12906                                 total, request);
12907                         ret = -EINVAL;
12908                         goto fail;
12909                 }
12910                 dd_dev_info(dd, "%u MSI-X interrupts allocated\n", total);
12911         }
12912
12913         /* mask all interrupts */
12914         set_intr_state(dd, 0);
12915         /* clear all pending interrupts */
12916         clear_all_interrupts(dd);
12917
12918         /* reset general handler mask, chip MSI-X mappings */
12919         reset_interrupts(dd);
12920
12921         if (single_interrupt)
12922                 ret = request_intx_irq(dd);
12923         else
12924                 ret = request_msix_irqs(dd);
12925         if (ret)
12926                 goto fail;
12927
12928         return 0;
12929
12930 fail:
12931         clean_up_interrupts(dd);
12932         return ret;
12933 }
12934
12935 /*
12936  * Set up context values in dd.  Sets:
12937  *
12938  *      num_rcv_contexts - number of contexts being used
12939  *      n_krcv_queues - number of kernel contexts
12940  *      first_user_ctxt - first non-kernel context in array of contexts
12941  *      freectxts  - number of free user contexts
12942  *      num_send_contexts - number of PIO send contexts being used
12943  */
12944 static int set_up_context_variables(struct hfi1_devdata *dd)
12945 {
12946         unsigned long num_kernel_contexts;
12947         int total_contexts;
12948         int ret;
12949         unsigned ngroups;
12950         int qos_rmt_count;
12951         int user_rmt_reduced;
12952
12953         /*
12954          * Kernel receive contexts:
12955          * - Context 0 - control context (VL15/multicast/error)
12956          * - Context 1 - first kernel context
12957          * - Context 2 - second kernel context
12958          * ...
12959          */
12960         if (n_krcvqs)
12961                 /*
12962                  * n_krcvqs is the sum of module parameter kernel receive
12963                  * contexts, krcvqs[].  It does not include the control
12964                  * context, so add that.
12965                  */
12966                 num_kernel_contexts = n_krcvqs + 1;
12967         else
12968                 num_kernel_contexts = DEFAULT_KRCVQS + 1;
12969         /*
12970          * Every kernel receive context needs an ACK send context.
12971          * one send context is allocated for each VL{0-7} and VL15
12972          */
12973         if (num_kernel_contexts > (dd->chip_send_contexts - num_vls - 1)) {
12974                 dd_dev_err(dd,
12975                            "Reducing # kernel rcv contexts to: %d, from %lu\n",
12976                            (int)(dd->chip_send_contexts - num_vls - 1),
12977                            num_kernel_contexts);
12978                 num_kernel_contexts = dd->chip_send_contexts - num_vls - 1;
12979         }
12980         /*
12981          * User contexts:
12982          *      - default to 1 user context per real (non-HT) CPU core if
12983          *        num_user_contexts is negative
12984          */
12985         if (num_user_contexts < 0)
12986                 num_user_contexts =
12987                         cpumask_weight(&node_affinity.real_cpu_mask);
12988
12989         total_contexts = num_kernel_contexts + num_user_contexts;
12990
12991         /*
12992          * Adjust the counts given a global max.
12993          */
12994         if (total_contexts > dd->chip_rcv_contexts) {
12995                 dd_dev_err(dd,
12996                            "Reducing # user receive contexts to: %d, from %d\n",
12997                            (int)(dd->chip_rcv_contexts - num_kernel_contexts),
12998                            (int)num_user_contexts);
12999                 num_user_contexts = dd->chip_rcv_contexts - num_kernel_contexts;
13000                 /* recalculate */
13001                 total_contexts = num_kernel_contexts + num_user_contexts;
13002         }
13003
13004         /* each user context requires an entry in the RMT */
13005         qos_rmt_count = qos_rmt_entries(dd, NULL, NULL);
13006         if (qos_rmt_count + num_user_contexts > NUM_MAP_ENTRIES) {
13007                 user_rmt_reduced = NUM_MAP_ENTRIES - qos_rmt_count;
13008                 dd_dev_err(dd,
13009                            "RMT size is reducing the number of user receive contexts from %d to %d\n",
13010                            (int)num_user_contexts,
13011                            user_rmt_reduced);
13012                 /* recalculate */
13013                 num_user_contexts = user_rmt_reduced;
13014                 total_contexts = num_kernel_contexts + num_user_contexts;
13015         }
13016
13017         /* the first N are kernel contexts, the rest are user contexts */
13018         dd->num_rcv_contexts = total_contexts;
13019         dd->n_krcv_queues = num_kernel_contexts;
13020         dd->first_user_ctxt = num_kernel_contexts;
13021         dd->num_user_contexts = num_user_contexts;
13022         dd->freectxts = num_user_contexts;
13023         dd_dev_info(dd,
13024                     "rcv contexts: chip %d, used %d (kernel %d, user %d)\n",
13025                     (int)dd->chip_rcv_contexts,
13026                     (int)dd->num_rcv_contexts,
13027                     (int)dd->n_krcv_queues,
13028                     (int)dd->num_rcv_contexts - dd->n_krcv_queues);
13029
13030         /*
13031          * Receive array allocation:
13032          *   All RcvArray entries are divided into groups of 8. This
13033          *   is required by the hardware and will speed up writes to
13034          *   consecutive entries by using write-combining of the entire
13035          *   cacheline.
13036          *
13037          *   The number of groups are evenly divided among all contexts.
13038          *   any left over groups will be given to the first N user
13039          *   contexts.
13040          */
13041         dd->rcv_entries.group_size = RCV_INCREMENT;
13042         ngroups = dd->chip_rcv_array_count / dd->rcv_entries.group_size;
13043         dd->rcv_entries.ngroups = ngroups / dd->num_rcv_contexts;
13044         dd->rcv_entries.nctxt_extra = ngroups -
13045                 (dd->num_rcv_contexts * dd->rcv_entries.ngroups);
13046         dd_dev_info(dd, "RcvArray groups %u, ctxts extra %u\n",
13047                     dd->rcv_entries.ngroups,
13048                     dd->rcv_entries.nctxt_extra);
13049         if (dd->rcv_entries.ngroups * dd->rcv_entries.group_size >
13050             MAX_EAGER_ENTRIES * 2) {
13051                 dd->rcv_entries.ngroups = (MAX_EAGER_ENTRIES * 2) /
13052                         dd->rcv_entries.group_size;
13053                 dd_dev_info(dd,
13054                             "RcvArray group count too high, change to %u\n",
13055                             dd->rcv_entries.ngroups);
13056                 dd->rcv_entries.nctxt_extra = 0;
13057         }
13058         /*
13059          * PIO send contexts
13060          */
13061         ret = init_sc_pools_and_sizes(dd);
13062         if (ret >= 0) { /* success */
13063                 dd->num_send_contexts = ret;
13064                 dd_dev_info(
13065                         dd,
13066                         "send contexts: chip %d, used %d (kernel %d, ack %d, user %d, vl15 %d)\n",
13067                         dd->chip_send_contexts,
13068                         dd->num_send_contexts,
13069                         dd->sc_sizes[SC_KERNEL].count,
13070                         dd->sc_sizes[SC_ACK].count,
13071                         dd->sc_sizes[SC_USER].count,
13072                         dd->sc_sizes[SC_VL15].count);
13073                 ret = 0;        /* success */
13074         }
13075
13076         return ret;
13077 }
13078
13079 /*
13080  * Set the device/port partition key table. The MAD code
13081  * will ensure that, at least, the partial management
13082  * partition key is present in the table.
13083  */
13084 static void set_partition_keys(struct hfi1_pportdata *ppd)
13085 {
13086         struct hfi1_devdata *dd = ppd->dd;
13087         u64 reg = 0;
13088         int i;
13089
13090         dd_dev_info(dd, "Setting partition keys\n");
13091         for (i = 0; i < hfi1_get_npkeys(dd); i++) {
13092                 reg |= (ppd->pkeys[i] &
13093                         RCV_PARTITION_KEY_PARTITION_KEY_A_MASK) <<
13094                         ((i % 4) *
13095                          RCV_PARTITION_KEY_PARTITION_KEY_B_SHIFT);
13096                 /* Each register holds 4 PKey values. */
13097                 if ((i % 4) == 3) {
13098                         write_csr(dd, RCV_PARTITION_KEY +
13099                                   ((i - 3) * 2), reg);
13100                         reg = 0;
13101                 }
13102         }
13103
13104         /* Always enable HW pkeys check when pkeys table is set */
13105         add_rcvctrl(dd, RCV_CTRL_RCV_PARTITION_KEY_ENABLE_SMASK);
13106 }
13107
13108 /*
13109  * These CSRs and memories are uninitialized on reset and must be
13110  * written before reading to set the ECC/parity bits.
13111  *
13112  * NOTE: All user context CSRs that are not mmaped write-only
13113  * (e.g. the TID flows) must be initialized even if the driver never
13114  * reads them.
13115  */
13116 static void write_uninitialized_csrs_and_memories(struct hfi1_devdata *dd)
13117 {
13118         int i, j;
13119
13120         /* CceIntMap */
13121         for (i = 0; i < CCE_NUM_INT_MAP_CSRS; i++)
13122                 write_csr(dd, CCE_INT_MAP + (8 * i), 0);
13123
13124         /* SendCtxtCreditReturnAddr */
13125         for (i = 0; i < dd->chip_send_contexts; i++)
13126                 write_kctxt_csr(dd, i, SEND_CTXT_CREDIT_RETURN_ADDR, 0);
13127
13128         /* PIO Send buffers */
13129         /* SDMA Send buffers */
13130         /*
13131          * These are not normally read, and (presently) have no method
13132          * to be read, so are not pre-initialized
13133          */
13134
13135         /* RcvHdrAddr */
13136         /* RcvHdrTailAddr */
13137         /* RcvTidFlowTable */
13138         for (i = 0; i < dd->chip_rcv_contexts; i++) {
13139                 write_kctxt_csr(dd, i, RCV_HDR_ADDR, 0);
13140                 write_kctxt_csr(dd, i, RCV_HDR_TAIL_ADDR, 0);
13141                 for (j = 0; j < RXE_NUM_TID_FLOWS; j++)
13142                         write_uctxt_csr(dd, i, RCV_TID_FLOW_TABLE + (8 * j), 0);
13143         }
13144
13145         /* RcvArray */
13146         for (i = 0; i < dd->chip_rcv_array_count; i++)
13147                 write_csr(dd, RCV_ARRAY + (8 * i),
13148                           RCV_ARRAY_RT_WRITE_ENABLE_SMASK);
13149
13150         /* RcvQPMapTable */
13151         for (i = 0; i < 32; i++)
13152                 write_csr(dd, RCV_QP_MAP_TABLE + (8 * i), 0);
13153 }
13154
13155 /*
13156  * Use the ctrl_bits in CceCtrl to clear the status_bits in CceStatus.
13157  */
13158 static void clear_cce_status(struct hfi1_devdata *dd, u64 status_bits,
13159                              u64 ctrl_bits)
13160 {
13161         unsigned long timeout;
13162         u64 reg;
13163
13164         /* is the condition present? */
13165         reg = read_csr(dd, CCE_STATUS);
13166         if ((reg & status_bits) == 0)
13167                 return;
13168
13169         /* clear the condition */
13170         write_csr(dd, CCE_CTRL, ctrl_bits);
13171
13172         /* wait for the condition to clear */
13173         timeout = jiffies + msecs_to_jiffies(CCE_STATUS_TIMEOUT);
13174         while (1) {
13175                 reg = read_csr(dd, CCE_STATUS);
13176                 if ((reg & status_bits) == 0)
13177                         return;
13178                 if (time_after(jiffies, timeout)) {
13179                         dd_dev_err(dd,
13180                                    "Timeout waiting for CceStatus to clear bits 0x%llx, remaining 0x%llx\n",
13181                                    status_bits, reg & status_bits);
13182                         return;
13183                 }
13184                 udelay(1);
13185         }
13186 }
13187
13188 /* set CCE CSRs to chip reset defaults */
13189 static void reset_cce_csrs(struct hfi1_devdata *dd)
13190 {
13191         int i;
13192
13193         /* CCE_REVISION read-only */
13194         /* CCE_REVISION2 read-only */
13195         /* CCE_CTRL - bits clear automatically */
13196         /* CCE_STATUS read-only, use CceCtrl to clear */
13197         clear_cce_status(dd, ALL_FROZE, CCE_CTRL_SPC_UNFREEZE_SMASK);
13198         clear_cce_status(dd, ALL_TXE_PAUSE, CCE_CTRL_TXE_RESUME_SMASK);
13199         clear_cce_status(dd, ALL_RXE_PAUSE, CCE_CTRL_RXE_RESUME_SMASK);
13200         for (i = 0; i < CCE_NUM_SCRATCH; i++)
13201                 write_csr(dd, CCE_SCRATCH + (8 * i), 0);
13202         /* CCE_ERR_STATUS read-only */
13203         write_csr(dd, CCE_ERR_MASK, 0);
13204         write_csr(dd, CCE_ERR_CLEAR, ~0ull);
13205         /* CCE_ERR_FORCE leave alone */
13206         for (i = 0; i < CCE_NUM_32_BIT_COUNTERS; i++)
13207                 write_csr(dd, CCE_COUNTER_ARRAY32 + (8 * i), 0);
13208         write_csr(dd, CCE_DC_CTRL, CCE_DC_CTRL_RESETCSR);
13209         /* CCE_PCIE_CTRL leave alone */
13210         for (i = 0; i < CCE_NUM_MSIX_VECTORS; i++) {
13211                 write_csr(dd, CCE_MSIX_TABLE_LOWER + (8 * i), 0);
13212                 write_csr(dd, CCE_MSIX_TABLE_UPPER + (8 * i),
13213                           CCE_MSIX_TABLE_UPPER_RESETCSR);
13214         }
13215         for (i = 0; i < CCE_NUM_MSIX_PBAS; i++) {
13216                 /* CCE_MSIX_PBA read-only */
13217                 write_csr(dd, CCE_MSIX_INT_GRANTED, ~0ull);
13218                 write_csr(dd, CCE_MSIX_VEC_CLR_WITHOUT_INT, ~0ull);
13219         }
13220         for (i = 0; i < CCE_NUM_INT_MAP_CSRS; i++)
13221                 write_csr(dd, CCE_INT_MAP, 0);
13222         for (i = 0; i < CCE_NUM_INT_CSRS; i++) {
13223                 /* CCE_INT_STATUS read-only */
13224                 write_csr(dd, CCE_INT_MASK + (8 * i), 0);
13225                 write_csr(dd, CCE_INT_CLEAR + (8 * i), ~0ull);
13226                 /* CCE_INT_FORCE leave alone */
13227                 /* CCE_INT_BLOCKED read-only */
13228         }
13229         for (i = 0; i < CCE_NUM_32_BIT_INT_COUNTERS; i++)
13230                 write_csr(dd, CCE_INT_COUNTER_ARRAY32 + (8 * i), 0);
13231 }
13232
13233 /* set MISC CSRs to chip reset defaults */
13234 static void reset_misc_csrs(struct hfi1_devdata *dd)
13235 {
13236         int i;
13237
13238         for (i = 0; i < 32; i++) {
13239                 write_csr(dd, MISC_CFG_RSA_R2 + (8 * i), 0);
13240                 write_csr(dd, MISC_CFG_RSA_SIGNATURE + (8 * i), 0);
13241                 write_csr(dd, MISC_CFG_RSA_MODULUS + (8 * i), 0);
13242         }
13243         /*
13244          * MISC_CFG_SHA_PRELOAD leave alone - always reads 0 and can
13245          * only be written 128-byte chunks
13246          */
13247         /* init RSA engine to clear lingering errors */
13248         write_csr(dd, MISC_CFG_RSA_CMD, 1);
13249         write_csr(dd, MISC_CFG_RSA_MU, 0);
13250         write_csr(dd, MISC_CFG_FW_CTRL, 0);
13251         /* MISC_STS_8051_DIGEST read-only */
13252         /* MISC_STS_SBM_DIGEST read-only */
13253         /* MISC_STS_PCIE_DIGEST read-only */
13254         /* MISC_STS_FAB_DIGEST read-only */
13255         /* MISC_ERR_STATUS read-only */
13256         write_csr(dd, MISC_ERR_MASK, 0);
13257         write_csr(dd, MISC_ERR_CLEAR, ~0ull);
13258         /* MISC_ERR_FORCE leave alone */
13259 }
13260
13261 /* set TXE CSRs to chip reset defaults */
13262 static void reset_txe_csrs(struct hfi1_devdata *dd)
13263 {
13264         int i;
13265
13266         /*
13267          * TXE Kernel CSRs
13268          */
13269         write_csr(dd, SEND_CTRL, 0);
13270         __cm_reset(dd, 0);      /* reset CM internal state */
13271         /* SEND_CONTEXTS read-only */
13272         /* SEND_DMA_ENGINES read-only */
13273         /* SEND_PIO_MEM_SIZE read-only */
13274         /* SEND_DMA_MEM_SIZE read-only */
13275         write_csr(dd, SEND_HIGH_PRIORITY_LIMIT, 0);
13276         pio_reset_all(dd);      /* SEND_PIO_INIT_CTXT */
13277         /* SEND_PIO_ERR_STATUS read-only */
13278         write_csr(dd, SEND_PIO_ERR_MASK, 0);
13279         write_csr(dd, SEND_PIO_ERR_CLEAR, ~0ull);
13280         /* SEND_PIO_ERR_FORCE leave alone */
13281         /* SEND_DMA_ERR_STATUS read-only */
13282         write_csr(dd, SEND_DMA_ERR_MASK, 0);
13283         write_csr(dd, SEND_DMA_ERR_CLEAR, ~0ull);
13284         /* SEND_DMA_ERR_FORCE leave alone */
13285         /* SEND_EGRESS_ERR_STATUS read-only */
13286         write_csr(dd, SEND_EGRESS_ERR_MASK, 0);
13287         write_csr(dd, SEND_EGRESS_ERR_CLEAR, ~0ull);
13288         /* SEND_EGRESS_ERR_FORCE leave alone */
13289         write_csr(dd, SEND_BTH_QP, 0);
13290         write_csr(dd, SEND_STATIC_RATE_CONTROL, 0);
13291         write_csr(dd, SEND_SC2VLT0, 0);
13292         write_csr(dd, SEND_SC2VLT1, 0);
13293         write_csr(dd, SEND_SC2VLT2, 0);
13294         write_csr(dd, SEND_SC2VLT3, 0);
13295         write_csr(dd, SEND_LEN_CHECK0, 0);
13296         write_csr(dd, SEND_LEN_CHECK1, 0);
13297         /* SEND_ERR_STATUS read-only */
13298         write_csr(dd, SEND_ERR_MASK, 0);
13299         write_csr(dd, SEND_ERR_CLEAR, ~0ull);
13300         /* SEND_ERR_FORCE read-only */
13301         for (i = 0; i < VL_ARB_LOW_PRIO_TABLE_SIZE; i++)
13302                 write_csr(dd, SEND_LOW_PRIORITY_LIST + (8 * i), 0);
13303         for (i = 0; i < VL_ARB_HIGH_PRIO_TABLE_SIZE; i++)
13304                 write_csr(dd, SEND_HIGH_PRIORITY_LIST + (8 * i), 0);
13305         for (i = 0; i < dd->chip_send_contexts / NUM_CONTEXTS_PER_SET; i++)
13306                 write_csr(dd, SEND_CONTEXT_SET_CTRL + (8 * i), 0);
13307         for (i = 0; i < TXE_NUM_32_BIT_COUNTER; i++)
13308                 write_csr(dd, SEND_COUNTER_ARRAY32 + (8 * i), 0);
13309         for (i = 0; i < TXE_NUM_64_BIT_COUNTER; i++)
13310                 write_csr(dd, SEND_COUNTER_ARRAY64 + (8 * i), 0);
13311         write_csr(dd, SEND_CM_CTRL, SEND_CM_CTRL_RESETCSR);
13312         write_csr(dd, SEND_CM_GLOBAL_CREDIT, SEND_CM_GLOBAL_CREDIT_RESETCSR);
13313         /* SEND_CM_CREDIT_USED_STATUS read-only */
13314         write_csr(dd, SEND_CM_TIMER_CTRL, 0);
13315         write_csr(dd, SEND_CM_LOCAL_AU_TABLE0_TO3, 0);
13316         write_csr(dd, SEND_CM_LOCAL_AU_TABLE4_TO7, 0);
13317         write_csr(dd, SEND_CM_REMOTE_AU_TABLE0_TO3, 0);
13318         write_csr(dd, SEND_CM_REMOTE_AU_TABLE4_TO7, 0);
13319         for (i = 0; i < TXE_NUM_DATA_VL; i++)
13320                 write_csr(dd, SEND_CM_CREDIT_VL + (8 * i), 0);
13321         write_csr(dd, SEND_CM_CREDIT_VL15, 0);
13322         /* SEND_CM_CREDIT_USED_VL read-only */
13323         /* SEND_CM_CREDIT_USED_VL15 read-only */
13324         /* SEND_EGRESS_CTXT_STATUS read-only */
13325         /* SEND_EGRESS_SEND_DMA_STATUS read-only */
13326         write_csr(dd, SEND_EGRESS_ERR_INFO, ~0ull);
13327         /* SEND_EGRESS_ERR_INFO read-only */
13328         /* SEND_EGRESS_ERR_SOURCE read-only */
13329
13330         /*
13331          * TXE Per-Context CSRs
13332          */
13333         for (i = 0; i < dd->chip_send_contexts; i++) {
13334                 write_kctxt_csr(dd, i, SEND_CTXT_CTRL, 0);
13335                 write_kctxt_csr(dd, i, SEND_CTXT_CREDIT_CTRL, 0);
13336                 write_kctxt_csr(dd, i, SEND_CTXT_CREDIT_RETURN_ADDR, 0);
13337                 write_kctxt_csr(dd, i, SEND_CTXT_CREDIT_FORCE, 0);
13338                 write_kctxt_csr(dd, i, SEND_CTXT_ERR_MASK, 0);
13339                 write_kctxt_csr(dd, i, SEND_CTXT_ERR_CLEAR, ~0ull);
13340                 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_ENABLE, 0);
13341                 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_VL, 0);
13342                 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_JOB_KEY, 0);
13343                 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_PARTITION_KEY, 0);
13344                 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_SLID, 0);
13345                 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_OPCODE, 0);
13346         }
13347
13348         /*
13349          * TXE Per-SDMA CSRs
13350          */
13351         for (i = 0; i < dd->chip_sdma_engines; i++) {
13352                 write_kctxt_csr(dd, i, SEND_DMA_CTRL, 0);
13353                 /* SEND_DMA_STATUS read-only */
13354                 write_kctxt_csr(dd, i, SEND_DMA_BASE_ADDR, 0);
13355                 write_kctxt_csr(dd, i, SEND_DMA_LEN_GEN, 0);
13356                 write_kctxt_csr(dd, i, SEND_DMA_TAIL, 0);
13357                 /* SEND_DMA_HEAD read-only */
13358                 write_kctxt_csr(dd, i, SEND_DMA_HEAD_ADDR, 0);
13359                 write_kctxt_csr(dd, i, SEND_DMA_PRIORITY_THLD, 0);
13360                 /* SEND_DMA_IDLE_CNT read-only */
13361                 write_kctxt_csr(dd, i, SEND_DMA_RELOAD_CNT, 0);
13362                 write_kctxt_csr(dd, i, SEND_DMA_DESC_CNT, 0);
13363                 /* SEND_DMA_DESC_FETCHED_CNT read-only */
13364                 /* SEND_DMA_ENG_ERR_STATUS read-only */
13365                 write_kctxt_csr(dd, i, SEND_DMA_ENG_ERR_MASK, 0);
13366                 write_kctxt_csr(dd, i, SEND_DMA_ENG_ERR_CLEAR, ~0ull);
13367                 /* SEND_DMA_ENG_ERR_FORCE leave alone */
13368                 write_kctxt_csr(dd, i, SEND_DMA_CHECK_ENABLE, 0);
13369                 write_kctxt_csr(dd, i, SEND_DMA_CHECK_VL, 0);
13370                 write_kctxt_csr(dd, i, SEND_DMA_CHECK_JOB_KEY, 0);
13371                 write_kctxt_csr(dd, i, SEND_DMA_CHECK_PARTITION_KEY, 0);
13372                 write_kctxt_csr(dd, i, SEND_DMA_CHECK_SLID, 0);
13373                 write_kctxt_csr(dd, i, SEND_DMA_CHECK_OPCODE, 0);
13374                 write_kctxt_csr(dd, i, SEND_DMA_MEMORY, 0);
13375         }
13376 }
13377
13378 /*
13379  * Expect on entry:
13380  * o Packet ingress is disabled, i.e. RcvCtrl.RcvPortEnable == 0
13381  */
13382 static void init_rbufs(struct hfi1_devdata *dd)
13383 {
13384         u64 reg;
13385         int count;
13386
13387         /*
13388          * Wait for DMA to stop: RxRbufPktPending and RxPktInProgress are
13389          * clear.
13390          */
13391         count = 0;
13392         while (1) {
13393                 reg = read_csr(dd, RCV_STATUS);
13394                 if ((reg & (RCV_STATUS_RX_RBUF_PKT_PENDING_SMASK
13395                             | RCV_STATUS_RX_PKT_IN_PROGRESS_SMASK)) == 0)
13396                         break;
13397                 /*
13398                  * Give up after 1ms - maximum wait time.
13399                  *
13400                  * RBuf size is 136KiB.  Slowest possible is PCIe Gen1 x1 at
13401                  * 250MB/s bandwidth.  Lower rate to 66% for overhead to get:
13402                  *      136 KB / (66% * 250MB/s) = 844us
13403                  */
13404                 if (count++ > 500) {
13405                         dd_dev_err(dd,
13406                                    "%s: in-progress DMA not clearing: RcvStatus 0x%llx, continuing\n",
13407                                    __func__, reg);
13408                         break;
13409                 }
13410                 udelay(2); /* do not busy-wait the CSR */
13411         }
13412
13413         /* start the init - expect RcvCtrl to be 0 */
13414         write_csr(dd, RCV_CTRL, RCV_CTRL_RX_RBUF_INIT_SMASK);
13415
13416         /*
13417          * Read to force the write of Rcvtrl.RxRbufInit.  There is a brief
13418          * period after the write before RcvStatus.RxRbufInitDone is valid.
13419          * The delay in the first run through the loop below is sufficient and
13420          * required before the first read of RcvStatus.RxRbufInintDone.
13421          */
13422         read_csr(dd, RCV_CTRL);
13423
13424         /* wait for the init to finish */
13425         count = 0;
13426         while (1) {
13427                 /* delay is required first time through - see above */
13428                 udelay(2); /* do not busy-wait the CSR */
13429                 reg = read_csr(dd, RCV_STATUS);
13430                 if (reg & (RCV_STATUS_RX_RBUF_INIT_DONE_SMASK))
13431                         break;
13432
13433                 /* give up after 100us - slowest possible at 33MHz is 73us */
13434                 if (count++ > 50) {
13435                         dd_dev_err(dd,
13436                                    "%s: RcvStatus.RxRbufInit not set, continuing\n",
13437                                    __func__);
13438                         break;
13439                 }
13440         }
13441 }
13442
13443 /* set RXE CSRs to chip reset defaults */
13444 static void reset_rxe_csrs(struct hfi1_devdata *dd)
13445 {
13446         int i, j;
13447
13448         /*
13449          * RXE Kernel CSRs
13450          */
13451         write_csr(dd, RCV_CTRL, 0);
13452         init_rbufs(dd);
13453         /* RCV_STATUS read-only */
13454         /* RCV_CONTEXTS read-only */
13455         /* RCV_ARRAY_CNT read-only */
13456         /* RCV_BUF_SIZE read-only */
13457         write_csr(dd, RCV_BTH_QP, 0);
13458         write_csr(dd, RCV_MULTICAST, 0);
13459         write_csr(dd, RCV_BYPASS, 0);
13460         write_csr(dd, RCV_VL15, 0);
13461         /* this is a clear-down */
13462         write_csr(dd, RCV_ERR_INFO,
13463                   RCV_ERR_INFO_RCV_EXCESS_BUFFER_OVERRUN_SMASK);
13464         /* RCV_ERR_STATUS read-only */
13465         write_csr(dd, RCV_ERR_MASK, 0);
13466         write_csr(dd, RCV_ERR_CLEAR, ~0ull);
13467         /* RCV_ERR_FORCE leave alone */
13468         for (i = 0; i < 32; i++)
13469                 write_csr(dd, RCV_QP_MAP_TABLE + (8 * i), 0);
13470         for (i = 0; i < 4; i++)
13471                 write_csr(dd, RCV_PARTITION_KEY + (8 * i), 0);
13472         for (i = 0; i < RXE_NUM_32_BIT_COUNTERS; i++)
13473                 write_csr(dd, RCV_COUNTER_ARRAY32 + (8 * i), 0);
13474         for (i = 0; i < RXE_NUM_64_BIT_COUNTERS; i++)
13475                 write_csr(dd, RCV_COUNTER_ARRAY64 + (8 * i), 0);
13476         for (i = 0; i < RXE_NUM_RSM_INSTANCES; i++) {
13477                 write_csr(dd, RCV_RSM_CFG + (8 * i), 0);
13478                 write_csr(dd, RCV_RSM_SELECT + (8 * i), 0);
13479                 write_csr(dd, RCV_RSM_MATCH + (8 * i), 0);
13480         }
13481         for (i = 0; i < 32; i++)
13482                 write_csr(dd, RCV_RSM_MAP_TABLE + (8 * i), 0);
13483
13484         /*
13485          * RXE Kernel and User Per-Context CSRs
13486          */
13487         for (i = 0; i < dd->chip_rcv_contexts; i++) {
13488                 /* kernel */
13489                 write_kctxt_csr(dd, i, RCV_CTXT_CTRL, 0);
13490                 /* RCV_CTXT_STATUS read-only */
13491                 write_kctxt_csr(dd, i, RCV_EGR_CTRL, 0);
13492                 write_kctxt_csr(dd, i, RCV_TID_CTRL, 0);
13493                 write_kctxt_csr(dd, i, RCV_KEY_CTRL, 0);
13494                 write_kctxt_csr(dd, i, RCV_HDR_ADDR, 0);
13495                 write_kctxt_csr(dd, i, RCV_HDR_CNT, 0);
13496                 write_kctxt_csr(dd, i, RCV_HDR_ENT_SIZE, 0);
13497                 write_kctxt_csr(dd, i, RCV_HDR_SIZE, 0);
13498                 write_kctxt_csr(dd, i, RCV_HDR_TAIL_ADDR, 0);
13499                 write_kctxt_csr(dd, i, RCV_AVAIL_TIME_OUT, 0);
13500                 write_kctxt_csr(dd, i, RCV_HDR_OVFL_CNT, 0);
13501
13502                 /* user */
13503                 /* RCV_HDR_TAIL read-only */
13504                 write_uctxt_csr(dd, i, RCV_HDR_HEAD, 0);
13505                 /* RCV_EGR_INDEX_TAIL read-only */
13506                 write_uctxt_csr(dd, i, RCV_EGR_INDEX_HEAD, 0);
13507                 /* RCV_EGR_OFFSET_TAIL read-only */
13508                 for (j = 0; j < RXE_NUM_TID_FLOWS; j++) {
13509                         write_uctxt_csr(dd, i,
13510                                         RCV_TID_FLOW_TABLE + (8 * j), 0);
13511                 }
13512         }
13513 }
13514
13515 /*
13516  * Set sc2vl tables.
13517  *
13518  * They power on to zeros, so to avoid send context errors
13519  * they need to be set:
13520  *
13521  * SC 0-7 -> VL 0-7 (respectively)
13522  * SC 15  -> VL 15
13523  * otherwise
13524  *        -> VL 0
13525  */
13526 static void init_sc2vl_tables(struct hfi1_devdata *dd)
13527 {
13528         int i;
13529         /* init per architecture spec, constrained by hardware capability */
13530
13531         /* HFI maps sent packets */
13532         write_csr(dd, SEND_SC2VLT0, SC2VL_VAL(
13533                 0,
13534                 0, 0, 1, 1,
13535                 2, 2, 3, 3,
13536                 4, 4, 5, 5,
13537                 6, 6, 7, 7));
13538         write_csr(dd, SEND_SC2VLT1, SC2VL_VAL(
13539                 1,
13540                 8, 0, 9, 0,
13541                 10, 0, 11, 0,
13542                 12, 0, 13, 0,
13543                 14, 0, 15, 15));
13544         write_csr(dd, SEND_SC2VLT2, SC2VL_VAL(
13545                 2,
13546                 16, 0, 17, 0,
13547                 18, 0, 19, 0,
13548                 20, 0, 21, 0,
13549                 22, 0, 23, 0));
13550         write_csr(dd, SEND_SC2VLT3, SC2VL_VAL(
13551                 3,
13552                 24, 0, 25, 0,
13553                 26, 0, 27, 0,
13554                 28, 0, 29, 0,
13555                 30, 0, 31, 0));
13556
13557         /* DC maps received packets */
13558         write_csr(dd, DCC_CFG_SC_VL_TABLE_15_0, DC_SC_VL_VAL(
13559                 15_0,
13560                 0, 0, 1, 1,  2, 2,  3, 3,  4, 4,  5, 5,  6, 6,  7,  7,
13561                 8, 0, 9, 0, 10, 0, 11, 0, 12, 0, 13, 0, 14, 0, 15, 15));
13562         write_csr(dd, DCC_CFG_SC_VL_TABLE_31_16, DC_SC_VL_VAL(
13563                 31_16,
13564                 16, 0, 17, 0, 18, 0, 19, 0, 20, 0, 21, 0, 22, 0, 23, 0,
13565                 24, 0, 25, 0, 26, 0, 27, 0, 28, 0, 29, 0, 30, 0, 31, 0));
13566
13567         /* initialize the cached sc2vl values consistently with h/w */
13568         for (i = 0; i < 32; i++) {
13569                 if (i < 8 || i == 15)
13570                         *((u8 *)(dd->sc2vl) + i) = (u8)i;
13571                 else
13572                         *((u8 *)(dd->sc2vl) + i) = 0;
13573         }
13574 }
13575
13576 /*
13577  * Read chip sizes and then reset parts to sane, disabled, values.  We cannot
13578  * depend on the chip going through a power-on reset - a driver may be loaded
13579  * and unloaded many times.
13580  *
13581  * Do not write any CSR values to the chip in this routine - there may be
13582  * a reset following the (possible) FLR in this routine.
13583  *
13584  */
13585 static void init_chip(struct hfi1_devdata *dd)
13586 {
13587         int i;
13588
13589         /*
13590          * Put the HFI CSRs in a known state.
13591          * Combine this with a DC reset.
13592          *
13593          * Stop the device from doing anything while we do a
13594          * reset.  We know there are no other active users of
13595          * the device since we are now in charge.  Turn off
13596          * off all outbound and inbound traffic and make sure
13597          * the device does not generate any interrupts.
13598          */
13599
13600         /* disable send contexts and SDMA engines */
13601         write_csr(dd, SEND_CTRL, 0);
13602         for (i = 0; i < dd->chip_send_contexts; i++)
13603                 write_kctxt_csr(dd, i, SEND_CTXT_CTRL, 0);
13604         for (i = 0; i < dd->chip_sdma_engines; i++)
13605                 write_kctxt_csr(dd, i, SEND_DMA_CTRL, 0);
13606         /* disable port (turn off RXE inbound traffic) and contexts */
13607         write_csr(dd, RCV_CTRL, 0);
13608         for (i = 0; i < dd->chip_rcv_contexts; i++)
13609                 write_csr(dd, RCV_CTXT_CTRL, 0);
13610         /* mask all interrupt sources */
13611         for (i = 0; i < CCE_NUM_INT_CSRS; i++)
13612                 write_csr(dd, CCE_INT_MASK + (8 * i), 0ull);
13613
13614         /*
13615          * DC Reset: do a full DC reset before the register clear.
13616          * A recommended length of time to hold is one CSR read,
13617          * so reread the CceDcCtrl.  Then, hold the DC in reset
13618          * across the clear.
13619          */
13620         write_csr(dd, CCE_DC_CTRL, CCE_DC_CTRL_DC_RESET_SMASK);
13621         (void)read_csr(dd, CCE_DC_CTRL);
13622
13623         if (use_flr) {
13624                 /*
13625                  * A FLR will reset the SPC core and part of the PCIe.
13626                  * The parts that need to be restored have already been
13627                  * saved.
13628                  */
13629                 dd_dev_info(dd, "Resetting CSRs with FLR\n");
13630
13631                 /* do the FLR, the DC reset will remain */
13632                 hfi1_pcie_flr(dd);
13633
13634                 /* restore command and BARs */
13635                 restore_pci_variables(dd);
13636
13637                 if (is_ax(dd)) {
13638                         dd_dev_info(dd, "Resetting CSRs with FLR\n");
13639                         hfi1_pcie_flr(dd);
13640                         restore_pci_variables(dd);
13641                 }
13642         } else {
13643                 dd_dev_info(dd, "Resetting CSRs with writes\n");
13644                 reset_cce_csrs(dd);
13645                 reset_txe_csrs(dd);
13646                 reset_rxe_csrs(dd);
13647                 reset_misc_csrs(dd);
13648         }
13649         /* clear the DC reset */
13650         write_csr(dd, CCE_DC_CTRL, 0);
13651
13652         /* Set the LED off */
13653         setextled(dd, 0);
13654
13655         /*
13656          * Clear the QSFP reset.
13657          * An FLR enforces a 0 on all out pins. The driver does not touch
13658          * ASIC_QSFPn_OUT otherwise.  This leaves RESET_N low and
13659          * anything plugged constantly in reset, if it pays attention
13660          * to RESET_N.
13661          * Prime examples of this are optical cables. Set all pins high.
13662          * I2CCLK and I2CDAT will change per direction, and INT_N and
13663          * MODPRS_N are input only and their value is ignored.
13664          */
13665         write_csr(dd, ASIC_QSFP1_OUT, 0x1f);
13666         write_csr(dd, ASIC_QSFP2_OUT, 0x1f);
13667         init_chip_resources(dd);
13668 }
13669
13670 static void init_early_variables(struct hfi1_devdata *dd)
13671 {
13672         int i;
13673
13674         /* assign link credit variables */
13675         dd->vau = CM_VAU;
13676         dd->link_credits = CM_GLOBAL_CREDITS;
13677         if (is_ax(dd))
13678                 dd->link_credits--;
13679         dd->vcu = cu_to_vcu(hfi1_cu);
13680         /* enough room for 8 MAD packets plus header - 17K */
13681         dd->vl15_init = (8 * (2048 + 128)) / vau_to_au(dd->vau);
13682         if (dd->vl15_init > dd->link_credits)
13683                 dd->vl15_init = dd->link_credits;
13684
13685         write_uninitialized_csrs_and_memories(dd);
13686
13687         if (HFI1_CAP_IS_KSET(PKEY_CHECK))
13688                 for (i = 0; i < dd->num_pports; i++) {
13689                         struct hfi1_pportdata *ppd = &dd->pport[i];
13690
13691                         set_partition_keys(ppd);
13692                 }
13693         init_sc2vl_tables(dd);
13694 }
13695
13696 static void init_kdeth_qp(struct hfi1_devdata *dd)
13697 {
13698         /* user changed the KDETH_QP */
13699         if (kdeth_qp != 0 && kdeth_qp >= 0xff) {
13700                 /* out of range or illegal value */
13701                 dd_dev_err(dd, "Invalid KDETH queue pair prefix, ignoring");
13702                 kdeth_qp = 0;
13703         }
13704         if (kdeth_qp == 0)      /* not set, or failed range check */
13705                 kdeth_qp = DEFAULT_KDETH_QP;
13706
13707         write_csr(dd, SEND_BTH_QP,
13708                   (kdeth_qp & SEND_BTH_QP_KDETH_QP_MASK) <<
13709                   SEND_BTH_QP_KDETH_QP_SHIFT);
13710
13711         write_csr(dd, RCV_BTH_QP,
13712                   (kdeth_qp & RCV_BTH_QP_KDETH_QP_MASK) <<
13713                   RCV_BTH_QP_KDETH_QP_SHIFT);
13714 }
13715
13716 /**
13717  * init_qpmap_table
13718  * @dd - device data
13719  * @first_ctxt - first context
13720  * @last_ctxt - first context
13721  *
13722  * This return sets the qpn mapping table that
13723  * is indexed by qpn[8:1].
13724  *
13725  * The routine will round robin the 256 settings
13726  * from first_ctxt to last_ctxt.
13727  *
13728  * The first/last looks ahead to having specialized
13729  * receive contexts for mgmt and bypass.  Normal
13730  * verbs traffic will assumed to be on a range
13731  * of receive contexts.
13732  */
13733 static void init_qpmap_table(struct hfi1_devdata *dd,
13734                              u32 first_ctxt,
13735                              u32 last_ctxt)
13736 {
13737         u64 reg = 0;
13738         u64 regno = RCV_QP_MAP_TABLE;
13739         int i;
13740         u64 ctxt = first_ctxt;
13741
13742         for (i = 0; i < 256; i++) {
13743                 reg |= ctxt << (8 * (i % 8));
13744                 ctxt++;
13745                 if (ctxt > last_ctxt)
13746                         ctxt = first_ctxt;
13747                 if (i % 8 == 7) {
13748                         write_csr(dd, regno, reg);
13749                         reg = 0;
13750                         regno += 8;
13751                 }
13752         }
13753
13754         add_rcvctrl(dd, RCV_CTRL_RCV_QP_MAP_ENABLE_SMASK
13755                         | RCV_CTRL_RCV_BYPASS_ENABLE_SMASK);
13756 }
13757
13758 struct rsm_map_table {
13759         u64 map[NUM_MAP_REGS];
13760         unsigned int used;
13761 };
13762
13763 struct rsm_rule_data {
13764         u8 offset;
13765         u8 pkt_type;
13766         u32 field1_off;
13767         u32 field2_off;
13768         u32 index1_off;
13769         u32 index1_width;
13770         u32 index2_off;
13771         u32 index2_width;
13772         u32 mask1;
13773         u32 value1;
13774         u32 mask2;
13775         u32 value2;
13776 };
13777
13778 /*
13779  * Return an initialized RMT map table for users to fill in.  OK if it
13780  * returns NULL, indicating no table.
13781  */
13782 static struct rsm_map_table *alloc_rsm_map_table(struct hfi1_devdata *dd)
13783 {
13784         struct rsm_map_table *rmt;
13785         u8 rxcontext = is_ax(dd) ? 0 : 0xff;  /* 0 is default if a0 ver. */
13786
13787         rmt = kmalloc(sizeof(*rmt), GFP_KERNEL);
13788         if (rmt) {
13789                 memset(rmt->map, rxcontext, sizeof(rmt->map));
13790                 rmt->used = 0;
13791         }
13792
13793         return rmt;
13794 }
13795
13796 /*
13797  * Write the final RMT map table to the chip and free the table.  OK if
13798  * table is NULL.
13799  */
13800 static void complete_rsm_map_table(struct hfi1_devdata *dd,
13801                                    struct rsm_map_table *rmt)
13802 {
13803         int i;
13804
13805         if (rmt) {
13806                 /* write table to chip */
13807                 for (i = 0; i < NUM_MAP_REGS; i++)
13808                         write_csr(dd, RCV_RSM_MAP_TABLE + (8 * i), rmt->map[i]);
13809
13810                 /* enable RSM */
13811                 add_rcvctrl(dd, RCV_CTRL_RCV_RSM_ENABLE_SMASK);
13812         }
13813 }
13814
13815 /*
13816  * Add a receive side mapping rule.
13817  */
13818 static void add_rsm_rule(struct hfi1_devdata *dd, u8 rule_index,
13819                          struct rsm_rule_data *rrd)
13820 {
13821         write_csr(dd, RCV_RSM_CFG + (8 * rule_index),
13822                   (u64)rrd->offset << RCV_RSM_CFG_OFFSET_SHIFT |
13823                   1ull << rule_index | /* enable bit */
13824                   (u64)rrd->pkt_type << RCV_RSM_CFG_PACKET_TYPE_SHIFT);
13825         write_csr(dd, RCV_RSM_SELECT + (8 * rule_index),
13826                   (u64)rrd->field1_off << RCV_RSM_SELECT_FIELD1_OFFSET_SHIFT |
13827                   (u64)rrd->field2_off << RCV_RSM_SELECT_FIELD2_OFFSET_SHIFT |
13828                   (u64)rrd->index1_off << RCV_RSM_SELECT_INDEX1_OFFSET_SHIFT |
13829                   (u64)rrd->index1_width << RCV_RSM_SELECT_INDEX1_WIDTH_SHIFT |
13830                   (u64)rrd->index2_off << RCV_RSM_SELECT_INDEX2_OFFSET_SHIFT |
13831                   (u64)rrd->index2_width << RCV_RSM_SELECT_INDEX2_WIDTH_SHIFT);
13832         write_csr(dd, RCV_RSM_MATCH + (8 * rule_index),
13833                   (u64)rrd->mask1 << RCV_RSM_MATCH_MASK1_SHIFT |
13834                   (u64)rrd->value1 << RCV_RSM_MATCH_VALUE1_SHIFT |
13835                   (u64)rrd->mask2 << RCV_RSM_MATCH_MASK2_SHIFT |
13836                   (u64)rrd->value2 << RCV_RSM_MATCH_VALUE2_SHIFT);
13837 }
13838
13839 /* return the number of RSM map table entries that will be used for QOS */
13840 static int qos_rmt_entries(struct hfi1_devdata *dd, unsigned int *mp,
13841                            unsigned int *np)
13842 {
13843         int i;
13844         unsigned int m, n;
13845         u8 max_by_vl = 0;
13846
13847         /* is QOS active at all? */
13848         if (dd->n_krcv_queues <= MIN_KERNEL_KCTXTS ||
13849             num_vls == 1 ||
13850             krcvqsset <= 1)
13851                 goto no_qos;
13852
13853         /* determine bits for qpn */
13854         for (i = 0; i < min_t(unsigned int, num_vls, krcvqsset); i++)
13855                 if (krcvqs[i] > max_by_vl)
13856                         max_by_vl = krcvqs[i];
13857         if (max_by_vl > 32)
13858                 goto no_qos;
13859         m = ilog2(__roundup_pow_of_two(max_by_vl));
13860
13861         /* determine bits for vl */
13862         n = ilog2(__roundup_pow_of_two(num_vls));
13863
13864         /* reject if too much is used */
13865         if ((m + n) > 7)
13866                 goto no_qos;
13867
13868         if (mp)
13869                 *mp = m;
13870         if (np)
13871                 *np = n;
13872
13873         return 1 << (m + n);
13874
13875 no_qos:
13876         if (mp)
13877                 *mp = 0;
13878         if (np)
13879                 *np = 0;
13880         return 0;
13881 }
13882
13883 /**
13884  * init_qos - init RX qos
13885  * @dd - device data
13886  * @rmt - RSM map table
13887  *
13888  * This routine initializes Rule 0 and the RSM map table to implement
13889  * quality of service (qos).
13890  *
13891  * If all of the limit tests succeed, qos is applied based on the array
13892  * interpretation of krcvqs where entry 0 is VL0.
13893  *
13894  * The number of vl bits (n) and the number of qpn bits (m) are computed to
13895  * feed both the RSM map table and the single rule.
13896  */
13897 static void init_qos(struct hfi1_devdata *dd, struct rsm_map_table *rmt)
13898 {
13899         struct rsm_rule_data rrd;
13900         unsigned qpns_per_vl, ctxt, i, qpn, n = 1, m;
13901         unsigned int rmt_entries;
13902         u64 reg;
13903
13904         if (!rmt)
13905                 goto bail;
13906         rmt_entries = qos_rmt_entries(dd, &m, &n);
13907         if (rmt_entries == 0)
13908                 goto bail;
13909         qpns_per_vl = 1 << m;
13910
13911         /* enough room in the map table? */
13912         rmt_entries = 1 << (m + n);
13913         if (rmt->used + rmt_entries >= NUM_MAP_ENTRIES)
13914                 goto bail;
13915
13916         /* add qos entries to the the RSM map table */
13917         for (i = 0, ctxt = FIRST_KERNEL_KCTXT; i < num_vls; i++) {
13918                 unsigned tctxt;
13919
13920                 for (qpn = 0, tctxt = ctxt;
13921                      krcvqs[i] && qpn < qpns_per_vl; qpn++) {
13922                         unsigned idx, regoff, regidx;
13923
13924                         /* generate the index the hardware will produce */
13925                         idx = rmt->used + ((qpn << n) ^ i);
13926                         regoff = (idx % 8) * 8;
13927                         regidx = idx / 8;
13928                         /* replace default with context number */
13929                         reg = rmt->map[regidx];
13930                         reg &= ~(RCV_RSM_MAP_TABLE_RCV_CONTEXT_A_MASK
13931                                 << regoff);
13932                         reg |= (u64)(tctxt++) << regoff;
13933                         rmt->map[regidx] = reg;
13934                         if (tctxt == ctxt + krcvqs[i])
13935                                 tctxt = ctxt;
13936                 }
13937                 ctxt += krcvqs[i];
13938         }
13939
13940         rrd.offset = rmt->used;
13941         rrd.pkt_type = 2;
13942         rrd.field1_off = LRH_BTH_MATCH_OFFSET;
13943         rrd.field2_off = LRH_SC_MATCH_OFFSET;
13944         rrd.index1_off = LRH_SC_SELECT_OFFSET;
13945         rrd.index1_width = n;
13946         rrd.index2_off = QPN_SELECT_OFFSET;
13947         rrd.index2_width = m + n;
13948         rrd.mask1 = LRH_BTH_MASK;
13949         rrd.value1 = LRH_BTH_VALUE;
13950         rrd.mask2 = LRH_SC_MASK;
13951         rrd.value2 = LRH_SC_VALUE;
13952
13953         /* add rule 0 */
13954         add_rsm_rule(dd, 0, &rrd);
13955
13956         /* mark RSM map entries as used */
13957         rmt->used += rmt_entries;
13958         /* map everything else to the mcast/err/vl15 context */
13959         init_qpmap_table(dd, HFI1_CTRL_CTXT, HFI1_CTRL_CTXT);
13960         dd->qos_shift = n + 1;
13961         return;
13962 bail:
13963         dd->qos_shift = 1;
13964         init_qpmap_table(dd, FIRST_KERNEL_KCTXT, dd->n_krcv_queues - 1);
13965 }
13966
13967 static void init_user_fecn_handling(struct hfi1_devdata *dd,
13968                                     struct rsm_map_table *rmt)
13969 {
13970         struct rsm_rule_data rrd;
13971         u64 reg;
13972         int i, idx, regoff, regidx;
13973         u8 offset;
13974
13975         /* there needs to be enough room in the map table */
13976         if (rmt->used + dd->num_user_contexts >= NUM_MAP_ENTRIES) {
13977                 dd_dev_err(dd, "User FECN handling disabled - too many user contexts allocated\n");
13978                 return;
13979         }
13980
13981         /*
13982          * RSM will extract the destination context as an index into the
13983          * map table.  The destination contexts are a sequential block
13984          * in the range first_user_ctxt...num_rcv_contexts-1 (inclusive).
13985          * Map entries are accessed as offset + extracted value.  Adjust
13986          * the added offset so this sequence can be placed anywhere in
13987          * the table - as long as the entries themselves do not wrap.
13988          * There are only enough bits in offset for the table size, so
13989          * start with that to allow for a "negative" offset.
13990          */
13991         offset = (u8)(NUM_MAP_ENTRIES + (int)rmt->used -
13992                                                 (int)dd->first_user_ctxt);
13993
13994         for (i = dd->first_user_ctxt, idx = rmt->used;
13995                                 i < dd->num_rcv_contexts; i++, idx++) {
13996                 /* replace with identity mapping */
13997                 regoff = (idx % 8) * 8;
13998                 regidx = idx / 8;
13999                 reg = rmt->map[regidx];
14000                 reg &= ~(RCV_RSM_MAP_TABLE_RCV_CONTEXT_A_MASK << regoff);
14001                 reg |= (u64)i << regoff;
14002                 rmt->map[regidx] = reg;
14003         }
14004
14005         /*
14006          * For RSM intercept of Expected FECN packets:
14007          * o packet type 0 - expected
14008          * o match on F (bit 95), using select/match 1, and
14009          * o match on SH (bit 133), using select/match 2.
14010          *
14011          * Use index 1 to extract the 8-bit receive context from DestQP
14012          * (start at bit 64).  Use that as the RSM map table index.
14013          */
14014         rrd.offset = offset;
14015         rrd.pkt_type = 0;
14016         rrd.field1_off = 95;
14017         rrd.field2_off = 133;
14018         rrd.index1_off = 64;
14019         rrd.index1_width = 8;
14020         rrd.index2_off = 0;
14021         rrd.index2_width = 0;
14022         rrd.mask1 = 1;
14023         rrd.value1 = 1;
14024         rrd.mask2 = 1;
14025         rrd.value2 = 1;
14026
14027         /* add rule 1 */
14028         add_rsm_rule(dd, 1, &rrd);
14029
14030         rmt->used += dd->num_user_contexts;
14031 }
14032
14033 static void init_rxe(struct hfi1_devdata *dd)
14034 {
14035         struct rsm_map_table *rmt;
14036
14037         /* enable all receive errors */
14038         write_csr(dd, RCV_ERR_MASK, ~0ull);
14039
14040         rmt = alloc_rsm_map_table(dd);
14041         /* set up QOS, including the QPN map table */
14042         init_qos(dd, rmt);
14043         init_user_fecn_handling(dd, rmt);
14044         complete_rsm_map_table(dd, rmt);
14045         kfree(rmt);
14046
14047         /*
14048          * make sure RcvCtrl.RcvWcb <= PCIe Device Control
14049          * Register Max_Payload_Size (PCI_EXP_DEVCTL in Linux PCIe config
14050          * space, PciCfgCap2.MaxPayloadSize in HFI).  There is only one
14051          * invalid configuration: RcvCtrl.RcvWcb set to its max of 256 and
14052          * Max_PayLoad_Size set to its minimum of 128.
14053          *
14054          * Presently, RcvCtrl.RcvWcb is not modified from its default of 0
14055          * (64 bytes).  Max_Payload_Size is possibly modified upward in
14056          * tune_pcie_caps() which is called after this routine.
14057          */
14058 }
14059
14060 static void init_other(struct hfi1_devdata *dd)
14061 {
14062         /* enable all CCE errors */
14063         write_csr(dd, CCE_ERR_MASK, ~0ull);
14064         /* enable *some* Misc errors */
14065         write_csr(dd, MISC_ERR_MASK, DRIVER_MISC_MASK);
14066         /* enable all DC errors, except LCB */
14067         write_csr(dd, DCC_ERR_FLG_EN, ~0ull);
14068         write_csr(dd, DC_DC8051_ERR_EN, ~0ull);
14069 }
14070
14071 /*
14072  * Fill out the given AU table using the given CU.  A CU is defined in terms
14073  * AUs.  The table is a an encoding: given the index, how many AUs does that
14074  * represent?
14075  *
14076  * NOTE: Assumes that the register layout is the same for the
14077  * local and remote tables.
14078  */
14079 static void assign_cm_au_table(struct hfi1_devdata *dd, u32 cu,
14080                                u32 csr0to3, u32 csr4to7)
14081 {
14082         write_csr(dd, csr0to3,
14083                   0ull << SEND_CM_LOCAL_AU_TABLE0_TO3_LOCAL_AU_TABLE0_SHIFT |
14084                   1ull << SEND_CM_LOCAL_AU_TABLE0_TO3_LOCAL_AU_TABLE1_SHIFT |
14085                   2ull * cu <<
14086                   SEND_CM_LOCAL_AU_TABLE0_TO3_LOCAL_AU_TABLE2_SHIFT |
14087                   4ull * cu <<
14088                   SEND_CM_LOCAL_AU_TABLE0_TO3_LOCAL_AU_TABLE3_SHIFT);
14089         write_csr(dd, csr4to7,
14090                   8ull * cu <<
14091                   SEND_CM_LOCAL_AU_TABLE4_TO7_LOCAL_AU_TABLE4_SHIFT |
14092                   16ull * cu <<
14093                   SEND_CM_LOCAL_AU_TABLE4_TO7_LOCAL_AU_TABLE5_SHIFT |
14094                   32ull * cu <<
14095                   SEND_CM_LOCAL_AU_TABLE4_TO7_LOCAL_AU_TABLE6_SHIFT |
14096                   64ull * cu <<
14097                   SEND_CM_LOCAL_AU_TABLE4_TO7_LOCAL_AU_TABLE7_SHIFT);
14098 }
14099
14100 static void assign_local_cm_au_table(struct hfi1_devdata *dd, u8 vcu)
14101 {
14102         assign_cm_au_table(dd, vcu_to_cu(vcu), SEND_CM_LOCAL_AU_TABLE0_TO3,
14103                            SEND_CM_LOCAL_AU_TABLE4_TO7);
14104 }
14105
14106 void assign_remote_cm_au_table(struct hfi1_devdata *dd, u8 vcu)
14107 {
14108         assign_cm_au_table(dd, vcu_to_cu(vcu), SEND_CM_REMOTE_AU_TABLE0_TO3,
14109                            SEND_CM_REMOTE_AU_TABLE4_TO7);
14110 }
14111
14112 static void init_txe(struct hfi1_devdata *dd)
14113 {
14114         int i;
14115
14116         /* enable all PIO, SDMA, general, and Egress errors */
14117         write_csr(dd, SEND_PIO_ERR_MASK, ~0ull);
14118         write_csr(dd, SEND_DMA_ERR_MASK, ~0ull);
14119         write_csr(dd, SEND_ERR_MASK, ~0ull);
14120         write_csr(dd, SEND_EGRESS_ERR_MASK, ~0ull);
14121
14122         /* enable all per-context and per-SDMA engine errors */
14123         for (i = 0; i < dd->chip_send_contexts; i++)
14124                 write_kctxt_csr(dd, i, SEND_CTXT_ERR_MASK, ~0ull);
14125         for (i = 0; i < dd->chip_sdma_engines; i++)
14126                 write_kctxt_csr(dd, i, SEND_DMA_ENG_ERR_MASK, ~0ull);
14127
14128         /* set the local CU to AU mapping */
14129         assign_local_cm_au_table(dd, dd->vcu);
14130
14131         /*
14132          * Set reasonable default for Credit Return Timer
14133          * Don't set on Simulator - causes it to choke.
14134          */
14135         if (dd->icode != ICODE_FUNCTIONAL_SIMULATOR)
14136                 write_csr(dd, SEND_CM_TIMER_CTRL, HFI1_CREDIT_RETURN_RATE);
14137 }
14138
14139 int hfi1_set_ctxt_jkey(struct hfi1_devdata *dd, unsigned ctxt, u16 jkey)
14140 {
14141         struct hfi1_ctxtdata *rcd = dd->rcd[ctxt];
14142         unsigned sctxt;
14143         int ret = 0;
14144         u64 reg;
14145
14146         if (!rcd || !rcd->sc) {
14147                 ret = -EINVAL;
14148                 goto done;
14149         }
14150         sctxt = rcd->sc->hw_context;
14151         reg = SEND_CTXT_CHECK_JOB_KEY_MASK_SMASK | /* mask is always 1's */
14152                 ((jkey & SEND_CTXT_CHECK_JOB_KEY_VALUE_MASK) <<
14153                  SEND_CTXT_CHECK_JOB_KEY_VALUE_SHIFT);
14154         /* JOB_KEY_ALLOW_PERMISSIVE is not allowed by default */
14155         if (HFI1_CAP_KGET_MASK(rcd->flags, ALLOW_PERM_JKEY))
14156                 reg |= SEND_CTXT_CHECK_JOB_KEY_ALLOW_PERMISSIVE_SMASK;
14157         write_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_JOB_KEY, reg);
14158         /*
14159          * Enable send-side J_KEY integrity check, unless this is A0 h/w
14160          */
14161         if (!is_ax(dd)) {
14162                 reg = read_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_ENABLE);
14163                 reg |= SEND_CTXT_CHECK_ENABLE_CHECK_JOB_KEY_SMASK;
14164                 write_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_ENABLE, reg);
14165         }
14166
14167         /* Enable J_KEY check on receive context. */
14168         reg = RCV_KEY_CTRL_JOB_KEY_ENABLE_SMASK |
14169                 ((jkey & RCV_KEY_CTRL_JOB_KEY_VALUE_MASK) <<
14170                  RCV_KEY_CTRL_JOB_KEY_VALUE_SHIFT);
14171         write_kctxt_csr(dd, ctxt, RCV_KEY_CTRL, reg);
14172 done:
14173         return ret;
14174 }
14175
14176 int hfi1_clear_ctxt_jkey(struct hfi1_devdata *dd, unsigned ctxt)
14177 {
14178         struct hfi1_ctxtdata *rcd = dd->rcd[ctxt];
14179         unsigned sctxt;
14180         int ret = 0;
14181         u64 reg;
14182
14183         if (!rcd || !rcd->sc) {
14184                 ret = -EINVAL;
14185                 goto done;
14186         }
14187         sctxt = rcd->sc->hw_context;
14188         write_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_JOB_KEY, 0);
14189         /*
14190          * Disable send-side J_KEY integrity check, unless this is A0 h/w.
14191          * This check would not have been enabled for A0 h/w, see
14192          * set_ctxt_jkey().
14193          */
14194         if (!is_ax(dd)) {
14195                 reg = read_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_ENABLE);
14196                 reg &= ~SEND_CTXT_CHECK_ENABLE_CHECK_JOB_KEY_SMASK;
14197                 write_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_ENABLE, reg);
14198         }
14199         /* Turn off the J_KEY on the receive side */
14200         write_kctxt_csr(dd, ctxt, RCV_KEY_CTRL, 0);
14201 done:
14202         return ret;
14203 }
14204
14205 int hfi1_set_ctxt_pkey(struct hfi1_devdata *dd, unsigned ctxt, u16 pkey)
14206 {
14207         struct hfi1_ctxtdata *rcd;
14208         unsigned sctxt;
14209         int ret = 0;
14210         u64 reg;
14211
14212         if (ctxt < dd->num_rcv_contexts) {
14213                 rcd = dd->rcd[ctxt];
14214         } else {
14215                 ret = -EINVAL;
14216                 goto done;
14217         }
14218         if (!rcd || !rcd->sc) {
14219                 ret = -EINVAL;
14220                 goto done;
14221         }
14222         sctxt = rcd->sc->hw_context;
14223         reg = ((u64)pkey & SEND_CTXT_CHECK_PARTITION_KEY_VALUE_MASK) <<
14224                 SEND_CTXT_CHECK_PARTITION_KEY_VALUE_SHIFT;
14225         write_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_PARTITION_KEY, reg);
14226         reg = read_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_ENABLE);
14227         reg |= SEND_CTXT_CHECK_ENABLE_CHECK_PARTITION_KEY_SMASK;
14228         reg &= ~SEND_CTXT_CHECK_ENABLE_DISALLOW_KDETH_PACKETS_SMASK;
14229         write_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_ENABLE, reg);
14230 done:
14231         return ret;
14232 }
14233
14234 int hfi1_clear_ctxt_pkey(struct hfi1_devdata *dd, unsigned ctxt)
14235 {
14236         struct hfi1_ctxtdata *rcd;
14237         unsigned sctxt;
14238         int ret = 0;
14239         u64 reg;
14240
14241         if (ctxt < dd->num_rcv_contexts) {
14242                 rcd = dd->rcd[ctxt];
14243         } else {
14244                 ret = -EINVAL;
14245                 goto done;
14246         }
14247         if (!rcd || !rcd->sc) {
14248                 ret = -EINVAL;
14249                 goto done;
14250         }
14251         sctxt = rcd->sc->hw_context;
14252         reg = read_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_ENABLE);
14253         reg &= ~SEND_CTXT_CHECK_ENABLE_CHECK_PARTITION_KEY_SMASK;
14254         write_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_ENABLE, reg);
14255         write_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_PARTITION_KEY, 0);
14256 done:
14257         return ret;
14258 }
14259
14260 /*
14261  * Start doing the clean up the the chip. Our clean up happens in multiple
14262  * stages and this is just the first.
14263  */
14264 void hfi1_start_cleanup(struct hfi1_devdata *dd)
14265 {
14266         aspm_exit(dd);
14267         free_cntrs(dd);
14268         free_rcverr(dd);
14269         clean_up_interrupts(dd);
14270         finish_chip_resources(dd);
14271 }
14272
14273 #define HFI_BASE_GUID(dev) \
14274         ((dev)->base_guid & ~(1ULL << GUID_HFI_INDEX_SHIFT))
14275
14276 /*
14277  * Information can be shared between the two HFIs on the same ASIC
14278  * in the same OS.  This function finds the peer device and sets
14279  * up a shared structure.
14280  */
14281 static int init_asic_data(struct hfi1_devdata *dd)
14282 {
14283         unsigned long flags;
14284         struct hfi1_devdata *tmp, *peer = NULL;
14285         struct hfi1_asic_data *asic_data;
14286         int ret = 0;
14287
14288         /* pre-allocate the asic structure in case we are the first device */
14289         asic_data = kzalloc(sizeof(*dd->asic_data), GFP_KERNEL);
14290         if (!asic_data)
14291                 return -ENOMEM;
14292
14293         spin_lock_irqsave(&hfi1_devs_lock, flags);
14294         /* Find our peer device */
14295         list_for_each_entry(tmp, &hfi1_dev_list, list) {
14296                 if ((HFI_BASE_GUID(dd) == HFI_BASE_GUID(tmp)) &&
14297                     dd->unit != tmp->unit) {
14298                         peer = tmp;
14299                         break;
14300                 }
14301         }
14302
14303         if (peer) {
14304                 /* use already allocated structure */
14305                 dd->asic_data = peer->asic_data;
14306                 kfree(asic_data);
14307         } else {
14308                 dd->asic_data = asic_data;
14309                 mutex_init(&dd->asic_data->asic_resource_mutex);
14310         }
14311         dd->asic_data->dds[dd->hfi1_id] = dd; /* self back-pointer */
14312         spin_unlock_irqrestore(&hfi1_devs_lock, flags);
14313
14314         /* first one through - set up i2c devices */
14315         if (!peer)
14316                 ret = set_up_i2c(dd, dd->asic_data);
14317
14318         return ret;
14319 }
14320
14321 /*
14322  * Set dd->boardname.  Use a generic name if a name is not returned from
14323  * EFI variable space.
14324  *
14325  * Return 0 on success, -ENOMEM if space could not be allocated.
14326  */
14327 static int obtain_boardname(struct hfi1_devdata *dd)
14328 {
14329         /* generic board description */
14330         const char generic[] =
14331                 "Intel Omni-Path Host Fabric Interface Adapter 100 Series";
14332         unsigned long size;
14333         int ret;
14334
14335         ret = read_hfi1_efi_var(dd, "description", &size,
14336                                 (void **)&dd->boardname);
14337         if (ret) {
14338                 dd_dev_info(dd, "Board description not found\n");
14339                 /* use generic description */
14340                 dd->boardname = kstrdup(generic, GFP_KERNEL);
14341                 if (!dd->boardname)
14342                         return -ENOMEM;
14343         }
14344         return 0;
14345 }
14346
14347 /*
14348  * Check the interrupt registers to make sure that they are mapped correctly.
14349  * It is intended to help user identify any mismapping by VMM when the driver
14350  * is running in a VM. This function should only be called before interrupt
14351  * is set up properly.
14352  *
14353  * Return 0 on success, -EINVAL on failure.
14354  */
14355 static int check_int_registers(struct hfi1_devdata *dd)
14356 {
14357         u64 reg;
14358         u64 all_bits = ~(u64)0;
14359         u64 mask;
14360
14361         /* Clear CceIntMask[0] to avoid raising any interrupts */
14362         mask = read_csr(dd, CCE_INT_MASK);
14363         write_csr(dd, CCE_INT_MASK, 0ull);
14364         reg = read_csr(dd, CCE_INT_MASK);
14365         if (reg)
14366                 goto err_exit;
14367
14368         /* Clear all interrupt status bits */
14369         write_csr(dd, CCE_INT_CLEAR, all_bits);
14370         reg = read_csr(dd, CCE_INT_STATUS);
14371         if (reg)
14372                 goto err_exit;
14373
14374         /* Set all interrupt status bits */
14375         write_csr(dd, CCE_INT_FORCE, all_bits);
14376         reg = read_csr(dd, CCE_INT_STATUS);
14377         if (reg != all_bits)
14378                 goto err_exit;
14379
14380         /* Restore the interrupt mask */
14381         write_csr(dd, CCE_INT_CLEAR, all_bits);
14382         write_csr(dd, CCE_INT_MASK, mask);
14383
14384         return 0;
14385 err_exit:
14386         write_csr(dd, CCE_INT_MASK, mask);
14387         dd_dev_err(dd, "Interrupt registers not properly mapped by VMM\n");
14388         return -EINVAL;
14389 }
14390
14391 /**
14392  * Allocate and initialize the device structure for the hfi.
14393  * @dev: the pci_dev for hfi1_ib device
14394  * @ent: pci_device_id struct for this dev
14395  *
14396  * Also allocates, initializes, and returns the devdata struct for this
14397  * device instance
14398  *
14399  * This is global, and is called directly at init to set up the
14400  * chip-specific function pointers for later use.
14401  */
14402 struct hfi1_devdata *hfi1_init_dd(struct pci_dev *pdev,
14403                                   const struct pci_device_id *ent)
14404 {
14405         struct hfi1_devdata *dd;
14406         struct hfi1_pportdata *ppd;
14407         u64 reg;
14408         int i, ret;
14409         static const char * const inames[] = { /* implementation names */
14410                 "RTL silicon",
14411                 "RTL VCS simulation",
14412                 "RTL FPGA emulation",
14413                 "Functional simulator"
14414         };
14415         struct pci_dev *parent = pdev->bus->self;
14416
14417         dd = hfi1_alloc_devdata(pdev, NUM_IB_PORTS *
14418                                 sizeof(struct hfi1_pportdata));
14419         if (IS_ERR(dd))
14420                 goto bail;
14421         ppd = dd->pport;
14422         for (i = 0; i < dd->num_pports; i++, ppd++) {
14423                 int vl;
14424                 /* init common fields */
14425                 hfi1_init_pportdata(pdev, ppd, dd, 0, 1);
14426                 /* DC supports 4 link widths */
14427                 ppd->link_width_supported =
14428                         OPA_LINK_WIDTH_1X | OPA_LINK_WIDTH_2X |
14429                         OPA_LINK_WIDTH_3X | OPA_LINK_WIDTH_4X;
14430                 ppd->link_width_downgrade_supported =
14431                         ppd->link_width_supported;
14432                 /* start out enabling only 4X */
14433                 ppd->link_width_enabled = OPA_LINK_WIDTH_4X;
14434                 ppd->link_width_downgrade_enabled =
14435                                         ppd->link_width_downgrade_supported;
14436                 /* link width active is 0 when link is down */
14437                 /* link width downgrade active is 0 when link is down */
14438
14439                 if (num_vls < HFI1_MIN_VLS_SUPPORTED ||
14440                     num_vls > HFI1_MAX_VLS_SUPPORTED) {
14441                         hfi1_early_err(&pdev->dev,
14442                                        "Invalid num_vls %u, using %u VLs\n",
14443                                     num_vls, HFI1_MAX_VLS_SUPPORTED);
14444                         num_vls = HFI1_MAX_VLS_SUPPORTED;
14445                 }
14446                 ppd->vls_supported = num_vls;
14447                 ppd->vls_operational = ppd->vls_supported;
14448                 ppd->actual_vls_operational = ppd->vls_supported;
14449                 /* Set the default MTU. */
14450                 for (vl = 0; vl < num_vls; vl++)
14451                         dd->vld[vl].mtu = hfi1_max_mtu;
14452                 dd->vld[15].mtu = MAX_MAD_PACKET;
14453                 /*
14454                  * Set the initial values to reasonable default, will be set
14455                  * for real when link is up.
14456                  */
14457                 ppd->lstate = IB_PORT_DOWN;
14458                 ppd->overrun_threshold = 0x4;
14459                 ppd->phy_error_threshold = 0xf;
14460                 ppd->port_crc_mode_enabled = link_crc_mask;
14461                 /* initialize supported LTP CRC mode */
14462                 ppd->port_ltp_crc_mode = cap_to_port_ltp(link_crc_mask) << 8;
14463                 /* initialize enabled LTP CRC mode */
14464                 ppd->port_ltp_crc_mode |= cap_to_port_ltp(link_crc_mask) << 4;
14465                 /* start in offline */
14466                 ppd->host_link_state = HLS_DN_OFFLINE;
14467                 init_vl_arb_caches(ppd);
14468                 ppd->last_pstate = 0xff; /* invalid value */
14469         }
14470
14471         dd->link_default = HLS_DN_POLL;
14472
14473         /*
14474          * Do remaining PCIe setup and save PCIe values in dd.
14475          * Any error printing is already done by the init code.
14476          * On return, we have the chip mapped.
14477          */
14478         ret = hfi1_pcie_ddinit(dd, pdev);
14479         if (ret < 0)
14480                 goto bail_free;
14481
14482         /* verify that reads actually work, save revision for reset check */
14483         dd->revision = read_csr(dd, CCE_REVISION);
14484         if (dd->revision == ~(u64)0) {
14485                 dd_dev_err(dd, "cannot read chip CSRs\n");
14486                 ret = -EINVAL;
14487                 goto bail_cleanup;
14488         }
14489         dd->majrev = (dd->revision >> CCE_REVISION_CHIP_REV_MAJOR_SHIFT)
14490                         & CCE_REVISION_CHIP_REV_MAJOR_MASK;
14491         dd->minrev = (dd->revision >> CCE_REVISION_CHIP_REV_MINOR_SHIFT)
14492                         & CCE_REVISION_CHIP_REV_MINOR_MASK;
14493
14494         /*
14495          * Check interrupt registers mapping if the driver has no access to
14496          * the upstream component. In this case, it is likely that the driver
14497          * is running in a VM.
14498          */
14499         if (!parent) {
14500                 ret = check_int_registers(dd);
14501                 if (ret)
14502                         goto bail_cleanup;
14503         }
14504
14505         /*
14506          * obtain the hardware ID - NOT related to unit, which is a
14507          * software enumeration
14508          */
14509         reg = read_csr(dd, CCE_REVISION2);
14510         dd->hfi1_id = (reg >> CCE_REVISION2_HFI_ID_SHIFT)
14511                                         & CCE_REVISION2_HFI_ID_MASK;
14512         /* the variable size will remove unwanted bits */
14513         dd->icode = reg >> CCE_REVISION2_IMPL_CODE_SHIFT;
14514         dd->irev = reg >> CCE_REVISION2_IMPL_REVISION_SHIFT;
14515         dd_dev_info(dd, "Implementation: %s, revision 0x%x\n",
14516                     dd->icode < ARRAY_SIZE(inames) ?
14517                     inames[dd->icode] : "unknown", (int)dd->irev);
14518
14519         /* speeds the hardware can support */
14520         dd->pport->link_speed_supported = OPA_LINK_SPEED_25G;
14521         /* speeds allowed to run at */
14522         dd->pport->link_speed_enabled = dd->pport->link_speed_supported;
14523         /* give a reasonable active value, will be set on link up */
14524         dd->pport->link_speed_active = OPA_LINK_SPEED_25G;
14525
14526         dd->chip_rcv_contexts = read_csr(dd, RCV_CONTEXTS);
14527         dd->chip_send_contexts = read_csr(dd, SEND_CONTEXTS);
14528         dd->chip_sdma_engines = read_csr(dd, SEND_DMA_ENGINES);
14529         dd->chip_pio_mem_size = read_csr(dd, SEND_PIO_MEM_SIZE);
14530         dd->chip_sdma_mem_size = read_csr(dd, SEND_DMA_MEM_SIZE);
14531         /* fix up link widths for emulation _p */
14532         ppd = dd->pport;
14533         if (dd->icode == ICODE_FPGA_EMULATION && is_emulator_p(dd)) {
14534                 ppd->link_width_supported =
14535                         ppd->link_width_enabled =
14536                         ppd->link_width_downgrade_supported =
14537                         ppd->link_width_downgrade_enabled =
14538                                 OPA_LINK_WIDTH_1X;
14539         }
14540         /* insure num_vls isn't larger than number of sdma engines */
14541         if (HFI1_CAP_IS_KSET(SDMA) && num_vls > dd->chip_sdma_engines) {
14542                 dd_dev_err(dd, "num_vls %u too large, using %u VLs\n",
14543                            num_vls, dd->chip_sdma_engines);
14544                 num_vls = dd->chip_sdma_engines;
14545                 ppd->vls_supported = dd->chip_sdma_engines;
14546                 ppd->vls_operational = ppd->vls_supported;
14547         }
14548
14549         /*
14550          * Convert the ns parameter to the 64 * cclocks used in the CSR.
14551          * Limit the max if larger than the field holds.  If timeout is
14552          * non-zero, then the calculated field will be at least 1.
14553          *
14554          * Must be after icode is set up - the cclock rate depends
14555          * on knowing the hardware being used.
14556          */
14557         dd->rcv_intr_timeout_csr = ns_to_cclock(dd, rcv_intr_timeout) / 64;
14558         if (dd->rcv_intr_timeout_csr >
14559                         RCV_AVAIL_TIME_OUT_TIME_OUT_RELOAD_MASK)
14560                 dd->rcv_intr_timeout_csr =
14561                         RCV_AVAIL_TIME_OUT_TIME_OUT_RELOAD_MASK;
14562         else if (dd->rcv_intr_timeout_csr == 0 && rcv_intr_timeout)
14563                 dd->rcv_intr_timeout_csr = 1;
14564
14565         /* needs to be done before we look for the peer device */
14566         read_guid(dd);
14567
14568         /* set up shared ASIC data with peer device */
14569         ret = init_asic_data(dd);
14570         if (ret)
14571                 goto bail_cleanup;
14572
14573         /* obtain chip sizes, reset chip CSRs */
14574         init_chip(dd);
14575
14576         /* read in the PCIe link speed information */
14577         ret = pcie_speeds(dd);
14578         if (ret)
14579                 goto bail_cleanup;
14580
14581         /* call before get_platform_config(), after init_chip_resources() */
14582         ret = eprom_init(dd);
14583         if (ret)
14584                 goto bail_free_rcverr;
14585
14586         /* Needs to be called before hfi1_firmware_init */
14587         get_platform_config(dd);
14588
14589         /* read in firmware */
14590         ret = hfi1_firmware_init(dd);
14591         if (ret)
14592                 goto bail_cleanup;
14593
14594         /*
14595          * In general, the PCIe Gen3 transition must occur after the
14596          * chip has been idled (so it won't initiate any PCIe transactions
14597          * e.g. an interrupt) and before the driver changes any registers
14598          * (the transition will reset the registers).
14599          *
14600          * In particular, place this call after:
14601          * - init_chip()     - the chip will not initiate any PCIe transactions
14602          * - pcie_speeds()   - reads the current link speed
14603          * - hfi1_firmware_init() - the needed firmware is ready to be
14604          *                          downloaded
14605          */
14606         ret = do_pcie_gen3_transition(dd);
14607         if (ret)
14608                 goto bail_cleanup;
14609
14610         /* start setting dd values and adjusting CSRs */
14611         init_early_variables(dd);
14612
14613         parse_platform_config(dd);
14614
14615         ret = obtain_boardname(dd);
14616         if (ret)
14617                 goto bail_cleanup;
14618
14619         snprintf(dd->boardversion, BOARD_VERS_MAX,
14620                  "ChipABI %u.%u, ChipRev %u.%u, SW Compat %llu\n",
14621                  HFI1_CHIP_VERS_MAJ, HFI1_CHIP_VERS_MIN,
14622                  (u32)dd->majrev,
14623                  (u32)dd->minrev,
14624                  (dd->revision >> CCE_REVISION_SW_SHIFT)
14625                     & CCE_REVISION_SW_MASK);
14626
14627         ret = set_up_context_variables(dd);
14628         if (ret)
14629                 goto bail_cleanup;
14630
14631         /* set initial RXE CSRs */
14632         init_rxe(dd);
14633         /* set initial TXE CSRs */
14634         init_txe(dd);
14635         /* set initial non-RXE, non-TXE CSRs */
14636         init_other(dd);
14637         /* set up KDETH QP prefix in both RX and TX CSRs */
14638         init_kdeth_qp(dd);
14639
14640         ret = hfi1_dev_affinity_init(dd);
14641         if (ret)
14642                 goto bail_cleanup;
14643
14644         /* send contexts must be set up before receive contexts */
14645         ret = init_send_contexts(dd);
14646         if (ret)
14647                 goto bail_cleanup;
14648
14649         ret = hfi1_create_ctxts(dd);
14650         if (ret)
14651                 goto bail_cleanup;
14652
14653         dd->rcvhdrsize = DEFAULT_RCVHDRSIZE;
14654         /*
14655          * rcd[0] is guaranteed to be valid by this point. Also, all
14656          * context are using the same value, as per the module parameter.
14657          */
14658         dd->rhf_offset = dd->rcd[0]->rcvhdrqentsize - sizeof(u64) / sizeof(u32);
14659
14660         ret = init_pervl_scs(dd);
14661         if (ret)
14662                 goto bail_cleanup;
14663
14664         /* sdma init */
14665         for (i = 0; i < dd->num_pports; ++i) {
14666                 ret = sdma_init(dd, i);
14667                 if (ret)
14668                         goto bail_cleanup;
14669         }
14670
14671         /* use contexts created by hfi1_create_ctxts */
14672         ret = set_up_interrupts(dd);
14673         if (ret)
14674                 goto bail_cleanup;
14675
14676         /* set up LCB access - must be after set_up_interrupts() */
14677         init_lcb_access(dd);
14678
14679         /*
14680          * Serial number is created from the base guid:
14681          * [27:24] = base guid [38:35]
14682          * [23: 0] = base guid [23: 0]
14683          */
14684         snprintf(dd->serial, SERIAL_MAX, "0x%08llx\n",
14685                  (dd->base_guid & 0xFFFFFF) |
14686                      ((dd->base_guid >> 11) & 0xF000000));
14687
14688         dd->oui1 = dd->base_guid >> 56 & 0xFF;
14689         dd->oui2 = dd->base_guid >> 48 & 0xFF;
14690         dd->oui3 = dd->base_guid >> 40 & 0xFF;
14691
14692         ret = load_firmware(dd); /* asymmetric with dispose_firmware() */
14693         if (ret)
14694                 goto bail_clear_intr;
14695
14696         thermal_init(dd);
14697
14698         ret = init_cntrs(dd);
14699         if (ret)
14700                 goto bail_clear_intr;
14701
14702         ret = init_rcverr(dd);
14703         if (ret)
14704                 goto bail_free_cntrs;
14705
14706         init_completion(&dd->user_comp);
14707
14708         /* The user refcount starts with one to inidicate an active device */
14709         atomic_set(&dd->user_refcount, 1);
14710
14711         goto bail;
14712
14713 bail_free_rcverr:
14714         free_rcverr(dd);
14715 bail_free_cntrs:
14716         free_cntrs(dd);
14717 bail_clear_intr:
14718         clean_up_interrupts(dd);
14719 bail_cleanup:
14720         hfi1_pcie_ddcleanup(dd);
14721 bail_free:
14722         hfi1_free_devdata(dd);
14723         dd = ERR_PTR(ret);
14724 bail:
14725         return dd;
14726 }
14727
14728 static u16 delay_cycles(struct hfi1_pportdata *ppd, u32 desired_egress_rate,
14729                         u32 dw_len)
14730 {
14731         u32 delta_cycles;
14732         u32 current_egress_rate = ppd->current_egress_rate;
14733         /* rates here are in units of 10^6 bits/sec */
14734
14735         if (desired_egress_rate == -1)
14736                 return 0; /* shouldn't happen */
14737
14738         if (desired_egress_rate >= current_egress_rate)
14739                 return 0; /* we can't help go faster, only slower */
14740
14741         delta_cycles = egress_cycles(dw_len * 4, desired_egress_rate) -
14742                         egress_cycles(dw_len * 4, current_egress_rate);
14743
14744         return (u16)delta_cycles;
14745 }
14746
14747 /**
14748  * create_pbc - build a pbc for transmission
14749  * @flags: special case flags or-ed in built pbc
14750  * @srate: static rate
14751  * @vl: vl
14752  * @dwlen: dword length (header words + data words + pbc words)
14753  *
14754  * Create a PBC with the given flags, rate, VL, and length.
14755  *
14756  * NOTE: The PBC created will not insert any HCRC - all callers but one are
14757  * for verbs, which does not use this PSM feature.  The lone other caller
14758  * is for the diagnostic interface which calls this if the user does not
14759  * supply their own PBC.
14760  */
14761 u64 create_pbc(struct hfi1_pportdata *ppd, u64 flags, int srate_mbs, u32 vl,
14762                u32 dw_len)
14763 {
14764         u64 pbc, delay = 0;
14765
14766         if (unlikely(srate_mbs))
14767                 delay = delay_cycles(ppd, srate_mbs, dw_len);
14768
14769         pbc = flags
14770                 | (delay << PBC_STATIC_RATE_CONTROL_COUNT_SHIFT)
14771                 | ((u64)PBC_IHCRC_NONE << PBC_INSERT_HCRC_SHIFT)
14772                 | (vl & PBC_VL_MASK) << PBC_VL_SHIFT
14773                 | (dw_len & PBC_LENGTH_DWS_MASK)
14774                         << PBC_LENGTH_DWS_SHIFT;
14775
14776         return pbc;
14777 }
14778
14779 #define SBUS_THERMAL    0x4f
14780 #define SBUS_THERM_MONITOR_MODE 0x1
14781
14782 #define THERM_FAILURE(dev, ret, reason) \
14783         dd_dev_err((dd),                                                \
14784                    "Thermal sensor initialization failed: %s (%d)\n",   \
14785                    (reason), (ret))
14786
14787 /*
14788  * Initialize the thermal sensor.
14789  *
14790  * After initialization, enable polling of thermal sensor through
14791  * SBus interface. In order for this to work, the SBus Master
14792  * firmware has to be loaded due to the fact that the HW polling
14793  * logic uses SBus interrupts, which are not supported with
14794  * default firmware. Otherwise, no data will be returned through
14795  * the ASIC_STS_THERM CSR.
14796  */
14797 static int thermal_init(struct hfi1_devdata *dd)
14798 {
14799         int ret = 0;
14800
14801         if (dd->icode != ICODE_RTL_SILICON ||
14802             check_chip_resource(dd, CR_THERM_INIT, NULL))
14803                 return ret;
14804
14805         ret = acquire_chip_resource(dd, CR_SBUS, SBUS_TIMEOUT);
14806         if (ret) {
14807                 THERM_FAILURE(dd, ret, "Acquire SBus");
14808                 return ret;
14809         }
14810
14811         dd_dev_info(dd, "Initializing thermal sensor\n");
14812         /* Disable polling of thermal readings */
14813         write_csr(dd, ASIC_CFG_THERM_POLL_EN, 0x0);
14814         msleep(100);
14815         /* Thermal Sensor Initialization */
14816         /*    Step 1: Reset the Thermal SBus Receiver */
14817         ret = sbus_request_slow(dd, SBUS_THERMAL, 0x0,
14818                                 RESET_SBUS_RECEIVER, 0);
14819         if (ret) {
14820                 THERM_FAILURE(dd, ret, "Bus Reset");
14821                 goto done;
14822         }
14823         /*    Step 2: Set Reset bit in Thermal block */
14824         ret = sbus_request_slow(dd, SBUS_THERMAL, 0x0,
14825                                 WRITE_SBUS_RECEIVER, 0x1);
14826         if (ret) {
14827                 THERM_FAILURE(dd, ret, "Therm Block Reset");
14828                 goto done;
14829         }
14830         /*    Step 3: Write clock divider value (100MHz -> 2MHz) */
14831         ret = sbus_request_slow(dd, SBUS_THERMAL, 0x1,
14832                                 WRITE_SBUS_RECEIVER, 0x32);
14833         if (ret) {
14834                 THERM_FAILURE(dd, ret, "Write Clock Div");
14835                 goto done;
14836         }
14837         /*    Step 4: Select temperature mode */
14838         ret = sbus_request_slow(dd, SBUS_THERMAL, 0x3,
14839                                 WRITE_SBUS_RECEIVER,
14840                                 SBUS_THERM_MONITOR_MODE);
14841         if (ret) {
14842                 THERM_FAILURE(dd, ret, "Write Mode Sel");
14843                 goto done;
14844         }
14845         /*    Step 5: De-assert block reset and start conversion */
14846         ret = sbus_request_slow(dd, SBUS_THERMAL, 0x0,
14847                                 WRITE_SBUS_RECEIVER, 0x2);
14848         if (ret) {
14849                 THERM_FAILURE(dd, ret, "Write Reset Deassert");
14850                 goto done;
14851         }
14852         /*    Step 5.1: Wait for first conversion (21.5ms per spec) */
14853         msleep(22);
14854
14855         /* Enable polling of thermal readings */
14856         write_csr(dd, ASIC_CFG_THERM_POLL_EN, 0x1);
14857
14858         /* Set initialized flag */
14859         ret = acquire_chip_resource(dd, CR_THERM_INIT, 0);
14860         if (ret)
14861                 THERM_FAILURE(dd, ret, "Unable to set thermal init flag");
14862
14863 done:
14864         release_chip_resource(dd, CR_SBUS);
14865         return ret;
14866 }
14867
14868 static void handle_temp_err(struct hfi1_devdata *dd)
14869 {
14870         struct hfi1_pportdata *ppd = &dd->pport[0];
14871         /*
14872          * Thermal Critical Interrupt
14873          * Put the device into forced freeze mode, take link down to
14874          * offline, and put DC into reset.
14875          */
14876         dd_dev_emerg(dd,
14877                      "Critical temperature reached! Forcing device into freeze mode!\n");
14878         dd->flags |= HFI1_FORCED_FREEZE;
14879         start_freeze_handling(ppd, FREEZE_SELF | FREEZE_ABORT);
14880         /*
14881          * Shut DC down as much and as quickly as possible.
14882          *
14883          * Step 1: Take the link down to OFFLINE. This will cause the
14884          *         8051 to put the Serdes in reset. However, we don't want to
14885          *         go through the entire link state machine since we want to
14886          *         shutdown ASAP. Furthermore, this is not a graceful shutdown
14887          *         but rather an attempt to save the chip.
14888          *         Code below is almost the same as quiet_serdes() but avoids
14889          *         all the extra work and the sleeps.
14890          */
14891         ppd->driver_link_ready = 0;
14892         ppd->link_enabled = 0;
14893         set_physical_link_state(dd, (OPA_LINKDOWN_REASON_SMA_DISABLED << 8) |
14894                                 PLS_OFFLINE);
14895         /*
14896          * Step 2: Shutdown LCB and 8051
14897          *         After shutdown, do not restore DC_CFG_RESET value.
14898          */
14899         dc_shutdown(dd);
14900 }