drivers/net/ethernet/chelsio/cxgb3/cxgb3_main.c

   1 /*
   2  * Copyright (c) 2003-2008 Chelsio, Inc. All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32
  33 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  34
  35 #include <linux/module.h>
  36 #include <linux/moduleparam.h>
  37 #include <linux/init.h>
  38 #include <linux/pci.h>
  39 #include <linux/dma-mapping.h>
  40 #include <linux/netdevice.h>
  41 #include <linux/etherdevice.h>
  42 #include <linux/if_vlan.h>
  43 #include <linux/mdio.h>
  44 #include <linux/sockios.h>
  45 #include <linux/workqueue.h>
  46 #include <linux/proc_fs.h>
  47 #include <linux/rtnetlink.h>
  48 #include <linux/firmware.h>
  49 #include <linux/log2.h>
  50 #include <linux/stringify.h>
  51 #include <linux/sched.h>
  52 #include <linux/slab.h>
  53 #include <linux/nospec.h>
  54 #include <asm/uaccess.h>
  55
  56 #include "common.h"
  57 #include "cxgb3_ioctl.h"
  58 #include "regs.h"
  59 #include "cxgb3_offload.h"
  60 #include "version.h"
  61
  62 #include "cxgb3_ctl_defs.h"
  63 #include "t3_cpl.h"
  64 #include "firmware_exports.h"
  65
  66 enum {
  67         MAX_TXQ_ENTRIES = 16384,
  68         MAX_CTRL_TXQ_ENTRIES = 1024,
  69         MAX_RSPQ_ENTRIES = 16384,
  70         MAX_RX_BUFFERS = 16384,
  71         MAX_RX_JUMBO_BUFFERS = 16384,
  72         MIN_TXQ_ENTRIES = 4,
  73         MIN_CTRL_TXQ_ENTRIES = 4,
  74         MIN_RSPQ_ENTRIES = 32,
  75         MIN_FL_ENTRIES = 32
  76 };
  77
  78 #define PORT_MASK ((1 << MAX_NPORTS) - 1)
  79
  80 #define DFLT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK | \
  81                          NETIF_MSG_TIMER | NETIF_MSG_IFDOWN | NETIF_MSG_IFUP |\
  82                          NETIF_MSG_RX_ERR | NETIF_MSG_TX_ERR)
  83
  84 #define EEPROM_MAGIC 0x38E2F10C
  85
  86 #define CH_DEVICE(devid, idx) \
  87         { PCI_VENDOR_ID_CHELSIO, devid, PCI_ANY_ID, PCI_ANY_ID, 0, 0, idx }
  88
  89 static const struct pci_device_id cxgb3_pci_tbl[] = {
  90         CH_DEVICE(0x20, 0),     /* PE9000 */
  91         CH_DEVICE(0x21, 1),     /* T302E */
  92         CH_DEVICE(0x22, 2),     /* T310E */
  93         CH_DEVICE(0x23, 3),     /* T320X */
  94         CH_DEVICE(0x24, 1),     /* T302X */
  95         CH_DEVICE(0x25, 3),     /* T320E */
  96         CH_DEVICE(0x26, 2),     /* T310X */
  97         CH_DEVICE(0x30, 2),     /* T3B10 */
  98         CH_DEVICE(0x31, 3),     /* T3B20 */
  99         CH_DEVICE(0x32, 1),     /* T3B02 */
 100         CH_DEVICE(0x35, 6),     /* T3C20-derived T3C10 */
 101         CH_DEVICE(0x36, 3),     /* S320E-CR */
 102         CH_DEVICE(0x37, 7),     /* N320E-G2 */
 103         {0,}
 104 };
 105
 106 MODULE_DESCRIPTION(DRV_DESC);
 107 MODULE_AUTHOR("Chelsio Communications");
 108 MODULE_LICENSE("Dual BSD/GPL");
 109 MODULE_VERSION(DRV_VERSION);
 110 MODULE_DEVICE_TABLE(pci, cxgb3_pci_tbl);
 111
 112 static int dflt_msg_enable = DFLT_MSG_ENABLE;
 113
 114 module_param(dflt_msg_enable, int, 0644);
 115 MODULE_PARM_DESC(dflt_msg_enable, "Chelsio T3 default message enable bitmap");
 116
 117 /*
 118  * The driver uses the best interrupt scheme available on a platform in the
 119  * order MSI-X, MSI, legacy pin interrupts.  This parameter determines which
 120  * of these schemes the driver may consider as follows:
 121  *
 122  * msi = 2: choose from among all three options
 123  * msi = 1: only consider MSI and pin interrupts
 124  * msi = 0: force pin interrupts
 125  */
 126 static int msi = 2;
 127
 128 module_param(msi, int, 0644);
 129 MODULE_PARM_DESC(msi, "whether to use MSI or MSI-X");
 130
 131 /*
 132  * The driver enables offload as a default.
 133  * To disable it, use ofld_disable = 1.
 134  */
 135
 136 static int ofld_disable = 0;
 137
 138 module_param(ofld_disable, int, 0644);
 139 MODULE_PARM_DESC(ofld_disable, "whether to enable offload at init time or not");
 140
 141 /*
 142  * We have work elements that we need to cancel when an interface is taken
 143  * down.  Normally the work elements would be executed by keventd but that
 144  * can deadlock because of linkwatch.  If our close method takes the rtnl
 145  * lock and linkwatch is ahead of our work elements in keventd, linkwatch
 146  * will block keventd as it needs the rtnl lock, and we'll deadlock waiting
 147  * for our work to complete.  Get our own work queue to solve this.
 148  */
 149 struct workqueue_struct *cxgb3_wq;
 150
 151 /**
 152  *      link_report - show link status and link speed/duplex
 153  *      @p: the port whose settings are to be reported
 154  *
 155  *      Shows the link status, speed, and duplex of a port.
 156  */
 157 static void link_report(struct net_device *dev)
 158 {
 159         if (!netif_carrier_ok(dev))
 160                 netdev_info(dev, "link down\n");
 161         else {
 162                 const char *s = "10Mbps";
 163                 const struct port_info *p = netdev_priv(dev);
 164
 165                 switch (p->link_config.speed) {
 166                 case SPEED_10000:
 167                         s = "10Gbps";
 168                         break;
 169                 case SPEED_1000:
 170                         s = "1000Mbps";
 171                         break;
 172                 case SPEED_100:
 173                         s = "100Mbps";
 174                         break;
 175                 }
 176
 177                 netdev_info(dev, "link up, %s, %s-duplex\n",
 178                             s, p->link_config.duplex == DUPLEX_FULL
 179                             ? "full" : "half");
 180         }
 181 }
 182
 183 static void enable_tx_fifo_drain(struct adapter *adapter,
 184                                  struct port_info *pi)
 185 {
 186         t3_set_reg_field(adapter, A_XGM_TXFIFO_CFG + pi->mac.offset, 0,
 187                          F_ENDROPPKT);
 188         t3_write_reg(adapter, A_XGM_RX_CTRL + pi->mac.offset, 0);
 189         t3_write_reg(adapter, A_XGM_TX_CTRL + pi->mac.offset, F_TXEN);
 190         t3_write_reg(adapter, A_XGM_RX_CTRL + pi->mac.offset, F_RXEN);
 191 }
 192
 193 static void disable_tx_fifo_drain(struct adapter *adapter,
 194                                   struct port_info *pi)
 195 {
 196         t3_set_reg_field(adapter, A_XGM_TXFIFO_CFG + pi->mac.offset,
 197                          F_ENDROPPKT, 0);
 198 }
 199
 200 void t3_os_link_fault(struct adapter *adap, int port_id, int state)
 201 {
 202         struct net_device *dev = adap->port[port_id];
 203         struct port_info *pi = netdev_priv(dev);
 204
 205         if (state == netif_carrier_ok(dev))
 206                 return;
 207
 208         if (state) {
 209                 struct cmac *mac = &pi->mac;
 210
 211                 netif_carrier_on(dev);
 212
 213                 disable_tx_fifo_drain(adap, pi);
 214
 215                 /* Clear local faults */
 216                 t3_xgm_intr_disable(adap, pi->port_id);
 217                 t3_read_reg(adap, A_XGM_INT_STATUS +
 218                                     pi->mac.offset);
 219                 t3_write_reg(adap,
 220                              A_XGM_INT_CAUSE + pi->mac.offset,
 221                              F_XGM_INT);
 222
 223                 t3_set_reg_field(adap,
 224                                  A_XGM_INT_ENABLE +
 225                                  pi->mac.offset,
 226                                  F_XGM_INT, F_XGM_INT);
 227                 t3_xgm_intr_enable(adap, pi->port_id);
 228
 229                 t3_mac_enable(mac, MAC_DIRECTION_TX);
 230         } else {
 231                 netif_carrier_off(dev);
 232
 233                 /* Flush TX FIFO */
 234                 enable_tx_fifo_drain(adap, pi);
 235         }
 236         link_report(dev);
 237 }
 238
 239 /**
 240  *      t3_os_link_changed - handle link status changes
 241  *      @adapter: the adapter associated with the link change
 242  *      @port_id: the port index whose limk status has changed
 243  *      @link_stat: the new status of the link
 244  *      @speed: the new speed setting
 245  *      @duplex: the new duplex setting
 246  *      @pause: the new flow-control setting
 247  *
 248  *      This is the OS-dependent handler for link status changes.  The OS
 249  *      neutral handler takes care of most of the processing for these events,
 250  *      then calls this handler for any OS-specific processing.
 251  */
 252 void t3_os_link_changed(struct adapter *adapter, int port_id, int link_stat,
 253                         int speed, int duplex, int pause)
 254 {
 255         struct net_device *dev = adapter->port[port_id];
 256         struct port_info *pi = netdev_priv(dev);
 257         struct cmac *mac = &pi->mac;
 258
 259         /* Skip changes from disabled ports. */
 260         if (!netif_running(dev))
 261                 return;
 262
 263         if (link_stat != netif_carrier_ok(dev)) {
 264                 if (link_stat) {
 265                         disable_tx_fifo_drain(adapter, pi);
 266
 267                         t3_mac_enable(mac, MAC_DIRECTION_RX);
 268
 269                         /* Clear local faults */
 270                         t3_xgm_intr_disable(adapter, pi->port_id);
 271                         t3_read_reg(adapter, A_XGM_INT_STATUS +
 272                                     pi->mac.offset);
 273                         t3_write_reg(adapter,
 274                                      A_XGM_INT_CAUSE + pi->mac.offset,
 275                                      F_XGM_INT);
 276
 277                         t3_set_reg_field(adapter,
 278                                          A_XGM_INT_ENABLE + pi->mac.offset,
 279                                          F_XGM_INT, F_XGM_INT);
 280                         t3_xgm_intr_enable(adapter, pi->port_id);
 281
 282                         netif_carrier_on(dev);
 283                 } else {
 284                         netif_carrier_off(dev);
 285
 286                         t3_xgm_intr_disable(adapter, pi->port_id);
 287                         t3_read_reg(adapter, A_XGM_INT_STATUS + pi->mac.offset);
 288                         t3_set_reg_field(adapter,
 289                                          A_XGM_INT_ENABLE + pi->mac.offset,
 290                                          F_XGM_INT, 0);
 291
 292                         if (is_10G(adapter))
 293                                 pi->phy.ops->power_down(&pi->phy, 1);
 294
 295                         t3_read_reg(adapter, A_XGM_INT_STATUS + pi->mac.offset);
 296                         t3_mac_disable(mac, MAC_DIRECTION_RX);
 297                         t3_link_start(&pi->phy, mac, &pi->link_config);
 298
 299                         /* Flush TX FIFO */
 300                         enable_tx_fifo_drain(adapter, pi);
 301                 }
 302
 303                 link_report(dev);
 304         }
 305 }
 306
 307 /**
 308  *      t3_os_phymod_changed - handle PHY module changes
 309  *      @phy: the PHY reporting the module change
 310  *      @mod_type: new module type
 311  *
 312  *      This is the OS-dependent handler for PHY module changes.  It is
 313  *      invoked when a PHY module is removed or inserted for any OS-specific
 314  *      processing.
 315  */
 316 void t3_os_phymod_changed(struct adapter *adap, int port_id)
 317 {
 318         static const char *mod_str[] = {
 319                 NULL, "SR", "LR", "LRM", "TWINAX", "TWINAX", "unknown"
 320         };
 321
 322         const struct net_device *dev = adap->port[port_id];
 323         const struct port_info *pi = netdev_priv(dev);
 324
 325         if (pi->phy.modtype == phy_modtype_none)
 326                 netdev_info(dev, "PHY module unplugged\n");
 327         else
 328                 netdev_info(dev, "%s PHY module inserted\n",
 329                             mod_str[pi->phy.modtype]);
 330 }
 331
 332 static void cxgb_set_rxmode(struct net_device *dev)
 333 {
 334         struct port_info *pi = netdev_priv(dev);
 335
 336         t3_mac_set_rx_mode(&pi->mac, dev);
 337 }
 338
 339 /**
 340  *      link_start - enable a port
 341  *      @dev: the device to enable
 342  *
 343  *      Performs the MAC and PHY actions needed to enable a port.
 344  */
 345 static void link_start(struct net_device *dev)
 346 {
 347         struct port_info *pi = netdev_priv(dev);
 348         struct cmac *mac = &pi->mac;
 349
 350         t3_mac_reset(mac);
 351         t3_mac_set_num_ucast(mac, MAX_MAC_IDX);
 352         t3_mac_set_mtu(mac, dev->mtu);
 353         t3_mac_set_address(mac, LAN_MAC_IDX, dev->dev_addr);
 354         t3_mac_set_address(mac, SAN_MAC_IDX, pi->iscsic.mac_addr);
 355         t3_mac_set_rx_mode(mac, dev);
 356         t3_link_start(&pi->phy, mac, &pi->link_config);
 357         t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX);
 358 }
 359
 360 static inline void cxgb_disable_msi(struct adapter *adapter)
 361 {
 362         if (adapter->flags & USING_MSIX) {
 363                 pci_disable_msix(adapter->pdev);
 364                 adapter->flags &= ~USING_MSIX;
 365         } else if (adapter->flags & USING_MSI) {
 366                 pci_disable_msi(adapter->pdev);
 367                 adapter->flags &= ~USING_MSI;
 368         }
 369 }
 370
 371 /*
 372  * Interrupt handler for asynchronous events used with MSI-X.
 373  */
 374 static irqreturn_t t3_async_intr_handler(int irq, void *cookie)
 375 {
 376         t3_slow_intr_handler(cookie);
 377         return IRQ_HANDLED;
 378 }
 379
 380 /*
 381  * Name the MSI-X interrupts.
 382  */
 383 static void name_msix_vecs(struct adapter *adap)
 384 {
 385         int i, j, msi_idx = 1, n = sizeof(adap->msix_info[0].desc) - 1;
 386
 387         snprintf(adap->msix_info[0].desc, n, "%s", adap->name);
 388         adap->msix_info[0].desc[n] = 0;
 389
 390         for_each_port(adap, j) {
 391                 struct net_device *d = adap->port[j];
 392                 const struct port_info *pi = netdev_priv(d);
 393
 394                 for (i = 0; i < pi->nqsets; i++, msi_idx++) {
 395                         snprintf(adap->msix_info[msi_idx].desc, n,
 396                                  "%s-%d", d->name, pi->first_qset + i);
 397                         adap->msix_info[msi_idx].desc[n] = 0;
 398                 }
 399         }
 400 }
 401
 402 static int request_msix_data_irqs(struct adapter *adap)
 403 {
 404         int i, j, err, qidx = 0;
 405
 406         for_each_port(adap, i) {
 407                 int nqsets = adap2pinfo(adap, i)->nqsets;
 408
 409                 for (j = 0; j < nqsets; ++j) {
 410                         err = request_irq(adap->msix_info[qidx + 1].vec,
 411                                           t3_intr_handler(adap,
 412                                                           adap->sge.qs[qidx].
 413                                                           rspq.polling), 0,
 414                                           adap->msix_info[qidx + 1].desc,
 415                                           &adap->sge.qs[qidx]);
 416                         if (err) {
 417                                 while (--qidx >= 0)
 418                                         free_irq(adap->msix_info[qidx + 1].vec,
 419                                                  &adap->sge.qs[qidx]);
 420                                 return err;
 421                         }
 422                         qidx++;
 423                 }
 424         }
 425         return 0;
 426 }
 427
 428 static void free_irq_resources(struct adapter *adapter)
 429 {
 430         if (adapter->flags & USING_MSIX) {
 431                 int i, n = 0;
 432
 433                 free_irq(adapter->msix_info[0].vec, adapter);
 434                 for_each_port(adapter, i)
 435                         n += adap2pinfo(adapter, i)->nqsets;
 436
 437                 for (i = 0; i < n; ++i)
 438                         free_irq(adapter->msix_info[i + 1].vec,
 439                                  &adapter->sge.qs[i]);
 440         } else
 441                 free_irq(adapter->pdev->irq, adapter);
 442 }
 443
 444 static int await_mgmt_replies(struct adapter *adap, unsigned long init_cnt,
 445                               unsigned long n)
 446 {
 447         int attempts = 10;
 448
 449         while (adap->sge.qs[0].rspq.offload_pkts < init_cnt + n) {
 450                 if (!--attempts)
 451                         return -ETIMEDOUT;
 452                 msleep(10);
 453         }
 454         return 0;
 455 }
 456
 457 static int init_tp_parity(struct adapter *adap)
 458 {
 459         int i;
 460         struct sk_buff *skb;
 461         struct cpl_set_tcb_field *greq;
 462         unsigned long cnt = adap->sge.qs[0].rspq.offload_pkts;
 463
 464         t3_tp_set_offload_mode(adap, 1);
 465
 466         for (i = 0; i < 16; i++) {
 467                 struct cpl_smt_write_req *req;
 468
 469                 skb = alloc_skb(sizeof(*req), GFP_KERNEL);
 470                 if (!skb)
 471                         skb = adap->nofail_skb;
 472                 if (!skb)
 473                         goto alloc_skb_fail;
 474
 475                 req = (struct cpl_smt_write_req *)__skb_put(skb, sizeof(*req));
 476                 memset(req, 0, sizeof(*req));
 477                 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
 478                 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SMT_WRITE_REQ, i));
 479                 req->mtu_idx = NMTUS - 1;
 480                 req->iff = i;
 481                 t3_mgmt_tx(adap, skb);
 482                 if (skb == adap->nofail_skb) {
 483                         await_mgmt_replies(adap, cnt, i + 1);
 484                         adap->nofail_skb = alloc_skb(sizeof(*greq), GFP_KERNEL);
 485                         if (!adap->nofail_skb)
 486                                 goto alloc_skb_fail;
 487                 }
 488         }
 489
 490         for (i = 0; i < 2048; i++) {
 491                 struct cpl_l2t_write_req *req;
 492
 493                 skb = alloc_skb(sizeof(*req), GFP_KERNEL);
 494                 if (!skb)
 495                         skb = adap->nofail_skb;
 496                 if (!skb)
 497                         goto alloc_skb_fail;
 498
 499                 req = (struct cpl_l2t_write_req *)__skb_put(skb, sizeof(*req));
 500                 memset(req, 0, sizeof(*req));
 501                 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
 502                 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_L2T_WRITE_REQ, i));
 503                 req->params = htonl(V_L2T_W_IDX(i));
 504                 t3_mgmt_tx(adap, skb);
 505                 if (skb == adap->nofail_skb) {
 506                         await_mgmt_replies(adap, cnt, 16 + i + 1);
 507                         adap->nofail_skb = alloc_skb(sizeof(*greq), GFP_KERNEL);
 508                         if (!adap->nofail_skb)
 509                                 goto alloc_skb_fail;
 510                 }
 511         }
 512
 513         for (i = 0; i < 2048; i++) {
 514                 struct cpl_rte_write_req *req;
 515
 516                 skb = alloc_skb(sizeof(*req), GFP_KERNEL);
 517                 if (!skb)
 518                         skb = adap->nofail_skb;
 519                 if (!skb)
 520                         goto alloc_skb_fail;
 521
 522                 req = (struct cpl_rte_write_req *)__skb_put(skb, sizeof(*req));
 523                 memset(req, 0, sizeof(*req));
 524                 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
 525                 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_RTE_WRITE_REQ, i));
 526                 req->l2t_idx = htonl(V_L2T_W_IDX(i));
 527                 t3_mgmt_tx(adap, skb);
 528                 if (skb == adap->nofail_skb) {
 529                         await_mgmt_replies(adap, cnt, 16 + 2048 + i + 1);
 530                         adap->nofail_skb = alloc_skb(sizeof(*greq), GFP_KERNEL);
 531                         if (!adap->nofail_skb)
 532                                 goto alloc_skb_fail;
 533                 }
 534         }
 535
 536         skb = alloc_skb(sizeof(*greq), GFP_KERNEL);
 537         if (!skb)
 538                 skb = adap->nofail_skb;
 539         if (!skb)
 540                 goto alloc_skb_fail;
 541
 542         greq = (struct cpl_set_tcb_field *)__skb_put(skb, sizeof(*greq));
 543         memset(greq, 0, sizeof(*greq));
 544         greq->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
 545         OPCODE_TID(greq) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, 0));
 546         greq->mask = cpu_to_be64(1);
 547         t3_mgmt_tx(adap, skb);
 548
 549         i = await_mgmt_replies(adap, cnt, 16 + 2048 + 2048 + 1);
 550         if (skb == adap->nofail_skb) {
 551                 i = await_mgmt_replies(adap, cnt, 16 + 2048 + 2048 + 1);
 552                 adap->nofail_skb = alloc_skb(sizeof(*greq), GFP_KERNEL);
 553         }
 554
 555         t3_tp_set_offload_mode(adap, 0);
 556         return i;
 557
 558 alloc_skb_fail:
 559         t3_tp_set_offload_mode(adap, 0);
 560         return -ENOMEM;
 561 }
 562
 563 /**
 564  *      setup_rss - configure RSS
 565  *      @adap: the adapter
 566  *
 567  *      Sets up RSS to distribute packets to multiple receive queues.  We
 568  *      configure the RSS CPU lookup table to distribute to the number of HW
 569  *      receive queues, and the response queue lookup table to narrow that
 570  *      down to the response queues actually configured for each port.
 571  *      We always configure the RSS mapping for two ports since the mapping
 572  *      table has plenty of entries.
 573  */
 574 static void setup_rss(struct adapter *adap)
 575 {
 576         int i;
 577         unsigned int nq0 = adap2pinfo(adap, 0)->nqsets;
 578         unsigned int nq1 = adap->port[1] ? adap2pinfo(adap, 1)->nqsets : 1;
 579         u8 cpus[SGE_QSETS + 1];
 580         u16 rspq_map[RSS_TABLE_SIZE];
 581
 582         for (i = 0; i < SGE_QSETS; ++i)
 583                 cpus[i] = i;
 584         cpus[SGE_QSETS] = 0xff; /* terminator */
 585
 586         for (i = 0; i < RSS_TABLE_SIZE / 2; ++i) {
 587                 rspq_map[i] = i % nq0;
 588                 rspq_map[i + RSS_TABLE_SIZE / 2] = (i % nq1) + nq0;
 589         }
 590
 591         t3_config_rss(adap, F_RQFEEDBACKENABLE | F_TNLLKPEN | F_TNLMAPEN |
 592                       F_TNLPRTEN | F_TNL2TUPEN | F_TNL4TUPEN |
 593                       V_RRCPLCPUSIZE(6) | F_HASHTOEPLITZ, cpus, rspq_map);
 594 }
 595
 596 static void ring_dbs(struct adapter *adap)
 597 {
 598         int i, j;
 599
 600         for (i = 0; i < SGE_QSETS; i++) {
 601                 struct sge_qset *qs = &adap->sge.qs[i];
 602
 603                 if (qs->adap)
 604                         for (j = 0; j < SGE_TXQ_PER_SET; j++)
 605                                 t3_write_reg(adap, A_SG_KDOORBELL, F_SELEGRCNTX | V_EGRCNTX(qs->txq[j].cntxt_id));
 606         }
 607 }
 608
 609 static void init_napi(struct adapter *adap)
 610 {
 611         int i;
 612
 613         for (i = 0; i < SGE_QSETS; i++) {
 614                 struct sge_qset *qs = &adap->sge.qs[i];
 615
 616                 if (qs->adap)
 617                         netif_napi_add(qs->netdev, &qs->napi, qs->napi.poll,
 618                                        64);
 619         }
 620
 621         /*
 622          * netif_napi_add() can be called only once per napi_struct because it
 623          * adds each new napi_struct to a list.  Be careful not to call it a
 624          * second time, e.g., during EEH recovery, by making a note of it.
 625          */
 626         adap->flags |= NAPI_INIT;
 627 }
 628
 629 /*
 630  * Wait until all NAPI handlers are descheduled.  This includes the handlers of
 631  * both netdevices representing interfaces and the dummy ones for the extra
 632  * queues.
 633  */
 634 static void quiesce_rx(struct adapter *adap)
 635 {
 636         int i;
 637
 638         for (i = 0; i < SGE_QSETS; i++)
 639                 if (adap->sge.qs[i].adap)
 640                         napi_disable(&adap->sge.qs[i].napi);
 641 }
 642
 643 static void enable_all_napi(struct adapter *adap)
 644 {
 645         int i;
 646         for (i = 0; i < SGE_QSETS; i++)
 647                 if (adap->sge.qs[i].adap)
 648                         napi_enable(&adap->sge.qs[i].napi);
 649 }
 650
 651 /**
 652  *      setup_sge_qsets - configure SGE Tx/Rx/response queues
 653  *      @adap: the adapter
 654  *
 655  *      Determines how many sets of SGE queues to use and initializes them.
 656  *      We support multiple queue sets per port if we have MSI-X, otherwise
 657  *      just one queue set per port.
 658  */
 659 static int setup_sge_qsets(struct adapter *adap)
 660 {
 661         int i, j, err, irq_idx = 0, qset_idx = 0;
 662         unsigned int ntxq = SGE_TXQ_PER_SET;
 663
 664         if (adap->params.rev > 0 && !(adap->flags & USING_MSI))
 665                 irq_idx = -1;
 666
 667         for_each_port(adap, i) {
 668                 struct net_device *dev = adap->port[i];
 669                 struct port_info *pi = netdev_priv(dev);
 670
 671                 pi->qs = &adap->sge.qs[pi->first_qset];
 672                 for (j = 0; j < pi->nqsets; ++j, ++qset_idx) {
 673                         err = t3_sge_alloc_qset(adap, qset_idx, 1,
 674                                 (adap->flags & USING_MSIX) ? qset_idx + 1 :
 675                                                              irq_idx,
 676                                 &adap->params.sge.qset[qset_idx], ntxq, dev,
 677                                 netdev_get_tx_queue(dev, j));
 678                         if (err) {
 679                                 t3_free_sge_resources(adap);
 680                                 return err;
 681                         }
 682                 }
 683         }
 684
 685         return 0;
 686 }
 687
 688 static ssize_t attr_show(struct device *d, char *buf,
 689                          ssize_t(*format) (struct net_device *, char *))
 690 {
 691         ssize_t len;
 692
 693         /* Synchronize with ioctls that may shut down the device */
 694         rtnl_lock();
 695         len = (*format) (to_net_dev(d), buf);
 696         rtnl_unlock();
 697         return len;
 698 }
 699
 700 static ssize_t attr_store(struct device *d,
 701                           const char *buf, size_t len,
 702                           ssize_t(*set) (struct net_device *, unsigned int),
 703                           unsigned int min_val, unsigned int max_val)
 704 {
 705         char *endp;
 706         ssize_t ret;
 707         unsigned int val;
 708
 709         if (!capable(CAP_NET_ADMIN))
 710                 return -EPERM;
 711
 712         val = simple_strtoul(buf, &endp, 0);
 713         if (endp == buf || val < min_val || val > max_val)
 714                 return -EINVAL;
 715
 716         rtnl_lock();
 717         ret = (*set) (to_net_dev(d), val);
 718         if (!ret)
 719                 ret = len;
 720         rtnl_unlock();
 721         return ret;
 722 }
 723
 724 #define CXGB3_SHOW(name, val_expr) \
 725 static ssize_t format_##name(struct net_device *dev, char *buf) \
 726 { \
 727         struct port_info *pi = netdev_priv(dev); \
 728         struct adapter *adap = pi->adapter; \
 729         return sprintf(buf, "%u\n", val_expr); \
 730 } \
 731 static ssize_t show_##name(struct device *d, struct device_attribute *attr, \
 732                            char *buf) \
 733 { \
 734         return attr_show(d, buf, format_##name); \
 735 }
 736
 737 static ssize_t set_nfilters(struct net_device *dev, unsigned int val)
 738 {
 739         struct port_info *pi = netdev_priv(dev);
 740         struct adapter *adap = pi->adapter;
 741         int min_tids = is_offload(adap) ? MC5_MIN_TIDS : 0;
 742
 743         if (adap->flags & FULL_INIT_DONE)
 744                 return -EBUSY;
 745         if (val && adap->params.rev == 0)
 746                 return -EINVAL;
 747         if (val > t3_mc5_size(&adap->mc5) - adap->params.mc5.nservers -
 748             min_tids)
 749                 return -EINVAL;
 750         adap->params.mc5.nfilters = val;
 751         return 0;
 752 }
 753
 754 static ssize_t store_nfilters(struct device *d, struct device_attribute *attr,
 755                               const char *buf, size_t len)
 756 {
 757         return attr_store(d, buf, len, set_nfilters, 0, ~0);
 758 }
 759
 760 static ssize_t set_nservers(struct net_device *dev, unsigned int val)
 761 {
 762         struct port_info *pi = netdev_priv(dev);
 763         struct adapter *adap = pi->adapter;
 764
 765         if (adap->flags & FULL_INIT_DONE)
 766                 return -EBUSY;
 767         if (val > t3_mc5_size(&adap->mc5) - adap->params.mc5.nfilters -
 768             MC5_MIN_TIDS)
 769                 return -EINVAL;
 770         adap->params.mc5.nservers = val;
 771         return 0;
 772 }
 773
 774 static ssize_t store_nservers(struct device *d, struct device_attribute *attr,
 775                               const char *buf, size_t len)
 776 {
 777         return attr_store(d, buf, len, set_nservers, 0, ~0);
 778 }
 779
 780 #define CXGB3_ATTR_R(name, val_expr) \
 781 CXGB3_SHOW(name, val_expr) \
 782 static DEVICE_ATTR(name, S_IRUGO, show_##name, NULL)
 783
 784 #define CXGB3_ATTR_RW(name, val_expr, store_method) \
 785 CXGB3_SHOW(name, val_expr) \
 786 static DEVICE_ATTR(name, S_IRUGO | S_IWUSR, show_##name, store_method)
 787
 788 CXGB3_ATTR_R(cam_size, t3_mc5_size(&adap->mc5));
 789 CXGB3_ATTR_RW(nfilters, adap->params.mc5.nfilters, store_nfilters);
 790 CXGB3_ATTR_RW(nservers, adap->params.mc5.nservers, store_nservers);
 791
 792 static struct attribute *cxgb3_attrs[] = {
 793         &dev_attr_cam_size.attr,
 794         &dev_attr_nfilters.attr,
 795         &dev_attr_nservers.attr,
 796         NULL
 797 };
 798
 799 static struct attribute_group cxgb3_attr_group = {.attrs = cxgb3_attrs };
 800
 801 static ssize_t tm_attr_show(struct device *d,
 802                             char *buf, int sched)
 803 {
 804         struct port_info *pi = netdev_priv(to_net_dev(d));
 805         struct adapter *adap = pi->adapter;
 806         unsigned int v, addr, bpt, cpt;
 807         ssize_t len;
 808
 809         addr = A_TP_TX_MOD_Q1_Q0_RATE_LIMIT - sched / 2;
 810         rtnl_lock();
 811         t3_write_reg(adap, A_TP_TM_PIO_ADDR, addr);
 812         v = t3_read_reg(adap, A_TP_TM_PIO_DATA);
 813         if (sched & 1)
 814                 v >>= 16;
 815         bpt = (v >> 8) & 0xff;
 816         cpt = v & 0xff;
 817         if (!cpt)
 818                 len = sprintf(buf, "disabled\n");
 819         else {
 820                 v = (adap->params.vpd.cclk * 1000) / cpt;
 821                 len = sprintf(buf, "%u Kbps\n", (v * bpt) / 125);
 822         }
 823         rtnl_unlock();
 824         return len;
 825 }
 826
 827 static ssize_t tm_attr_store(struct device *d,
 828                              const char *buf, size_t len, int sched)
 829 {
 830         struct port_info *pi = netdev_priv(to_net_dev(d));
 831         struct adapter *adap = pi->adapter;
 832         unsigned int val;
 833         char *endp;
 834         ssize_t ret;
 835
 836         if (!capable(CAP_NET_ADMIN))
 837                 return -EPERM;
 838
 839         val = simple_strtoul(buf, &endp, 0);
 840         if (endp == buf || val > 10000000)
 841                 return -EINVAL;
 842
 843         rtnl_lock();
 844         ret = t3_config_sched(adap, val, sched);
 845         if (!ret)
 846                 ret = len;
 847         rtnl_unlock();
 848         return ret;
 849 }
 850
 851 #define TM_ATTR(name, sched) \
 852 static ssize_t show_##name(struct device *d, struct device_attribute *attr, \
 853                            char *buf) \
 854 { \
 855         return tm_attr_show(d, buf, sched); \
 856 } \
 857 static ssize_t store_##name(struct device *d, struct device_attribute *attr, \
 858                             const char *buf, size_t len) \
 859 { \
 860         return tm_attr_store(d, buf, len, sched); \
 861 } \
 862 static DEVICE_ATTR(name, S_IRUGO | S_IWUSR, show_##name, store_##name)
 863
 864 TM_ATTR(sched0, 0);
 865 TM_ATTR(sched1, 1);
 866 TM_ATTR(sched2, 2);
 867 TM_ATTR(sched3, 3);
 868 TM_ATTR(sched4, 4);
 869 TM_ATTR(sched5, 5);
 870 TM_ATTR(sched6, 6);
 871 TM_ATTR(sched7, 7);
 872
 873 static struct attribute *offload_attrs[] = {
 874         &dev_attr_sched0.attr,
 875         &dev_attr_sched1.attr,
 876         &dev_attr_sched2.attr,
 877         &dev_attr_sched3.attr,
 878         &dev_attr_sched4.attr,
 879         &dev_attr_sched5.attr,
 880         &dev_attr_sched6.attr,
 881         &dev_attr_sched7.attr,
 882         NULL
 883 };
 884
 885 static struct attribute_group offload_attr_group = {.attrs = offload_attrs };
 886
 887 /*
 888  * Sends an sk_buff to an offload queue driver
 889  * after dealing with any active network taps.
 890  */
 891 static inline int offload_tx(struct t3cdev *tdev, struct sk_buff *skb)
 892 {
 893         int ret;
 894
 895         local_bh_disable();
 896         ret = t3_offload_tx(tdev, skb);
 897         local_bh_enable();
 898         return ret;
 899 }
 900
 901 static int write_smt_entry(struct adapter *adapter, int idx)
 902 {
 903         struct cpl_smt_write_req *req;
 904         struct port_info *pi = netdev_priv(adapter->port[idx]);
 905         struct sk_buff *skb = alloc_skb(sizeof(*req), GFP_KERNEL);
 906
 907         if (!skb)
 908                 return -ENOMEM;
 909
 910         req = (struct cpl_smt_write_req *)__skb_put(skb, sizeof(*req));
 911         req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
 912         OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SMT_WRITE_REQ, idx));
 913         req->mtu_idx = NMTUS - 1;       /* should be 0 but there's a T3 bug */
 914         req->iff = idx;
 915         memcpy(req->src_mac0, adapter->port[idx]->dev_addr, ETH_ALEN);
 916         memcpy(req->src_mac1, pi->iscsic.mac_addr, ETH_ALEN);
 917         skb->priority = 1;
 918         offload_tx(&adapter->tdev, skb);
 919         return 0;
 920 }
 921
 922 static int init_smt(struct adapter *adapter)
 923 {
 924         int i;
 925
 926         for_each_port(adapter, i)
 927             write_smt_entry(adapter, i);
 928         return 0;
 929 }
 930
 931 static void init_port_mtus(struct adapter *adapter)
 932 {
 933         unsigned int mtus = adapter->port[0]->mtu;
 934
 935         if (adapter->port[1])
 936                 mtus |= adapter->port[1]->mtu << 16;
 937         t3_write_reg(adapter, A_TP_MTU_PORT_TABLE, mtus);
 938 }
 939
 940 static int send_pktsched_cmd(struct adapter *adap, int sched, int qidx, int lo,
 941                               int hi, int port)
 942 {
 943         struct sk_buff *skb;
 944         struct mngt_pktsched_wr *req;
 945         int ret;
 946
 947         skb = alloc_skb(sizeof(*req), GFP_KERNEL);
 948         if (!skb)
 949                 skb = adap->nofail_skb;
 950         if (!skb)
 951                 return -ENOMEM;
 952
 953         req = (struct mngt_pktsched_wr *)skb_put(skb, sizeof(*req));
 954         req->wr_hi = htonl(V_WR_OP(FW_WROPCODE_MNGT));
 955         req->mngt_opcode = FW_MNGTOPCODE_PKTSCHED_SET;
 956         req->sched = sched;
 957         req->idx = qidx;
 958         req->min = lo;
 959         req->max = hi;
 960         req->binding = port;
 961         ret = t3_mgmt_tx(adap, skb);
 962         if (skb == adap->nofail_skb) {
 963                 adap->nofail_skb = alloc_skb(sizeof(struct cpl_set_tcb_field),
 964                                              GFP_KERNEL);
 965                 if (!adap->nofail_skb)
 966                         ret = -ENOMEM;
 967         }
 968
 969         return ret;
 970 }
 971
 972 static int bind_qsets(struct adapter *adap)
 973 {
 974         int i, j, err = 0;
 975
 976         for_each_port(adap, i) {
 977                 const struct port_info *pi = adap2pinfo(adap, i);
 978
 979                 for (j = 0; j < pi->nqsets; ++j) {
 980                         int ret = send_pktsched_cmd(adap, 1,
 981                                                     pi->first_qset + j, -1,
 982                                                     -1, i);
 983                         if (ret)
 984                                 err = ret;
 985                 }
 986         }
 987
 988         return err;
 989 }
 990
 991 /*(DEBLOBBED)*/
 992 #define FW_FNAME "/*(DEBLOBBED)*/"
 993 /*(DEBLOBBED)*/
 994 #define TPSRAM_NAME "/*(DEBLOBBED)*/"
 995 #define AEL2005_OPT_EDC_NAME "/*(DEBLOBBED)*/"
 996 #define AEL2005_TWX_EDC_NAME "/*(DEBLOBBED)*/"
 997 #define AEL2020_TWX_EDC_NAME "/*(DEBLOBBED)*/"
 998 /*(DEBLOBBED)*/
 999
1000 static inline const char *get_edc_fw_name(int edc_idx)
1001 {
1002         const char *fw_name = NULL;
1003
1004         switch (edc_idx) {
1005         case EDC_OPT_AEL2005:
1006                 fw_name = AEL2005_OPT_EDC_NAME;
1007                 break;
1008         case EDC_TWX_AEL2005:
1009                 fw_name = AEL2005_TWX_EDC_NAME;
1010                 break;
1011         case EDC_TWX_AEL2020:
1012                 fw_name = AEL2020_TWX_EDC_NAME;
1013                 break;
1014         }
1015         return fw_name;
1016 }
1017
1018 int t3_get_edc_fw(struct cphy *phy, int edc_idx, int size)
1019 {
1020         struct adapter *adapter = phy->adapter;
1021         const struct firmware *fw;
1022         const char *fw_name;
1023         u32 csum;
1024         const __be32 *p;
1025         u16 *cache = phy->phy_cache;
1026         int i, ret = -EINVAL;
1027
1028         fw_name = get_edc_fw_name(edc_idx);
1029         if (fw_name)
1030                 ret = reject_firmware(&fw, fw_name, &adapter->pdev->dev);
1031         if (ret < 0) {
1032                 dev_err(&adapter->pdev->dev,
1033                         "could not upgrade firmware: unable to load %s\n",
1034                         fw_name);
1035                 return ret;
1036         }
1037
1038         /* check size, take checksum in account */
1039         if (fw->size > size + 4) {
1040                 CH_ERR(adapter, "firmware image too large %u, expected %d\n",
1041                        (unsigned int)fw->size, size + 4);
1042                 ret = -EINVAL;
1043         }
1044
1045         /* compute checksum */
1046         p = (const __be32 *)fw->data;
1047         for (csum = 0, i = 0; i < fw->size / sizeof(csum); i++)
1048                 csum += ntohl(p[i]);
1049
1050         if (csum != 0xffffffff) {
1051                 CH_ERR(adapter, "corrupted firmware image, checksum %u\n",
1052                        csum);
1053                 ret = -EINVAL;
1054         }
1055
1056         for (i = 0; i < size / 4 ; i++) {
1057                 *cache++ = (be32_to_cpu(p[i]) & 0xffff0000) >> 16;
1058                 *cache++ = be32_to_cpu(p[i]) & 0xffff;
1059         }
1060
1061         release_firmware(fw);
1062
1063         return ret;
1064 }
1065
1066 static int upgrade_fw(struct adapter *adap)
1067 {
1068         int ret;
1069         const struct firmware *fw;
1070         struct device *dev = &adap->pdev->dev;
1071
1072         ret = reject_firmware(&fw, FW_FNAME, dev);
1073         if (ret < 0) {
1074                 dev_err(dev, "could not upgrade firmware: unable to load %s\n",
1075                         FW_FNAME);
1076                 return ret;
1077         }
1078         ret = t3_load_fw(adap, fw->data, fw->size);
1079         release_firmware(fw);
1080
1081         if (ret == 0)
1082                 dev_info(dev, "successful upgrade to firmware %d.%d.%d\n",
1083                          FW_VERSION_MAJOR, FW_VERSION_MINOR, FW_VERSION_MICRO);
1084         else
1085                 dev_err(dev, "failed to upgrade to firmware %d.%d.%d\n",
1086                         FW_VERSION_MAJOR, FW_VERSION_MINOR, FW_VERSION_MICRO);
1087
1088         return ret;
1089 }
1090
1091 static inline char t3rev2char(struct adapter *adapter)
1092 {
1093         char rev = 0;
1094
1095         switch(adapter->params.rev) {
1096         case T3_REV_B:
1097         case T3_REV_B2:
1098                 rev = 'b';
1099                 break;
1100         case T3_REV_C:
1101                 rev = 'c';
1102                 break;
1103         }
1104         return rev;
1105 }
1106
1107 static int update_tpsram(struct adapter *adap)
1108 {
1109         const struct firmware *tpsram;
1110         char buf[64];
1111         struct device *dev = &adap->pdev->dev;
1112         int ret;
1113         char rev;
1114
1115         rev = t3rev2char(adap);
1116         if (!rev)
1117                 return 0;
1118
1119         snprintf(buf, sizeof(buf), TPSRAM_NAME, rev);
1120
1121         ret = reject_firmware(&tpsram, buf, dev);
1122         if (ret < 0) {
1123                 dev_err(dev, "could not load TP SRAM: unable to load %s\n",
1124                         buf);
1125                 return ret;
1126         }
1127
1128         ret = t3_check_tpsram(adap, tpsram->data, tpsram->size);
1129         if (ret)
1130                 goto release_tpsram;
1131
1132         ret = t3_set_proto_sram(adap, tpsram->data);
1133         if (ret == 0)
1134                 dev_info(dev,
1135                          "successful update of protocol engine "
1136                          "to %d.%d.%d\n",
1137                          TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO);
1138         else
1139                 dev_err(dev, "failed to update of protocol engine %d.%d.%d\n",
1140                         TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO);
1141         if (ret)
1142                 dev_err(dev, "loading protocol SRAM failed\n");
1143
1144 release_tpsram:
1145         release_firmware(tpsram);
1146
1147         return ret;
1148 }
1149
1150 /**
1151  * t3_synchronize_rx - wait for current Rx processing on a port to complete
1152  * @adap: the adapter
1153  * @p: the port
1154  *
1155  * Ensures that current Rx processing on any of the queues associated with
1156  * the given port completes before returning.  We do this by acquiring and
1157  * releasing the locks of the response queues associated with the port.
1158  */
1159 static void t3_synchronize_rx(struct adapter *adap, const struct port_info *p)
1160 {
1161         int i;
1162
1163         for (i = p->first_qset; i < p->first_qset + p->nqsets; i++) {
1164                 struct sge_rspq *q = &adap->sge.qs[i].rspq;
1165
1166                 spin_lock_irq(&q->lock);
1167                 spin_unlock_irq(&q->lock);
1168         }
1169 }
1170
1171 static void cxgb_vlan_mode(struct net_device *dev, netdev_features_t features)
1172 {
1173         struct port_info *pi = netdev_priv(dev);
1174         struct adapter *adapter = pi->adapter;
1175
1176         if (adapter->params.rev > 0) {
1177                 t3_set_vlan_accel(adapter, 1 << pi->port_id,
1178                                   features & NETIF_F_HW_VLAN_CTAG_RX);
1179         } else {
1180                 /* single control for all ports */
1181                 unsigned int i, have_vlans = features & NETIF_F_HW_VLAN_CTAG_RX;
1182
1183                 for_each_port(adapter, i)
1184                         have_vlans |=
1185                                 adapter->port[i]->features &
1186                                 NETIF_F_HW_VLAN_CTAG_RX;
1187
1188                 t3_set_vlan_accel(adapter, 1, have_vlans);
1189         }
1190         t3_synchronize_rx(adapter, pi);
1191 }
1192
1193 /**
1194  *      cxgb_up - enable the adapter
1195  *      @adapter: adapter being enabled
1196  *
1197  *      Called when the first port is enabled, this function performs the
1198  *      actions necessary to make an adapter operational, such as completing
1199  *      the initialization of HW modules, and enabling interrupts.
1200  *
1201  *      Must be called with the rtnl lock held.
1202  */
1203 static int cxgb_up(struct adapter *adap)
1204 {
1205         int i, err;
1206
1207         if (!(adap->flags & FULL_INIT_DONE)) {
1208                 err = t3_check_fw_version(adap);
1209                 if (err == -EINVAL) {
1210                         err = upgrade_fw(adap);
1211                         CH_WARN(adap, "FW upgrade to %d.%d.%d %s\n",
1212                                 FW_VERSION_MAJOR, FW_VERSION_MINOR,
1213                                 FW_VERSION_MICRO, err ? "failed" : "succeeded");
1214                 }
1215
1216                 err = t3_check_tpsram_version(adap);
1217                 if (err == -EINVAL) {
1218                         err = update_tpsram(adap);
1219                         CH_WARN(adap, "TP upgrade to %d.%d.%d %s\n",
1220                                 TP_VERSION_MAJOR, TP_VERSION_MINOR,
1221                                 TP_VERSION_MICRO, err ? "failed" : "succeeded");
1222                 }
1223
1224                 /*
1225                  * Clear interrupts now to catch errors if t3_init_hw fails.
1226                  * We clear them again later as initialization may trigger
1227                  * conditions that can interrupt.
1228                  */
1229                 t3_intr_clear(adap);
1230
1231                 err = t3_init_hw(adap, 0);
1232                 if (err)
1233                         goto out;
1234
1235                 t3_set_reg_field(adap, A_TP_PARA_REG5, 0, F_RXDDPOFFINIT);
1236                 t3_write_reg(adap, A_ULPRX_TDDP_PSZ, V_HPZ0(PAGE_SHIFT - 12));
1237
1238                 err = setup_sge_qsets(adap);
1239                 if (err)
1240                         goto out;
1241
1242                 for_each_port(adap, i)
1243                         cxgb_vlan_mode(adap->port[i], adap->port[i]->features);
1244
1245                 setup_rss(adap);
1246                 if (!(adap->flags & NAPI_INIT))
1247                         init_napi(adap);
1248
1249                 t3_start_sge_timers(adap);
1250                 adap->flags |= FULL_INIT_DONE;
1251         }
1252
1253         t3_intr_clear(adap);
1254
1255         if (adap->flags & USING_MSIX) {
1256                 name_msix_vecs(adap);
1257                 err = request_irq(adap->msix_info[0].vec,
1258                                   t3_async_intr_handler, 0,
1259                                   adap->msix_info[0].desc, adap);
1260                 if (err)
1261                         goto irq_err;
1262
1263                 err = request_msix_data_irqs(adap);
1264                 if (err) {
1265                         free_irq(adap->msix_info[0].vec, adap);
1266                         goto irq_err;
1267                 }
1268         } else if ((err = request_irq(adap->pdev->irq,
1269                                       t3_intr_handler(adap,
1270                                                       adap->sge.qs[0].rspq.
1271                                                       polling),
1272                                       (adap->flags & USING_MSI) ?
1273                                        0 : IRQF_SHARED,
1274                                       adap->name, adap)))
1275                 goto irq_err;
1276
1277         enable_all_napi(adap);
1278         t3_sge_start(adap);
1279         t3_intr_enable(adap);
1280
1281         if (adap->params.rev >= T3_REV_C && !(adap->flags & TP_PARITY_INIT) &&
1282             is_offload(adap) && init_tp_parity(adap) == 0)
1283                 adap->flags |= TP_PARITY_INIT;
1284
1285         if (adap->flags & TP_PARITY_INIT) {
1286                 t3_write_reg(adap, A_TP_INT_CAUSE,
1287                              F_CMCACHEPERR | F_ARPLUTPERR);
1288                 t3_write_reg(adap, A_TP_INT_ENABLE, 0x7fbfffff);
1289         }
1290
1291         if (!(adap->flags & QUEUES_BOUND)) {
1292                 int ret = bind_qsets(adap);
1293
1294                 if (ret < 0) {
1295                         CH_ERR(adap, "failed to bind qsets, err %d\n", ret);
1296                         t3_intr_disable(adap);
1297                         free_irq_resources(adap);
1298                         err = ret;
1299                         goto out;
1300                 }
1301                 adap->flags |= QUEUES_BOUND;
1302         }
1303
1304 out:
1305         return err;
1306 irq_err:
1307         CH_ERR(adap, "request_irq failed, err %d\n", err);
1308         goto out;
1309 }
1310
1311 /*
1312  * Release resources when all the ports and offloading have been stopped.
1313  */
1314 static void cxgb_down(struct adapter *adapter, int on_wq)
1315 {
1316         t3_sge_stop(adapter);
1317         spin_lock_irq(&adapter->work_lock);     /* sync with PHY intr task */
1318         t3_intr_disable(adapter);
1319         spin_unlock_irq(&adapter->work_lock);
1320
1321         free_irq_resources(adapter);
1322         quiesce_rx(adapter);
1323         t3_sge_stop(adapter);
1324         if (!on_wq)
1325                 flush_workqueue(cxgb3_wq);/* wait for external IRQ handler */
1326 }
1327
1328 static void schedule_chk_task(struct adapter *adap)
1329 {
1330         unsigned int timeo;
1331
1332         timeo = adap->params.linkpoll_period ?
1333             (HZ * adap->params.linkpoll_period) / 10 :
1334             adap->params.stats_update_period * HZ;
1335         if (timeo)
1336                 queue_delayed_work(cxgb3_wq, &adap->adap_check_task, timeo);
1337 }
1338
1339 static int offload_open(struct net_device *dev)
1340 {
1341         struct port_info *pi = netdev_priv(dev);
1342         struct adapter *adapter = pi->adapter;
1343         struct t3cdev *tdev = dev2t3cdev(dev);
1344         int adap_up = adapter->open_device_map & PORT_MASK;
1345         int err;
1346
1347         if (test_and_set_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map))
1348                 return 0;
1349
1350         if (!adap_up && (err = cxgb_up(adapter)) < 0)
1351                 goto out;
1352
1353         t3_tp_set_offload_mode(adapter, 1);
1354         tdev->lldev = adapter->port[0];
1355         err = cxgb3_offload_activate(adapter);
1356         if (err)
1357                 goto out;
1358
1359         init_port_mtus(adapter);
1360         t3_load_mtus(adapter, adapter->params.mtus, adapter->params.a_wnd,
1361                      adapter->params.b_wnd,
1362                      adapter->params.rev == 0 ?
1363                      adapter->port[0]->mtu : 0xffff);
1364         init_smt(adapter);
1365
1366         if (sysfs_create_group(&tdev->lldev->dev.kobj, &offload_attr_group))
1367                 dev_dbg(&dev->dev, "cannot create sysfs group\n");
1368
1369         /* Call back all registered clients */
1370         cxgb3_add_clients(tdev);
1371
1372 out:
1373         /* restore them in case the offload module has changed them */
1374         if (err) {
1375                 t3_tp_set_offload_mode(adapter, 0);
1376                 clear_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map);
1377                 cxgb3_set_dummy_ops(tdev);
1378         }
1379         return err;
1380 }
1381
1382 static int offload_close(struct t3cdev *tdev)
1383 {
1384         struct adapter *adapter = tdev2adap(tdev);
1385         struct t3c_data *td = T3C_DATA(tdev);
1386
1387         if (!test_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map))
1388                 return 0;
1389
1390         /* Call back all registered clients */
1391         cxgb3_remove_clients(tdev);
1392
1393         sysfs_remove_group(&tdev->lldev->dev.kobj, &offload_attr_group);
1394
1395         /* Flush work scheduled while releasing TIDs */
1396         flush_work(&td->tid_release_task);
1397
1398         tdev->lldev = NULL;
1399         cxgb3_set_dummy_ops(tdev);
1400         t3_tp_set_offload_mode(adapter, 0);
1401         clear_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map);
1402
1403         if (!adapter->open_device_map)
1404                 cxgb_down(adapter, 0);
1405
1406         cxgb3_offload_deactivate(adapter);
1407         return 0;
1408 }
1409
1410 static int cxgb_open(struct net_device *dev)
1411 {
1412         struct port_info *pi = netdev_priv(dev);
1413         struct adapter *adapter = pi->adapter;
1414         int other_ports = adapter->open_device_map & PORT_MASK;
1415         int err;
1416
1417         if (!adapter->open_device_map && (err = cxgb_up(adapter)) < 0)
1418                 return err;
1419
1420         set_bit(pi->port_id, &adapter->open_device_map);
1421         if (is_offload(adapter) && !ofld_disable) {
1422                 err = offload_open(dev);
1423                 if (err)
1424                         pr_warn("Could not initialize offload capabilities\n");
1425         }
1426
1427         netif_set_real_num_tx_queues(dev, pi->nqsets);
1428         err = netif_set_real_num_rx_queues(dev, pi->nqsets);
1429         if (err)
1430                 return err;
1431         link_start(dev);
1432         t3_port_intr_enable(adapter, pi->port_id);
1433         netif_tx_start_all_queues(dev);
1434         if (!other_ports)
1435                 schedule_chk_task(adapter);
1436
1437         cxgb3_event_notify(&adapter->tdev, OFFLOAD_PORT_UP, pi->port_id);
1438         return 0;
1439 }
1440
1441 static int __cxgb_close(struct net_device *dev, int on_wq)
1442 {
1443         struct port_info *pi = netdev_priv(dev);
1444         struct adapter *adapter = pi->adapter;
1445
1446
1447         if (!adapter->open_device_map)
1448                 return 0;
1449
1450         /* Stop link fault interrupts */
1451         t3_xgm_intr_disable(adapter, pi->port_id);
1452         t3_read_reg(adapter, A_XGM_INT_STATUS + pi->mac.offset);
1453
1454         t3_port_intr_disable(adapter, pi->port_id);
1455         netif_tx_stop_all_queues(dev);
1456         pi->phy.ops->power_down(&pi->phy, 1);
1457         netif_carrier_off(dev);
1458         t3_mac_disable(&pi->mac, MAC_DIRECTION_TX | MAC_DIRECTION_RX);
1459
1460         spin_lock_irq(&adapter->work_lock);     /* sync with update task */
1461         clear_bit(pi->port_id, &adapter->open_device_map);
1462         spin_unlock_irq(&adapter->work_lock);
1463
1464         if (!(adapter->open_device_map & PORT_MASK))
1465                 cancel_delayed_work_sync(&adapter->adap_check_task);
1466
1467         if (!adapter->open_device_map)
1468                 cxgb_down(adapter, on_wq);
1469
1470         cxgb3_event_notify(&adapter->tdev, OFFLOAD_PORT_DOWN, pi->port_id);
1471         return 0;
1472 }
1473
1474 static int cxgb_close(struct net_device *dev)
1475 {
1476         return __cxgb_close(dev, 0);
1477 }
1478
1479 static struct net_device_stats *cxgb_get_stats(struct net_device *dev)
1480 {
1481         struct port_info *pi = netdev_priv(dev);
1482         struct adapter *adapter = pi->adapter;
1483         struct net_device_stats *ns = &pi->netstats;
1484         const struct mac_stats *pstats;
1485
1486         spin_lock(&adapter->stats_lock);
1487         pstats = t3_mac_update_stats(&pi->mac);
1488         spin_unlock(&adapter->stats_lock);
1489
1490         ns->tx_bytes = pstats->tx_octets;
1491         ns->tx_packets = pstats->tx_frames;
1492         ns->rx_bytes = pstats->rx_octets;
1493         ns->rx_packets = pstats->rx_frames;
1494         ns->multicast = pstats->rx_mcast_frames;
1495
1496         ns->tx_errors = pstats->tx_underrun;
1497         ns->rx_errors = pstats->rx_symbol_errs + pstats->rx_fcs_errs +
1498             pstats->rx_too_long + pstats->rx_jabber + pstats->rx_short +
1499             pstats->rx_fifo_ovfl;
1500
1501         /* detailed rx_errors */
1502         ns->rx_length_errors = pstats->rx_jabber + pstats->rx_too_long;
1503         ns->rx_over_errors = 0;
1504         ns->rx_crc_errors = pstats->rx_fcs_errs;
1505         ns->rx_frame_errors = pstats->rx_symbol_errs;
1506         ns->rx_fifo_errors = pstats->rx_fifo_ovfl;
1507         ns->rx_missed_errors = pstats->rx_cong_drops;
1508
1509         /* detailed tx_errors */
1510         ns->tx_aborted_errors = 0;
1511         ns->tx_carrier_errors = 0;
1512         ns->tx_fifo_errors = pstats->tx_underrun;
1513         ns->tx_heartbeat_errors = 0;
1514         ns->tx_window_errors = 0;
1515         return ns;
1516 }
1517
1518 static u32 get_msglevel(struct net_device *dev)
1519 {
1520         struct port_info *pi = netdev_priv(dev);
1521         struct adapter *adapter = pi->adapter;
1522
1523         return adapter->msg_enable;
1524 }
1525
1526 static void set_msglevel(struct net_device *dev, u32 val)
1527 {
1528         struct port_info *pi = netdev_priv(dev);
1529         struct adapter *adapter = pi->adapter;
1530
1531         adapter->msg_enable = val;
1532 }
1533
1534 static const char stats_strings[][ETH_GSTRING_LEN] = {
1535         "TxOctetsOK         ",
1536         "TxFramesOK         ",
1537         "TxMulticastFramesOK",
1538         "TxBroadcastFramesOK",
1539         "TxPauseFrames      ",
1540         "TxUnderrun         ",
1541         "TxExtUnderrun      ",
1542
1543         "TxFrames64         ",
1544         "TxFrames65To127    ",
1545         "TxFrames128To255   ",
1546         "TxFrames256To511   ",
1547         "TxFrames512To1023  ",
1548         "TxFrames1024To1518 ",
1549         "TxFrames1519ToMax  ",
1550
1551         "RxOctetsOK         ",
1552         "RxFramesOK         ",
1553         "RxMulticastFramesOK",
1554         "RxBroadcastFramesOK",
1555         "RxPauseFrames      ",
1556         "RxFCSErrors        ",
1557         "RxSymbolErrors     ",
1558         "RxShortErrors      ",
1559         "RxJabberErrors     ",
1560         "RxLengthErrors     ",
1561         "RxFIFOoverflow     ",
1562
1563         "RxFrames64         ",
1564         "RxFrames65To127    ",
1565         "RxFrames128To255   ",
1566         "RxFrames256To511   ",
1567         "RxFrames512To1023  ",
1568         "RxFrames1024To1518 ",
1569         "RxFrames1519ToMax  ",
1570
1571         "PhyFIFOErrors      ",
1572         "TSO                ",
1573         "VLANextractions    ",
1574         "VLANinsertions     ",
1575         "TxCsumOffload      ",
1576         "RxCsumGood         ",
1577         "LroAggregated      ",
1578         "LroFlushed         ",
1579         "LroNoDesc          ",
1580         "RxDrops            ",
1581
1582         "CheckTXEnToggled   ",
1583         "CheckResets        ",
1584
1585         "LinkFaults         ",
1586 };
1587
1588 static int get_sset_count(struct net_device *dev, int sset)
1589 {
1590         switch (sset) {
1591         case ETH_SS_STATS:
1592                 return ARRAY_SIZE(stats_strings);
1593         default:
1594                 return -EOPNOTSUPP;
1595         }
1596 }
1597
1598 #define T3_REGMAP_SIZE (3 * 1024)
1599
1600 static int get_regs_len(struct net_device *dev)
1601 {
1602         return T3_REGMAP_SIZE;
1603 }
1604
1605 static int get_eeprom_len(struct net_device *dev)
1606 {
1607         return EEPROMSIZE;
1608 }
1609
1610 static void get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
1611 {
1612         struct port_info *pi = netdev_priv(dev);
1613         struct adapter *adapter = pi->adapter;
1614         u32 fw_vers = 0;
1615         u32 tp_vers = 0;
1616
1617         spin_lock(&adapter->stats_lock);
1618         t3_get_fw_version(adapter, &fw_vers);
1619         t3_get_tp_version(adapter, &tp_vers);
1620         spin_unlock(&adapter->stats_lock);
1621
1622         strlcpy(info->driver, DRV_NAME, sizeof(info->driver));
1623         strlcpy(info->version, DRV_VERSION, sizeof(info->version));
1624         strlcpy(info->bus_info, pci_name(adapter->pdev),
1625                 sizeof(info->bus_info));
1626         if (fw_vers)
1627                 snprintf(info->fw_version, sizeof(info->fw_version),
1628                          "%s %u.%u.%u TP %u.%u.%u",
1629                          G_FW_VERSION_TYPE(fw_vers) ? "T" : "N",
1630                          G_FW_VERSION_MAJOR(fw_vers),
1631                          G_FW_VERSION_MINOR(fw_vers),
1632                          G_FW_VERSION_MICRO(fw_vers),
1633                          G_TP_VERSION_MAJOR(tp_vers),
1634                          G_TP_VERSION_MINOR(tp_vers),
1635                          G_TP_VERSION_MICRO(tp_vers));
1636 }
1637
1638 static void get_strings(struct net_device *dev, u32 stringset, u8 * data)
1639 {
1640         if (stringset == ETH_SS_STATS)
1641                 memcpy(data, stats_strings, sizeof(stats_strings));
1642 }
1643
1644 static unsigned long collect_sge_port_stats(struct adapter *adapter,
1645                                             struct port_info *p, int idx)
1646 {
1647         int i;
1648         unsigned long tot = 0;
1649
1650         for (i = p->first_qset; i < p->first_qset + p->nqsets; ++i)
1651                 tot += adapter->sge.qs[i].port_stats[idx];
1652         return tot;
1653 }
1654
1655 static void get_stats(struct net_device *dev, struct ethtool_stats *stats,
1656                       u64 *data)
1657 {
1658         struct port_info *pi = netdev_priv(dev);
1659         struct adapter *adapter = pi->adapter;
1660         const struct mac_stats *s;
1661
1662         spin_lock(&adapter->stats_lock);
1663         s = t3_mac_update_stats(&pi->mac);
1664         spin_unlock(&adapter->stats_lock);
1665
1666         *data++ = s->tx_octets;
1667         *data++ = s->tx_frames;
1668         *data++ = s->tx_mcast_frames;
1669         *data++ = s->tx_bcast_frames;
1670         *data++ = s->tx_pause;
1671         *data++ = s->tx_underrun;
1672         *data++ = s->tx_fifo_urun;
1673
1674         *data++ = s->tx_frames_64;
1675         *data++ = s->tx_frames_65_127;
1676         *data++ = s->tx_frames_128_255;
1677         *data++ = s->tx_frames_256_511;
1678         *data++ = s->tx_frames_512_1023;
1679         *data++ = s->tx_frames_1024_1518;
1680         *data++ = s->tx_frames_1519_max;
1681
1682         *data++ = s->rx_octets;
1683         *data++ = s->rx_frames;
1684         *data++ = s->rx_mcast_frames;
1685         *data++ = s->rx_bcast_frames;
1686         *data++ = s->rx_pause;
1687         *data++ = s->rx_fcs_errs;
1688         *data++ = s->rx_symbol_errs;
1689         *data++ = s->rx_short;
1690         *data++ = s->rx_jabber;
1691         *data++ = s->rx_too_long;
1692         *data++ = s->rx_fifo_ovfl;
1693
1694         *data++ = s->rx_frames_64;
1695         *data++ = s->rx_frames_65_127;
1696         *data++ = s->rx_frames_128_255;
1697         *data++ = s->rx_frames_256_511;
1698         *data++ = s->rx_frames_512_1023;
1699         *data++ = s->rx_frames_1024_1518;
1700         *data++ = s->rx_frames_1519_max;
1701
1702         *data++ = pi->phy.fifo_errors;
1703
1704         *data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_TSO);
1705         *data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_VLANEX);
1706         *data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_VLANINS);
1707         *data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_TX_CSUM);
1708         *data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_RX_CSUM_GOOD);
1709         *data++ = 0;
1710         *data++ = 0;
1711         *data++ = 0;
1712         *data++ = s->rx_cong_drops;
1713
1714         *data++ = s->num_toggled;
1715         *data++ = s->num_resets;
1716
1717         *data++ = s->link_faults;
1718 }
1719
1720 static inline void reg_block_dump(struct adapter *ap, void *buf,
1721                                   unsigned int start, unsigned int end)
1722 {
1723         u32 *p = buf + start;
1724
1725         for (; start <= end; start += sizeof(u32))
1726                 *p++ = t3_read_reg(ap, start);
1727 }
1728
1729 static void get_regs(struct net_device *dev, struct ethtool_regs *regs,
1730                      void *buf)
1731 {
1732         struct port_info *pi = netdev_priv(dev);
1733         struct adapter *ap = pi->adapter;
1734
1735         /*
1736          * Version scheme:
1737          * bits 0..9: chip version
1738          * bits 10..15: chip revision
1739          * bit 31: set for PCIe cards
1740          */
1741         regs->version = 3 | (ap->params.rev << 10) | (is_pcie(ap) << 31);
1742
1743         /*
1744          * We skip the MAC statistics registers because they are clear-on-read.
1745          * Also reading multi-register stats would need to synchronize with the
1746          * periodic mac stats accumulation.  Hard to justify the complexity.
1747          */
1748         memset(buf, 0, T3_REGMAP_SIZE);
1749         reg_block_dump(ap, buf, 0, A_SG_RSPQ_CREDIT_RETURN);
1750         reg_block_dump(ap, buf, A_SG_HI_DRB_HI_THRSH, A_ULPRX_PBL_ULIMIT);
1751         reg_block_dump(ap, buf, A_ULPTX_CONFIG, A_MPS_INT_CAUSE);
1752         reg_block_dump(ap, buf, A_CPL_SWITCH_CNTRL, A_CPL_MAP_TBL_DATA);
1753         reg_block_dump(ap, buf, A_SMB_GLOBAL_TIME_CFG, A_XGM_SERDES_STAT3);
1754         reg_block_dump(ap, buf, A_XGM_SERDES_STATUS0,
1755                        XGM_REG(A_XGM_SERDES_STAT3, 1));
1756         reg_block_dump(ap, buf, XGM_REG(A_XGM_SERDES_STATUS0, 1),
1757                        XGM_REG(A_XGM_RX_SPI4_SOP_EOP_CNT, 1));
1758 }
1759
1760 static int restart_autoneg(struct net_device *dev)
1761 {
1762         struct port_info *p = netdev_priv(dev);
1763
1764         if (!netif_running(dev))
1765                 return -EAGAIN;
1766         if (p->link_config.autoneg != AUTONEG_ENABLE)
1767                 return -EINVAL;
1768         p->phy.ops->autoneg_restart(&p->phy);
1769         return 0;
1770 }
1771
1772 static int set_phys_id(struct net_device *dev,
1773                        enum ethtool_phys_id_state state)
1774 {
1775         struct port_info *pi = netdev_priv(dev);
1776         struct adapter *adapter = pi->adapter;
1777
1778         switch (state) {
1779         case ETHTOOL_ID_ACTIVE:
1780                 return 1;       /* cycle on/off once per second */
1781
1782         case ETHTOOL_ID_OFF:
1783                 t3_set_reg_field(adapter, A_T3DBG_GPIO_EN, F_GPIO0_OUT_VAL, 0);
1784                 break;
1785
1786         case ETHTOOL_ID_ON:
1787         case ETHTOOL_ID_INACTIVE:
1788                 t3_set_reg_field(adapter, A_T3DBG_GPIO_EN, F_GPIO0_OUT_VAL,
1789                          F_GPIO0_OUT_VAL);
1790         }
1791
1792         return 0;
1793 }
1794
1795 static int get_settings(struct net_device *dev, struct ethtool_cmd *cmd)
1796 {
1797         struct port_info *p = netdev_priv(dev);
1798
1799         cmd->supported = p->link_config.supported;
1800         cmd->advertising = p->link_config.advertising;
1801
1802         if (netif_carrier_ok(dev)) {
1803                 ethtool_cmd_speed_set(cmd, p->link_config.speed);
1804                 cmd->duplex = p->link_config.duplex;
1805         } else {
1806                 ethtool_cmd_speed_set(cmd, SPEED_UNKNOWN);
1807                 cmd->duplex = DUPLEX_UNKNOWN;
1808         }
1809
1810         cmd->port = (cmd->supported & SUPPORTED_TP) ? PORT_TP : PORT_FIBRE;
1811         cmd->phy_address = p->phy.mdio.prtad;
1812         cmd->transceiver = XCVR_EXTERNAL;
1813         cmd->autoneg = p->link_config.autoneg;
1814         cmd->maxtxpkt = 0;
1815         cmd->maxrxpkt = 0;
1816         return 0;
1817 }
1818
1819 static int speed_duplex_to_caps(int speed, int duplex)
1820 {
1821         int cap = 0;
1822
1823         switch (speed) {
1824         case SPEED_10:
1825                 if (duplex == DUPLEX_FULL)
1826                         cap = SUPPORTED_10baseT_Full;
1827                 else
1828                         cap = SUPPORTED_10baseT_Half;
1829                 break;
1830         case SPEED_100:
1831                 if (duplex == DUPLEX_FULL)
1832                         cap = SUPPORTED_100baseT_Full;
1833                 else
1834                         cap = SUPPORTED_100baseT_Half;
1835                 break;
1836         case SPEED_1000:
1837                 if (duplex == DUPLEX_FULL)
1838                         cap = SUPPORTED_1000baseT_Full;
1839                 else
1840                         cap = SUPPORTED_1000baseT_Half;
1841                 break;
1842         case SPEED_10000:
1843                 if (duplex == DUPLEX_FULL)
1844                         cap = SUPPORTED_10000baseT_Full;
1845         }
1846         return cap;
1847 }
1848
1849 #define ADVERTISED_MASK (ADVERTISED_10baseT_Half | ADVERTISED_10baseT_Full | \
1850                       ADVERTISED_100baseT_Half | ADVERTISED_100baseT_Full | \
1851                       ADVERTISED_1000baseT_Half | ADVERTISED_1000baseT_Full | \
1852                       ADVERTISED_10000baseT_Full)
1853
1854 static int set_settings(struct net_device *dev, struct ethtool_cmd *cmd)
1855 {
1856         struct port_info *p = netdev_priv(dev);
1857         struct link_config *lc = &p->link_config;
1858
1859         if (!(lc->supported & SUPPORTED_Autoneg)) {
1860                 /*
1861                  * PHY offers a single speed/duplex.  See if that's what's
1862                  * being requested.
1863                  */
1864                 if (cmd->autoneg == AUTONEG_DISABLE) {
1865                         u32 speed = ethtool_cmd_speed(cmd);
1866                         int cap = speed_duplex_to_caps(speed, cmd->duplex);
1867                         if (lc->supported & cap)
1868                                 return 0;
1869                 }
1870                 return -EINVAL;
1871         }
1872
1873         if (cmd->autoneg == AUTONEG_DISABLE) {
1874                 u32 speed = ethtool_cmd_speed(cmd);
1875                 int cap = speed_duplex_to_caps(speed, cmd->duplex);
1876
1877                 if (!(lc->supported & cap) || (speed == SPEED_1000))
1878                         return -EINVAL;
1879                 lc->requested_speed = speed;
1880                 lc->requested_duplex = cmd->duplex;
1881                 lc->advertising = 0;
1882         } else {
1883                 cmd->advertising &= ADVERTISED_MASK;
1884                 cmd->advertising &= lc->supported;
1885                 if (!cmd->advertising)
1886                         return -EINVAL;
1887                 lc->requested_speed = SPEED_INVALID;
1888                 lc->requested_duplex = DUPLEX_INVALID;
1889                 lc->advertising = cmd->advertising | ADVERTISED_Autoneg;
1890         }
1891         lc->autoneg = cmd->autoneg;
1892         if (netif_running(dev))
1893                 t3_link_start(&p->phy, &p->mac, lc);
1894         return 0;
1895 }
1896
1897 static void get_pauseparam(struct net_device *dev,
1898                            struct ethtool_pauseparam *epause)
1899 {
1900         struct port_info *p = netdev_priv(dev);
1901
1902         epause->autoneg = (p->link_config.requested_fc & PAUSE_AUTONEG) != 0;
1903         epause->rx_pause = (p->link_config.fc & PAUSE_RX) != 0;
1904         epause->tx_pause = (p->link_config.fc & PAUSE_TX) != 0;
1905 }
1906
1907 static int set_pauseparam(struct net_device *dev,
1908                           struct ethtool_pauseparam *epause)
1909 {
1910         struct port_info *p = netdev_priv(dev);
1911         struct link_config *lc = &p->link_config;
1912
1913         if (epause->autoneg == AUTONEG_DISABLE)
1914                 lc->requested_fc = 0;
1915         else if (lc->supported & SUPPORTED_Autoneg)
1916                 lc->requested_fc = PAUSE_AUTONEG;
1917         else
1918                 return -EINVAL;
1919
1920         if (epause->rx_pause)
1921                 lc->requested_fc |= PAUSE_RX;
1922         if (epause->tx_pause)
1923                 lc->requested_fc |= PAUSE_TX;
1924         if (lc->autoneg == AUTONEG_ENABLE) {
1925                 if (netif_running(dev))
1926                         t3_link_start(&p->phy, &p->mac, lc);
1927         } else {
1928                 lc->fc = lc->requested_fc & (PAUSE_RX | PAUSE_TX);
1929                 if (netif_running(dev))
1930                         t3_mac_set_speed_duplex_fc(&p->mac, -1, -1, lc->fc);
1931         }
1932         return 0;
1933 }
1934
1935 static void get_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
1936 {
1937         struct port_info *pi = netdev_priv(dev);
1938         struct adapter *adapter = pi->adapter;
1939         const struct qset_params *q = &adapter->params.sge.qset[pi->first_qset];
1940
1941         e->rx_max_pending = MAX_RX_BUFFERS;
1942         e->rx_jumbo_max_pending = MAX_RX_JUMBO_BUFFERS;
1943         e->tx_max_pending = MAX_TXQ_ENTRIES;
1944
1945         e->rx_pending = q->fl_size;
1946         e->rx_mini_pending = q->rspq_size;
1947         e->rx_jumbo_pending = q->jumbo_size;
1948         e->tx_pending = q->txq_size[0];
1949 }
1950
1951 static int set_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
1952 {
1953         struct port_info *pi = netdev_priv(dev);
1954         struct adapter *adapter = pi->adapter;
1955         struct qset_params *q;
1956         int i;
1957
1958         if (e->rx_pending > MAX_RX_BUFFERS ||
1959             e->rx_jumbo_pending > MAX_RX_JUMBO_BUFFERS ||
1960             e->tx_pending > MAX_TXQ_ENTRIES ||
1961             e->rx_mini_pending > MAX_RSPQ_ENTRIES ||
1962             e->rx_mini_pending < MIN_RSPQ_ENTRIES ||
1963             e->rx_pending < MIN_FL_ENTRIES ||
1964             e->rx_jumbo_pending < MIN_FL_ENTRIES ||
1965             e->tx_pending < adapter->params.nports * MIN_TXQ_ENTRIES)
1966                 return -EINVAL;
1967
1968         if (adapter->flags & FULL_INIT_DONE)
1969                 return -EBUSY;
1970
1971         q = &adapter->params.sge.qset[pi->first_qset];
1972         for (i = 0; i < pi->nqsets; ++i, ++q) {
1973                 q->rspq_size = e->rx_mini_pending;
1974                 q->fl_size = e->rx_pending;
1975                 q->jumbo_size = e->rx_jumbo_pending;
1976                 q->txq_size[0] = e->tx_pending;
1977                 q->txq_size[1] = e->tx_pending;
1978                 q->txq_size[2] = e->tx_pending;
1979         }
1980         return 0;
1981 }
1982
1983 static int set_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
1984 {
1985         struct port_info *pi = netdev_priv(dev);
1986         struct adapter *adapter = pi->adapter;
1987         struct qset_params *qsp;
1988         struct sge_qset *qs;
1989         int i;
1990
1991         if (c->rx_coalesce_usecs * 10 > M_NEWTIMER)
1992                 return -EINVAL;
1993
1994         for (i = 0; i < pi->nqsets; i++) {
1995                 qsp = &adapter->params.sge.qset[i];
1996                 qs = &adapter->sge.qs[i];
1997                 qsp->coalesce_usecs = c->rx_coalesce_usecs;
1998                 t3_update_qset_coalesce(qs, qsp);
1999         }
2000
2001         return 0;
2002 }
2003
2004 static int get_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
2005 {
2006         struct port_info *pi = netdev_priv(dev);
2007         struct adapter *adapter = pi->adapter;
2008         struct qset_params *q = adapter->params.sge.qset;
2009
2010         c->rx_coalesce_usecs = q->coalesce_usecs;
2011         return 0;
2012 }
2013
2014 static int get_eeprom(struct net_device *dev, struct ethtool_eeprom *e,
2015                       u8 * data)
2016 {
2017         struct port_info *pi = netdev_priv(dev);
2018         struct adapter *adapter = pi->adapter;
2019         int i, err = 0;
2020
2021         u8 *buf = kmalloc(EEPROMSIZE, GFP_KERNEL);
2022         if (!buf)
2023                 return -ENOMEM;
2024
2025         e->magic = EEPROM_MAGIC;
2026         for (i = e->offset & ~3; !err && i < e->offset + e->len; i += 4)
2027                 err = t3_seeprom_read(adapter, i, (__le32 *) & buf[i]);
2028
2029         if (!err)
2030                 memcpy(data, buf + e->offset, e->len);
2031         kfree(buf);
2032         return err;
2033 }
2034
2035 static int set_eeprom(struct net_device *dev, struct ethtool_eeprom *eeprom,
2036                       u8 * data)
2037 {
2038         struct port_info *pi = netdev_priv(dev);
2039         struct adapter *adapter = pi->adapter;
2040         u32 aligned_offset, aligned_len;
2041         __le32 *p;
2042         u8 *buf;
2043         int err;
2044
2045         if (eeprom->magic != EEPROM_MAGIC)
2046                 return -EINVAL;
2047
2048         aligned_offset = eeprom->offset & ~3;
2049         aligned_len = (eeprom->len + (eeprom->offset & 3) + 3) & ~3;
2050
2051         if (aligned_offset != eeprom->offset || aligned_len != eeprom->len) {
2052                 buf = kmalloc(aligned_len, GFP_KERNEL);
2053                 if (!buf)
2054                         return -ENOMEM;
2055                 err = t3_seeprom_read(adapter, aligned_offset, (__le32 *) buf);
2056                 if (!err && aligned_len > 4)
2057                         err = t3_seeprom_read(adapter,
2058                                               aligned_offset + aligned_len - 4,
2059                                               (__le32 *) & buf[aligned_len - 4]);
2060                 if (err)
2061                         goto out;
2062                 memcpy(buf + (eeprom->offset & 3), data, eeprom->len);
2063         } else
2064                 buf = data;
2065
2066         err = t3_seeprom_wp(adapter, 0);
2067         if (err)
2068                 goto out;
2069
2070         for (p = (__le32 *) buf; !err && aligned_len; aligned_len -= 4, p++) {
2071                 err = t3_seeprom_write(adapter, aligned_offset, *p);
2072                 aligned_offset += 4;
2073         }
2074
2075         if (!err)
2076                 err = t3_seeprom_wp(adapter, 1);
2077 out:
2078         if (buf != data)
2079                 kfree(buf);
2080         return err;
2081 }
2082
2083 static void get_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
2084 {
2085         wol->supported = 0;
2086         wol->wolopts = 0;
2087         memset(&wol->sopass, 0, sizeof(wol->sopass));
2088 }
2089
2090 static const struct ethtool_ops cxgb_ethtool_ops = {
2091         .get_settings = get_settings,
2092         .set_settings = set_settings,
2093         .get_drvinfo = get_drvinfo,
2094         .get_msglevel = get_msglevel,
2095         .set_msglevel = set_msglevel,
2096         .get_ringparam = get_sge_param,
2097         .set_ringparam = set_sge_param,
2098         .get_coalesce = get_coalesce,
2099         .set_coalesce = set_coalesce,
2100         .get_eeprom_len = get_eeprom_len,
2101         .get_eeprom = get_eeprom,
2102         .set_eeprom = set_eeprom,
2103         .get_pauseparam = get_pauseparam,
2104         .set_pauseparam = set_pauseparam,
2105         .get_link = ethtool_op_get_link,
2106         .get_strings = get_strings,
2107         .set_phys_id = set_phys_id,
2108         .nway_reset = restart_autoneg,
2109         .get_sset_count = get_sset_count,
2110         .get_ethtool_stats = get_stats,
2111         .get_regs_len = get_regs_len,
2112         .get_regs = get_regs,
2113         .get_wol = get_wol,
2114 };
2115
2116 static int in_range(int val, int lo, int hi)
2117 {
2118         return val < 0 || (val <= hi && val >= lo);
2119 }
2120
2121 static int cxgb_extension_ioctl(struct net_device *dev, void __user *useraddr)
2122 {
2123         struct port_info *pi = netdev_priv(dev);
2124         struct adapter *adapter = pi->adapter;
2125         u32 cmd;
2126         int ret;
2127
2128         if (copy_from_user(&cmd, useraddr, sizeof(cmd)))
2129                 return -EFAULT;
2130
2131         switch (cmd) {
2132         case CHELSIO_SET_QSET_PARAMS:{
2133                 int i;
2134                 struct qset_params *q;
2135                 struct ch_qset_params t;
2136                 int q1 = pi->first_qset;
2137                 int nqsets = pi->nqsets;
2138
2139                 if (!capable(CAP_NET_ADMIN))
2140                         return -EPERM;
2141                 if (copy_from_user(&t, useraddr, sizeof(t)))
2142                         return -EFAULT;
2143                 if (t.cmd != CHELSIO_SET_QSET_PARAMS)
2144                         return -EINVAL;
2145                 if (t.qset_idx >= SGE_QSETS)
2146                         return -EINVAL;
2147                 if (!in_range(t.intr_lat, 0, M_NEWTIMER) ||
2148                     !in_range(t.cong_thres, 0, 255) ||
2149                     !in_range(t.txq_size[0], MIN_TXQ_ENTRIES,
2150                               MAX_TXQ_ENTRIES) ||
2151                     !in_range(t.txq_size[1], MIN_TXQ_ENTRIES,
2152                               MAX_TXQ_ENTRIES) ||
2153                     !in_range(t.txq_size[2], MIN_CTRL_TXQ_ENTRIES,
2154                               MAX_CTRL_TXQ_ENTRIES) ||
2155                     !in_range(t.fl_size[0], MIN_FL_ENTRIES,
2156                               MAX_RX_BUFFERS) ||
2157                     !in_range(t.fl_size[1], MIN_FL_ENTRIES,
2158                               MAX_RX_JUMBO_BUFFERS) ||
2159                     !in_range(t.rspq_size, MIN_RSPQ_ENTRIES,
2160                               MAX_RSPQ_ENTRIES))
2161                         return -EINVAL;
2162
2163                 if ((adapter->flags & FULL_INIT_DONE) &&
2164                         (t.rspq_size >= 0 || t.fl_size[0] >= 0 ||
2165                         t.fl_size[1] >= 0 || t.txq_size[0] >= 0 ||
2166                         t.txq_size[1] >= 0 || t.txq_size[2] >= 0 ||
2167                         t.polling >= 0 || t.cong_thres >= 0))
2168                         return -EBUSY;
2169
2170                 /* Allow setting of any available qset when offload enabled */
2171                 if (test_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map)) {
2172                         q1 = 0;
2173                         for_each_port(adapter, i) {
2174                                 pi = adap2pinfo(adapter, i);
2175                                 nqsets += pi->first_qset + pi->nqsets;
2176                         }
2177                 }
2178
2179                 if (t.qset_idx < q1)
2180                         return -EINVAL;
2181                 if (t.qset_idx > q1 + nqsets - 1)
2182                         return -EINVAL;
2183
2184                 q = &adapter->params.sge.qset[t.qset_idx];
2185
2186                 if (t.rspq_size >= 0)
2187                         q->rspq_size = t.rspq_size;
2188                 if (t.fl_size[0] >= 0)
2189                         q->fl_size = t.fl_size[0];
2190                 if (t.fl_size[1] >= 0)
2191                         q->jumbo_size = t.fl_size[1];
2192                 if (t.txq_size[0] >= 0)
2193                         q->txq_size[0] = t.txq_size[0];
2194                 if (t.txq_size[1] >= 0)
2195                         q->txq_size[1] = t.txq_size[1];
2196                 if (t.txq_size[2] >= 0)
2197                         q->txq_size[2] = t.txq_size[2];
2198                 if (t.cong_thres >= 0)
2199                         q->cong_thres = t.cong_thres;
2200                 if (t.intr_lat >= 0) {
2201                         struct sge_qset *qs =
2202                                 &adapter->sge.qs[t.qset_idx];
2203
2204                         q->coalesce_usecs = t.intr_lat;
2205                         t3_update_qset_coalesce(qs, q);
2206                 }
2207                 if (t.polling >= 0) {
2208                         if (adapter->flags & USING_MSIX)
2209                                 q->polling = t.polling;
2210                         else {
2211                                 /* No polling with INTx for T3A */
2212                                 if (adapter->params.rev == 0 &&
2213                                         !(adapter->flags & USING_MSI))
2214                                         t.polling = 0;
2215
2216                                 for (i = 0; i < SGE_QSETS; i++) {
2217                                         q = &adapter->params.sge.
2218                                                 qset[i];
2219                                         q->polling = t.polling;
2220                                 }
2221                         }
2222                 }
2223
2224                 if (t.lro >= 0) {
2225                         if (t.lro)
2226                                 dev->wanted_features |= NETIF_F_GRO;
2227                         else
2228                                 dev->wanted_features &= ~NETIF_F_GRO;
2229                         netdev_update_features(dev);
2230                 }
2231
2232                 break;
2233         }
2234         case CHELSIO_GET_QSET_PARAMS:{
2235                 struct qset_params *q;
2236                 struct ch_qset_params t;
2237                 int q1 = pi->first_qset;
2238                 int nqsets = pi->nqsets;
2239                 int i;
2240
2241                 if (copy_from_user(&t, useraddr, sizeof(t)))
2242                         return -EFAULT;
2243
2244                 if (t.cmd != CHELSIO_GET_QSET_PARAMS)
2245                         return -EINVAL;
2246
2247                 /* Display qsets for all ports when offload enabled */
2248                 if (test_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map)) {
2249                         q1 = 0;
2250                         for_each_port(adapter, i) {
2251                                 pi = adap2pinfo(adapter, i);
2252                                 nqsets = pi->first_qset + pi->nqsets;
2253                         }
2254                 }
2255
2256                 if (t.qset_idx >= nqsets)
2257                         return -EINVAL;
2258                 t.qset_idx = array_index_nospec(t.qset_idx, nqsets);
2259
2260                 q = &adapter->params.sge.qset[q1 + t.qset_idx];
2261                 t.rspq_size = q->rspq_size;
2262                 t.txq_size[0] = q->txq_size[0];
2263                 t.txq_size[1] = q->txq_size[1];
2264                 t.txq_size[2] = q->txq_size[2];
2265                 t.fl_size[0] = q->fl_size;
2266                 t.fl_size[1] = q->jumbo_size;
2267                 t.polling = q->polling;
2268                 t.lro = !!(dev->features & NETIF_F_GRO);
2269                 t.intr_lat = q->coalesce_usecs;
2270                 t.cong_thres = q->cong_thres;
2271                 t.qnum = q1;
2272
2273                 if (adapter->flags & USING_MSIX)
2274                         t.vector = adapter->msix_info[q1 + t.qset_idx + 1].vec;
2275                 else
2276                         t.vector = adapter->pdev->irq;
2277
2278                 if (copy_to_user(useraddr, &t, sizeof(t)))
2279                         return -EFAULT;
2280                 break;
2281         }
2282         case CHELSIO_SET_QSET_NUM:{
2283                 struct ch_reg edata;
2284                 unsigned int i, first_qset = 0, other_qsets = 0;
2285
2286                 if (!capable(CAP_NET_ADMIN))
2287                         return -EPERM;
2288                 if (adapter->flags & FULL_INIT_DONE)
2289                         return -EBUSY;
2290                 if (copy_from_user(&edata, useraddr, sizeof(edata)))
2291                         return -EFAULT;
2292                 if (edata.cmd != CHELSIO_SET_QSET_NUM)
2293                         return -EINVAL;
2294                 if (edata.val < 1 ||
2295                         (edata.val > 1 && !(adapter->flags & USING_MSIX)))
2296                         return -EINVAL;
2297
2298                 for_each_port(adapter, i)
2299                         if (adapter->port[i] && adapter->port[i] != dev)
2300                                 other_qsets += adap2pinfo(adapter, i)->nqsets;
2301
2302                 if (edata.val + other_qsets > SGE_QSETS)
2303                         return -EINVAL;
2304
2305                 pi->nqsets = edata.val;
2306
2307                 for_each_port(adapter, i)
2308                         if (adapter->port[i]) {
2309                                 pi = adap2pinfo(adapter, i);
2310                                 pi->first_qset = first_qset;
2311                                 first_qset += pi->nqsets;
2312                         }
2313                 break;
2314         }
2315         case CHELSIO_GET_QSET_NUM:{
2316                 struct ch_reg edata;
2317
2318                 memset(&edata, 0, sizeof(struct ch_reg));
2319
2320                 edata.cmd = CHELSIO_GET_QSET_NUM;
2321                 edata.val = pi->nqsets;
2322                 if (copy_to_user(useraddr, &edata, sizeof(edata)))
2323                         return -EFAULT;
2324                 break;
2325         }
2326         case CHELSIO_LOAD_FW:{
2327                 u8 *fw_data;
2328                 struct ch_mem_range t;
2329
2330                 if (!capable(CAP_SYS_RAWIO))
2331                         return -EPERM;
2332                 if (copy_from_user(&t, useraddr, sizeof(t)))
2333                         return -EFAULT;
2334                 if (t.cmd != CHELSIO_LOAD_FW)
2335                         return -EINVAL;
2336                 /* Check t.len sanity ? */
2337                 fw_data = memdup_user(useraddr + sizeof(t), t.len);
2338                 if (IS_ERR(fw_data))
2339                         return PTR_ERR(fw_data);
2340
2341                 ret = t3_load_fw(adapter, fw_data, t.len);
2342                 kfree(fw_data);
2343                 if (ret)
2344                         return ret;
2345                 break;
2346         }
2347         case CHELSIO_SETMTUTAB:{
2348                 struct ch_mtus m;
2349                 int i;
2350
2351                 if (!is_offload(adapter))
2352                         return -EOPNOTSUPP;
2353                 if (!capable(CAP_NET_ADMIN))
2354                         return -EPERM;
2355                 if (offload_running(adapter))
2356                         return -EBUSY;
2357                 if (copy_from_user(&m, useraddr, sizeof(m)))
2358                         return -EFAULT;
2359                 if (m.cmd != CHELSIO_SETMTUTAB)
2360                         return -EINVAL;
2361                 if (m.nmtus != NMTUS)
2362                         return -EINVAL;
2363                 if (m.mtus[0] < 81)     /* accommodate SACK */
2364                         return -EINVAL;
2365
2366                 /* MTUs must be in ascending order */
2367                 for (i = 1; i < NMTUS; ++i)
2368                         if (m.mtus[i] < m.mtus[i - 1])
2369                                 return -EINVAL;
2370
2371                 memcpy(adapter->params.mtus, m.mtus,
2372                         sizeof(adapter->params.mtus));
2373                 break;
2374         }
2375         case CHELSIO_GET_PM:{
2376                 struct tp_params *p = &adapter->params.tp;
2377                 struct ch_pm m = {.cmd = CHELSIO_GET_PM };
2378
2379                 if (!is_offload(adapter))
2380                         return -EOPNOTSUPP;
2381                 m.tx_pg_sz = p->tx_pg_size;
2382                 m.tx_num_pg = p->tx_num_pgs;
2383                 m.rx_pg_sz = p->rx_pg_size;
2384                 m.rx_num_pg = p->rx_num_pgs;
2385                 m.pm_total = p->pmtx_size + p->chan_rx_size * p->nchan;
2386                 if (copy_to_user(useraddr, &m, sizeof(m)))
2387                         return -EFAULT;
2388                 break;
2389         }
2390         case CHELSIO_SET_PM:{
2391                 struct ch_pm m;
2392                 struct tp_params *p = &adapter->params.tp;
2393
2394                 if (!is_offload(adapter))
2395                         return -EOPNOTSUPP;
2396                 if (!capable(CAP_NET_ADMIN))
2397                         return -EPERM;
2398                 if (adapter->flags & FULL_INIT_DONE)
2399                         return -EBUSY;
2400                 if (copy_from_user(&m, useraddr, sizeof(m)))
2401                         return -EFAULT;
2402                 if (m.cmd != CHELSIO_SET_PM)
2403                         return -EINVAL;
2404                 if (!is_power_of_2(m.rx_pg_sz) ||
2405                         !is_power_of_2(m.tx_pg_sz))
2406                         return -EINVAL; /* not power of 2 */
2407                 if (!(m.rx_pg_sz & 0x14000))
2408                         return -EINVAL; /* not 16KB or 64KB */
2409                 if (!(m.tx_pg_sz & 0x1554000))
2410                         return -EINVAL;
2411                 if (m.tx_num_pg == -1)
2412                         m.tx_num_pg = p->tx_num_pgs;
2413                 if (m.rx_num_pg == -1)
2414                         m.rx_num_pg = p->rx_num_pgs;
2415                 if (m.tx_num_pg % 24 || m.rx_num_pg % 24)
2416                         return -EINVAL;
2417                 if (m.rx_num_pg * m.rx_pg_sz > p->chan_rx_size ||
2418                         m.tx_num_pg * m.tx_pg_sz > p->chan_tx_size)
2419                         return -EINVAL;
2420                 p->rx_pg_size = m.rx_pg_sz;
2421                 p->tx_pg_size = m.tx_pg_sz;
2422                 p->rx_num_pgs = m.rx_num_pg;
2423                 p->tx_num_pgs = m.tx_num_pg;
2424                 break;
2425         }
2426         case CHELSIO_GET_MEM:{
2427                 struct ch_mem_range t;
2428                 struct mc7 *mem;
2429                 u64 buf[32];
2430
2431                 if (!is_offload(adapter))
2432                         return -EOPNOTSUPP;
2433                 if (!capable(CAP_NET_ADMIN))
2434                         return -EPERM;
2435                 if (!(adapter->flags & FULL_INIT_DONE))
2436                         return -EIO;    /* need the memory controllers */
2437                 if (copy_from_user(&t, useraddr, sizeof(t)))
2438                         return -EFAULT;
2439                 if (t.cmd != CHELSIO_GET_MEM)
2440                         return -EINVAL;
2441                 if ((t.addr & 7) || (t.len & 7))
2442                         return -EINVAL;
2443                 if (t.mem_id == MEM_CM)
2444                         mem = &adapter->cm;
2445                 else if (t.mem_id == MEM_PMRX)
2446                         mem = &adapter->pmrx;
2447                 else if (t.mem_id == MEM_PMTX)
2448                         mem = &adapter->pmtx;
2449                 else
2450                         return -EINVAL;
2451
2452                 /*
2453                  * Version scheme:
2454                  * bits 0..9: chip version
2455                  * bits 10..15: chip revision
2456                  */
2457                 t.version = 3 | (adapter->params.rev << 10);
2458                 if (copy_to_user(useraddr, &t, sizeof(t)))
2459                         return -EFAULT;
2460
2461                 /*
2462                  * Read 256 bytes at a time as len can be large and we don't
2463                  * want to use huge intermediate buffers.
2464                  */
2465                 useraddr += sizeof(t);  /* advance to start of buffer */
2466                 while (t.len) {
2467                         unsigned int chunk =
2468                                 min_t(unsigned int, t.len, sizeof(buf));
2469
2470                         ret =
2471                                 t3_mc7_bd_read(mem, t.addr / 8, chunk / 8,
2472                                                 buf);
2473                         if (ret)
2474                                 return ret;
2475                         if (copy_to_user(useraddr, buf, chunk))
2476                                 return -EFAULT;
2477                         useraddr += chunk;
2478                         t.addr += chunk;
2479                         t.len -= chunk;
2480                 }
2481                 break;
2482         }
2483         case CHELSIO_SET_TRACE_FILTER:{
2484                 struct ch_trace t;
2485                 const struct trace_params *tp;
2486
2487                 if (!capable(CAP_NET_ADMIN))
2488                         return -EPERM;
2489                 if (!offload_running(adapter))
2490                         return -EAGAIN;
2491                 if (copy_from_user(&t, useraddr, sizeof(t)))
2492                         return -EFAULT;
2493                 if (t.cmd != CHELSIO_SET_TRACE_FILTER)
2494                         return -EINVAL;
2495
2496                 tp = (const struct trace_params *)&t.sip;
2497                 if (t.config_tx)
2498                         t3_config_trace_filter(adapter, tp, 0,
2499                                                 t.invert_match,
2500                                                 t.trace_tx);
2501                 if (t.config_rx)
2502                         t3_config_trace_filter(adapter, tp, 1,
2503                                                 t.invert_match,
2504                                                 t.trace_rx);
2505                 break;
2506         }
2507         default:
2508                 return -EOPNOTSUPP;
2509         }
2510         return 0;
2511 }
2512
2513 static int cxgb_ioctl(struct net_device *dev, struct ifreq *req, int cmd)
2514 {
2515         struct mii_ioctl_data *data = if_mii(req);
2516         struct port_info *pi = netdev_priv(dev);
2517         struct adapter *adapter = pi->adapter;
2518
2519         switch (cmd) {
2520         case SIOCGMIIREG:
2521         case SIOCSMIIREG:
2522                 /* Convert phy_id from older PRTAD/DEVAD format */
2523                 if (is_10G(adapter) &&
2524                     !mdio_phy_id_is_c45(data->phy_id) &&
2525                     (data->phy_id & 0x1f00) &&
2526                     !(data->phy_id & 0xe0e0))
2527                         data->phy_id = mdio_phy_id_c45(data->phy_id >> 8,
2528                                                        data->phy_id & 0x1f);
2529                 /* FALLTHRU */
2530         case SIOCGMIIPHY:
2531                 return mdio_mii_ioctl(&pi->phy.mdio, data, cmd);
2532         case SIOCCHIOCTL:
2533                 return cxgb_extension_ioctl(dev, req->ifr_data);
2534         default:
2535                 return -EOPNOTSUPP;
2536         }
2537 }
2538
2539 static int cxgb_change_mtu(struct net_device *dev, int new_mtu)
2540 {
2541         struct port_info *pi = netdev_priv(dev);
2542         struct adapter *adapter = pi->adapter;
2543         int ret;
2544
2545         if (new_mtu < 81)       /* accommodate SACK */
2546                 return -EINVAL;
2547         if ((ret = t3_mac_set_mtu(&pi->mac, new_mtu)))
2548                 return ret;
2549         dev->mtu = new_mtu;
2550         init_port_mtus(adapter);
2551         if (adapter->params.rev == 0 && offload_running(adapter))
2552                 t3_load_mtus(adapter, adapter->params.mtus,
2553                              adapter->params.a_wnd, adapter->params.b_wnd,
2554                              adapter->port[0]->mtu);
2555         return 0;
2556 }
2557
2558 static int cxgb_set_mac_addr(struct net_device *dev, void *p)
2559 {
2560         struct port_info *pi = netdev_priv(dev);
2561         struct adapter *adapter = pi->adapter;
2562         struct sockaddr *addr = p;
2563
2564         if (!is_valid_ether_addr(addr->sa_data))
2565                 return -EADDRNOTAVAIL;
2566
2567         memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
2568         t3_mac_set_address(&pi->mac, LAN_MAC_IDX, dev->dev_addr);
2569         if (offload_running(adapter))
2570                 write_smt_entry(adapter, pi->port_id);
2571         return 0;
2572 }
2573
2574 static netdev_features_t cxgb_fix_features(struct net_device *dev,
2575         netdev_features_t features)
2576 {
2577         /*
2578          * Since there is no support for separate rx/tx vlan accel
2579          * enable/disable make sure tx flag is always in same state as rx.
2580          */
2581         if (features & NETIF_F_HW_VLAN_CTAG_RX)
2582                 features |= NETIF_F_HW_VLAN_CTAG_TX;
2583         else
2584                 features &= ~NETIF_F_HW_VLAN_CTAG_TX;
2585
2586         return features;
2587 }
2588
2589 static int cxgb_set_features(struct net_device *dev, netdev_features_t features)
2590 {
2591         netdev_features_t changed = dev->features ^ features;
2592
2593         if (changed & NETIF_F_HW_VLAN_CTAG_RX)
2594                 cxgb_vlan_mode(dev, features);
2595
2596         return 0;
2597 }
2598
2599 #ifdef CONFIG_NET_POLL_CONTROLLER
2600 static void cxgb_netpoll(struct net_device *dev)
2601 {
2602         struct port_info *pi = netdev_priv(dev);
2603         struct adapter *adapter = pi->adapter;
2604         int qidx;
2605
2606         for (qidx = pi->first_qset; qidx < pi->first_qset + pi->nqsets; qidx++) {
2607                 struct sge_qset *qs = &adapter->sge.qs[qidx];
2608                 void *source;
2609
2610                 if (adapter->flags & USING_MSIX)
2611                         source = qs;
2612                 else
2613                         source = adapter;
2614
2615                 t3_intr_handler(adapter, qs->rspq.polling) (0, source);
2616         }
2617 }
2618 #endif
2619
2620 /*
2621  * Periodic accumulation of MAC statistics.
2622  */
2623 static void mac_stats_update(struct adapter *adapter)
2624 {
2625         int i;
2626
2627         for_each_port(adapter, i) {
2628                 struct net_device *dev = adapter->port[i];
2629                 struct port_info *p = netdev_priv(dev);
2630
2631                 if (netif_running(dev)) {
2632                         spin_lock(&adapter->stats_lock);
2633                         t3_mac_update_stats(&p->mac);
2634                         spin_unlock(&adapter->stats_lock);
2635                 }
2636         }
2637 }
2638
2639 static void check_link_status(struct adapter *adapter)
2640 {
2641         int i;
2642
2643         for_each_port(adapter, i) {
2644                 struct net_device *dev = adapter->port[i];
2645                 struct port_info *p = netdev_priv(dev);
2646                 int link_fault;
2647
2648                 spin_lock_irq(&adapter->work_lock);
2649                 link_fault = p->link_fault;
2650                 spin_unlock_irq(&adapter->work_lock);
2651
2652                 if (link_fault) {
2653                         t3_link_fault(adapter, i);
2654                         continue;
2655                 }
2656
2657                 if (!(p->phy.caps & SUPPORTED_IRQ) && netif_running(dev)) {
2658                         t3_xgm_intr_disable(adapter, i);
2659                         t3_read_reg(adapter, A_XGM_INT_STATUS + p->mac.offset);
2660
2661                         t3_link_changed(adapter, i);
2662                         t3_xgm_intr_enable(adapter, i);
2663                 }
2664         }
2665 }
2666
2667 static void check_t3b2_mac(struct adapter *adapter)
2668 {
2669         int i;
2670
2671         if (!rtnl_trylock())    /* synchronize with ifdown */
2672                 return;
2673
2674         for_each_port(adapter, i) {
2675                 struct net_device *dev = adapter->port[i];
2676                 struct port_info *p = netdev_priv(dev);
2677                 int status;
2678
2679                 if (!netif_running(dev))
2680                         continue;
2681
2682                 status = 0;
2683                 if (netif_running(dev) && netif_carrier_ok(dev))
2684                         status = t3b2_mac_watchdog_task(&p->mac);
2685                 if (status == 1)
2686                         p->mac.stats.num_toggled++;
2687                 else if (status == 2) {
2688                         struct cmac *mac = &p->mac;
2689
2690                         t3_mac_set_mtu(mac, dev->mtu);
2691                         t3_mac_set_address(mac, LAN_MAC_IDX, dev->dev_addr);
2692                         cxgb_set_rxmode(dev);
2693                         t3_link_start(&p->phy, mac, &p->link_config);
2694                         t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX);
2695                         t3_port_intr_enable(adapter, p->port_id);
2696                         p->mac.stats.num_resets++;
2697                 }
2698         }
2699         rtnl_unlock();
2700 }
2701
2702
2703 static void t3_adap_check_task(struct work_struct *work)
2704 {
2705         struct adapter *adapter = container_of(work, struct adapter,
2706                                                adap_check_task.work);
2707         const struct adapter_params *p = &adapter->params;
2708         int port;
2709         unsigned int v, status, reset;
2710
2711         adapter->check_task_cnt++;
2712
2713         check_link_status(adapter);
2714
2715         /* Accumulate MAC stats if needed */
2716         if (!p->linkpoll_period ||
2717             (adapter->check_task_cnt * p->linkpoll_period) / 10 >=
2718             p->stats_update_period) {
2719                 mac_stats_update(adapter);
2720                 adapter->check_task_cnt = 0;
2721         }
2722
2723         if (p->rev == T3_REV_B2)
2724                 check_t3b2_mac(adapter);
2725
2726         /*
2727          * Scan the XGMAC's to check for various conditions which we want to
2728          * monitor in a periodic polling manner rather than via an interrupt
2729          * condition.  This is used for conditions which would otherwise flood
2730          * the system with interrupts and we only really need to know that the
2731          * conditions are "happening" ...  For each condition we count the
2732          * detection of the condition and reset it for the next polling loop.
2733          */
2734         for_each_port(adapter, port) {
2735                 struct cmac *mac =  &adap2pinfo(adapter, port)->mac;
2736                 u32 cause;
2737
2738                 cause = t3_read_reg(adapter, A_XGM_INT_CAUSE + mac->offset);
2739                 reset = 0;
2740                 if (cause & F_RXFIFO_OVERFLOW) {
2741                         mac->stats.rx_fifo_ovfl++;
2742                         reset |= F_RXFIFO_OVERFLOW;
2743                 }
2744
2745                 t3_write_reg(adapter, A_XGM_INT_CAUSE + mac->offset, reset);
2746         }
2747
2748         /*
2749          * We do the same as above for FL_EMPTY interrupts.
2750          */
2751         status = t3_read_reg(adapter, A_SG_INT_CAUSE);
2752         reset = 0;
2753
2754         if (status & F_FLEMPTY) {
2755                 struct sge_qset *qs = &adapter->sge.qs[0];
2756                 int i = 0;
2757
2758                 reset |= F_FLEMPTY;
2759
2760                 v = (t3_read_reg(adapter, A_SG_RSPQ_FL_STATUS) >> S_FL0EMPTY) &
2761                     0xffff;
2762
2763                 while (v) {
2764                         qs->fl[i].empty += (v & 1);
2765                         if (i)
2766                                 qs++;
2767                         i ^= 1;
2768                         v >>= 1;
2769                 }
2770         }
2771
2772         t3_write_reg(adapter, A_SG_INT_CAUSE, reset);
2773
2774         /* Schedule the next check update if any port is active. */
2775         spin_lock_irq(&adapter->work_lock);
2776         if (adapter->open_device_map & PORT_MASK)
2777                 schedule_chk_task(adapter);
2778         spin_unlock_irq(&adapter->work_lock);
2779 }
2780
2781 static void db_full_task(struct work_struct *work)
2782 {
2783         struct adapter *adapter = container_of(work, struct adapter,
2784                                                db_full_task);
2785
2786         cxgb3_event_notify(&adapter->tdev, OFFLOAD_DB_FULL, 0);
2787 }
2788
2789 static void db_empty_task(struct work_struct *work)
2790 {
2791         struct adapter *adapter = container_of(work, struct adapter,
2792                                                db_empty_task);
2793
2794         cxgb3_event_notify(&adapter->tdev, OFFLOAD_DB_EMPTY, 0);
2795 }
2796
2797 static void db_drop_task(struct work_struct *work)
2798 {
2799         struct adapter *adapter = container_of(work, struct adapter,
2800                                                db_drop_task);
2801         unsigned long delay = 1000;
2802         unsigned short r;
2803
2804         cxgb3_event_notify(&adapter->tdev, OFFLOAD_DB_DROP, 0);
2805
2806         /*
2807          * Sleep a while before ringing the driver qset dbs.
2808          * The delay is between 1000-2023 usecs.
2809          */
2810         get_random_bytes(&r, 2);
2811         delay += r & 1023;
2812         set_current_state(TASK_UNINTERRUPTIBLE);
2813         schedule_timeout(usecs_to_jiffies(delay));
2814         ring_dbs(adapter);
2815 }
2816
2817 /*
2818  * Processes external (PHY) interrupts in process context.
2819  */
2820 static void ext_intr_task(struct work_struct *work)
2821 {
2822         struct adapter *adapter = container_of(work, struct adapter,
2823                                                ext_intr_handler_task);
2824         int i;
2825
2826         /* Disable link fault interrupts */
2827         for_each_port(adapter, i) {
2828                 struct net_device *dev = adapter->port[i];
2829                 struct port_info *p = netdev_priv(dev);
2830
2831                 t3_xgm_intr_disable(adapter, i);
2832                 t3_read_reg(adapter, A_XGM_INT_STATUS + p->mac.offset);
2833         }
2834
2835         /* Re-enable link fault interrupts */
2836         t3_phy_intr_handler(adapter);
2837
2838         for_each_port(adapter, i)
2839                 t3_xgm_intr_enable(adapter, i);
2840
2841         /* Now reenable external interrupts */
2842         spin_lock_irq(&adapter->work_lock);
2843         if (adapter->slow_intr_mask) {
2844                 adapter->slow_intr_mask |= F_T3DBG;
2845                 t3_write_reg(adapter, A_PL_INT_CAUSE0, F_T3DBG);
2846                 t3_write_reg(adapter, A_PL_INT_ENABLE0,
2847                              adapter->slow_intr_mask);
2848         }
2849         spin_unlock_irq(&adapter->work_lock);
2850 }
2851
2852 /*
2853  * Interrupt-context handler for external (PHY) interrupts.
2854  */
2855 void t3_os_ext_intr_handler(struct adapter *adapter)
2856 {
2857         /*
2858          * Schedule a task to handle external interrupts as they may be slow
2859          * and we use a mutex to protect MDIO registers.  We disable PHY
2860          * interrupts in the meantime and let the task reenable them when
2861          * it's done.
2862          */
2863         spin_lock(&adapter->work_lock);
2864         if (adapter->slow_intr_mask) {
2865                 adapter->slow_intr_mask &= ~F_T3DBG;
2866                 t3_write_reg(adapter, A_PL_INT_ENABLE0,
2867                              adapter->slow_intr_mask);
2868                 queue_work(cxgb3_wq, &adapter->ext_intr_handler_task);
2869         }
2870         spin_unlock(&adapter->work_lock);
2871 }
2872
2873 void t3_os_link_fault_handler(struct adapter *adapter, int port_id)
2874 {
2875         struct net_device *netdev = adapter->port[port_id];
2876         struct port_info *pi = netdev_priv(netdev);
2877
2878         spin_lock(&adapter->work_lock);
2879         pi->link_fault = 1;
2880         spin_unlock(&adapter->work_lock);
2881 }
2882
2883 static int t3_adapter_error(struct adapter *adapter, int reset, int on_wq)
2884 {
2885         int i, ret = 0;
2886
2887         if (is_offload(adapter) &&
2888             test_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map)) {
2889                 cxgb3_event_notify(&adapter->tdev, OFFLOAD_STATUS_DOWN, 0);
2890                 offload_close(&adapter->tdev);
2891         }
2892
2893         /* Stop all ports */
2894         for_each_port(adapter, i) {
2895                 struct net_device *netdev = adapter->port[i];
2896
2897                 if (netif_running(netdev))
2898                         __cxgb_close(netdev, on_wq);
2899         }
2900
2901         /* Stop SGE timers */
2902         t3_stop_sge_timers(adapter);
2903
2904         adapter->flags &= ~FULL_INIT_DONE;
2905
2906         if (reset)
2907                 ret = t3_reset_adapter(adapter);
2908
2909         pci_disable_device(adapter->pdev);
2910
2911         return ret;
2912 }
2913
2914 static int t3_reenable_adapter(struct adapter *adapter)
2915 {
2916         if (pci_enable_device(adapter->pdev)) {
2917                 dev_err(&adapter->pdev->dev,
2918                         "Cannot re-enable PCI device after reset.\n");
2919                 goto err;
2920         }
2921         pci_set_master(adapter->pdev);
2922         pci_restore_state(adapter->pdev);
2923         pci_save_state(adapter->pdev);
2924
2925         /* Free sge resources */
2926         t3_free_sge_resources(adapter);
2927
2928         if (t3_replay_prep_adapter(adapter))
2929                 goto err;
2930
2931         return 0;
2932 err:
2933         return -1;
2934 }
2935
2936 static void t3_resume_ports(struct adapter *adapter)
2937 {
2938         int i;
2939
2940         /* Restart the ports */
2941         for_each_port(adapter, i) {
2942                 struct net_device *netdev = adapter->port[i];
2943
2944                 if (netif_running(netdev)) {
2945                         if (cxgb_open(netdev)) {
2946                                 dev_err(&adapter->pdev->dev,
2947                                         "can't bring device back up"
2948                                         " after reset\n");
2949                                 continue;
2950                         }
2951                 }
2952         }
2953
2954         if (is_offload(adapter) && !ofld_disable)
2955                 cxgb3_event_notify(&adapter->tdev, OFFLOAD_STATUS_UP, 0);
2956 }
2957
2958 /*
2959  * processes a fatal error.
2960  * Bring the ports down, reset the chip, bring the ports back up.
2961  */
2962 static void fatal_error_task(struct work_struct *work)
2963 {
2964         struct adapter *adapter = container_of(work, struct adapter,
2965                                                fatal_error_handler_task);
2966         int err = 0;
2967
2968         rtnl_lock();
2969         err = t3_adapter_error(adapter, 1, 1);
2970         if (!err)
2971                 err = t3_reenable_adapter(adapter);
2972         if (!err)
2973                 t3_resume_ports(adapter);
2974
2975         CH_ALERT(adapter, "adapter reset %s\n", err ? "failed" : "succeeded");
2976         rtnl_unlock();
2977 }
2978
2979 void t3_fatal_err(struct adapter *adapter)
2980 {
2981         unsigned int fw_status[4];
2982
2983         if (adapter->flags & FULL_INIT_DONE) {
2984                 t3_sge_stop(adapter);
2985                 t3_write_reg(adapter, A_XGM_TX_CTRL, 0);
2986                 t3_write_reg(adapter, A_XGM_RX_CTRL, 0);
2987                 t3_write_reg(adapter, XGM_REG(A_XGM_TX_CTRL, 1), 0);
2988                 t3_write_reg(adapter, XGM_REG(A_XGM_RX_CTRL, 1), 0);
2989
2990                 spin_lock(&adapter->work_lock);
2991                 t3_intr_disable(adapter);
2992                 queue_work(cxgb3_wq, &adapter->fatal_error_handler_task);
2993                 spin_unlock(&adapter->work_lock);
2994         }
2995         CH_ALERT(adapter, "encountered fatal error, operation suspended\n");
2996         if (!t3_cim_ctl_blk_read(adapter, 0xa0, 4, fw_status))
2997                 CH_ALERT(adapter, "FW status: 0x%x, 0x%x, 0x%x, 0x%x\n",
2998                          fw_status[0], fw_status[1],
2999                          fw_status[2], fw_status[3]);
3000 }
3001
3002 /**
3003  * t3_io_error_detected - called when PCI error is detected
3004  * @pdev: Pointer to PCI device
3005  * @state: The current pci connection state
3006  *
3007  * This function is called after a PCI bus error affecting
3008  * this device has been detected.
3009  */
3010 static pci_ers_result_t t3_io_error_detected(struct pci_dev *pdev,
3011                                              pci_channel_state_t state)
3012 {
3013         struct adapter *adapter = pci_get_drvdata(pdev);
3014
3015         if (state == pci_channel_io_perm_failure)
3016                 return PCI_ERS_RESULT_DISCONNECT;
3017
3018         t3_adapter_error(adapter, 0, 0);
3019
3020         /* Request a slot reset. */
3021         return PCI_ERS_RESULT_NEED_RESET;
3022 }
3023
3024 /**
3025  * t3_io_slot_reset - called after the pci bus has been reset.
3026  * @pdev: Pointer to PCI device
3027  *
3028  * Restart the card from scratch, as if from a cold-boot.
3029  */
3030 static pci_ers_result_t t3_io_slot_reset(struct pci_dev *pdev)
3031 {
3032         struct adapter *adapter = pci_get_drvdata(pdev);
3033
3034         if (!t3_reenable_adapter(adapter))
3035                 return PCI_ERS_RESULT_RECOVERED;
3036
3037         return PCI_ERS_RESULT_DISCONNECT;
3038 }
3039
3040 /**
3041  * t3_io_resume - called when traffic can start flowing again.
3042  * @pdev: Pointer to PCI device
3043  *
3044  * This callback is called when the error recovery driver tells us that
3045  * its OK to resume normal operation.
3046  */
3047 static void t3_io_resume(struct pci_dev *pdev)
3048 {
3049         struct adapter *adapter = pci_get_drvdata(pdev);
3050
3051         CH_ALERT(adapter, "adapter recovering, PEX ERR 0x%x\n",
3052                  t3_read_reg(adapter, A_PCIE_PEX_ERR));
3053
3054         rtnl_lock();
3055         t3_resume_ports(adapter);
3056         rtnl_unlock();
3057 }
3058
3059 static const struct pci_error_handlers t3_err_handler = {
3060         .error_detected = t3_io_error_detected,
3061         .slot_reset = t3_io_slot_reset,
3062         .resume = t3_io_resume,
3063 };
3064
3065 /*
3066  * Set the number of qsets based on the number of CPUs and the number of ports,
3067  * not to exceed the number of available qsets, assuming there are enough qsets
3068  * per port in HW.
3069  */
3070 static void set_nqsets(struct adapter *adap)
3071 {
3072         int i, j = 0;
3073         int num_cpus = netif_get_num_default_rss_queues();
3074         int hwports = adap->params.nports;
3075         int nqsets = adap->msix_nvectors - 1;
3076
3077         if (adap->params.rev > 0 && adap->flags & USING_MSIX) {
3078                 if (hwports == 2 &&
3079                     (hwports * nqsets > SGE_QSETS ||
3080                      num_cpus >= nqsets / hwports))
3081                         nqsets /= hwports;
3082                 if (nqsets > num_cpus)
3083                         nqsets = num_cpus;
3084                 if (nqsets < 1 || hwports == 4)
3085                         nqsets = 1;
3086         } else
3087                 nqsets = 1;
3088
3089         for_each_port(adap, i) {
3090                 struct port_info *pi = adap2pinfo(adap, i);
3091
3092                 pi->first_qset = j;
3093                 pi->nqsets = nqsets;
3094                 j = pi->first_qset + nqsets;
3095
3096                 dev_info(&adap->pdev->dev,
3097                          "Port %d using %d queue sets.\n", i, nqsets);
3098         }
3099 }
3100
3101 static int cxgb_enable_msix(struct adapter *adap)
3102 {
3103         struct msix_entry entries[SGE_QSETS + 1];
3104         int vectors;
3105         int i;
3106
3107         vectors = ARRAY_SIZE(entries);
3108         for (i = 0; i < vectors; ++i)
3109                 entries[i].entry = i;
3110
3111         vectors = pci_enable_msix_range(adap->pdev, entries,
3112                                         adap->params.nports + 1, vectors);
3113         if (vectors < 0)
3114                 return vectors;
3115
3116         for (i = 0; i < vectors; ++i)
3117                 adap->msix_info[i].vec = entries[i].vector;
3118         adap->msix_nvectors = vectors;
3119
3120         return 0;
3121 }
3122
3123 static void print_port_info(struct adapter *adap, const struct adapter_info *ai)
3124 {
3125         static const char *pci_variant[] = {
3126                 "PCI", "PCI-X", "PCI-X ECC", "PCI-X 266", "PCI Express"
3127         };
3128
3129         int i;
3130         char buf[80];
3131
3132         if (is_pcie(adap))
3133                 snprintf(buf, sizeof(buf), "%s x%d",
3134                          pci_variant[adap->params.pci.variant],
3135                          adap->params.pci.width);
3136         else
3137                 snprintf(buf, sizeof(buf), "%s %dMHz/%d-bit",
3138                          pci_variant[adap->params.pci.variant],
3139                          adap->params.pci.speed, adap->params.pci.width);
3140
3141         for_each_port(adap, i) {
3142                 struct net_device *dev = adap->port[i];
3143                 const struct port_info *pi = netdev_priv(dev);
3144
3145                 if (!test_bit(i, &adap->registered_device_map))
3146                         continue;
3147                 netdev_info(dev, "%s %s %sNIC (rev %d) %s%s\n",
3148                             ai->desc, pi->phy.desc,
3149                             is_offload(adap) ? "R" : "", adap->params.rev, buf,
3150                             (adap->flags & USING_MSIX) ? " MSI-X" :
3151                             (adap->flags & USING_MSI) ? " MSI" : "");
3152                 if (adap->name == dev->name && adap->params.vpd.mclk)
3153                         pr_info("%s: %uMB CM, %uMB PMTX, %uMB PMRX, S/N: %s\n",
3154                                adap->name, t3_mc7_size(&adap->cm) >> 20,
3155                                t3_mc7_size(&adap->pmtx) >> 20,
3156                                t3_mc7_size(&adap->pmrx) >> 20,
3157                                adap->params.vpd.sn);
3158         }
3159 }
3160
3161 static const struct net_device_ops cxgb_netdev_ops = {
3162         .ndo_open               = cxgb_open,
3163         .ndo_stop               = cxgb_close,
3164         .ndo_start_xmit         = t3_eth_xmit,
3165         .ndo_get_stats          = cxgb_get_stats,
3166         .ndo_validate_addr      = eth_validate_addr,
3167         .ndo_set_rx_mode        = cxgb_set_rxmode,
3168         .ndo_do_ioctl           = cxgb_ioctl,
3169         .ndo_change_mtu         = cxgb_change_mtu,
3170         .ndo_set_mac_address    = cxgb_set_mac_addr,
3171         .ndo_fix_features       = cxgb_fix_features,
3172         .ndo_set_features       = cxgb_set_features,
3173 #ifdef CONFIG_NET_POLL_CONTROLLER
3174         .ndo_poll_controller    = cxgb_netpoll,
3175 #endif
3176 };
3177
3178 static void cxgb3_init_iscsi_mac(struct net_device *dev)
3179 {
3180         struct port_info *pi = netdev_priv(dev);
3181
3182         memcpy(pi->iscsic.mac_addr, dev->dev_addr, ETH_ALEN);
3183         pi->iscsic.mac_addr[3] |= 0x80;
3184 }
3185
3186 #define TSO_FLAGS (NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_TSO_ECN)
3187 #define VLAN_FEAT (NETIF_F_SG | NETIF_F_IP_CSUM | TSO_FLAGS | \
3188                         NETIF_F_IPV6_CSUM | NETIF_F_HIGHDMA)
3189 static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
3190 {
3191         int i, err, pci_using_dac = 0;
3192         resource_size_t mmio_start, mmio_len;
3193         const struct adapter_info *ai;
3194         struct adapter *adapter = NULL;
3195         struct port_info *pi;
3196
3197         pr_info_once("%s - version %s\n", DRV_DESC, DRV_VERSION);
3198
3199         if (!cxgb3_wq) {
3200                 cxgb3_wq = create_singlethread_workqueue(DRV_NAME);
3201                 if (!cxgb3_wq) {
3202                         pr_err("cannot initialize work queue\n");
3203                         return -ENOMEM;
3204                 }
3205         }
3206
3207         err = pci_enable_device(pdev);
3208         if (err) {
3209                 dev_err(&pdev->dev, "cannot enable PCI device\n");
3210                 goto out;
3211         }
3212
3213         err = pci_request_regions(pdev, DRV_NAME);
3214         if (err) {
3215                 /* Just info, some other driver may have claimed the device. */
3216                 dev_info(&pdev->dev, "cannot obtain PCI resources\n");
3217                 goto out_disable_device;
3218         }
3219
3220         if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) {
3221                 pci_using_dac = 1;
3222                 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
3223                 if (err) {
3224                         dev_err(&pdev->dev, "unable to obtain 64-bit DMA for "
3225                                "coherent allocations\n");
3226                         goto out_release_regions;
3227                 }
3228         } else if ((err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32))) != 0) {
3229                 dev_err(&pdev->dev, "no usable DMA configuration\n");
3230                 goto out_release_regions;
3231         }
3232
3233         pci_set_master(pdev);
3234         pci_save_state(pdev);
3235
3236         mmio_start = pci_resource_start(pdev, 0);
3237         mmio_len = pci_resource_len(pdev, 0);
3238         ai = t3_get_adapter_info(ent->driver_data);
3239
3240         adapter = kzalloc(sizeof(*adapter), GFP_KERNEL);
3241         if (!adapter) {
3242                 err = -ENOMEM;
3243                 goto out_release_regions;
3244         }
3245
3246         adapter->nofail_skb =
3247                 alloc_skb(sizeof(struct cpl_set_tcb_field), GFP_KERNEL);
3248         if (!adapter->nofail_skb) {
3249                 dev_err(&pdev->dev, "cannot allocate nofail buffer\n");
3250                 err = -ENOMEM;
3251                 goto out_free_adapter;
3252         }
3253
3254         adapter->regs = ioremap_nocache(mmio_start, mmio_len);
3255         if (!adapter->regs) {
3256                 dev_err(&pdev->dev, "cannot map device registers\n");
3257                 err = -ENOMEM;
3258                 goto out_free_adapter_nofail;
3259         }
3260
3261         adapter->pdev = pdev;
3262         adapter->name = pci_name(pdev);
3263         adapter->msg_enable = dflt_msg_enable;
3264         adapter->mmio_len = mmio_len;
3265
3266         mutex_init(&adapter->mdio_lock);
3267         spin_lock_init(&adapter->work_lock);
3268         spin_lock_init(&adapter->stats_lock);
3269
3270         INIT_LIST_HEAD(&adapter->adapter_list);
3271         INIT_WORK(&adapter->ext_intr_handler_task, ext_intr_task);
3272         INIT_WORK(&adapter->fatal_error_handler_task, fatal_error_task);
3273
3274         INIT_WORK(&adapter->db_full_task, db_full_task);
3275         INIT_WORK(&adapter->db_empty_task, db_empty_task);
3276         INIT_WORK(&adapter->db_drop_task, db_drop_task);
3277
3278         INIT_DELAYED_WORK(&adapter->adap_check_task, t3_adap_check_task);
3279
3280         for (i = 0; i < ai->nports0 + ai->nports1; ++i) {
3281                 struct net_device *netdev;
3282
3283                 netdev = alloc_etherdev_mq(sizeof(struct port_info), SGE_QSETS);
3284                 if (!netdev) {
3285                         err = -ENOMEM;
3286                         goto out_free_dev;
3287                 }
3288
3289                 SET_NETDEV_DEV(netdev, &pdev->dev);
3290
3291                 adapter->port[i] = netdev;
3292                 pi = netdev_priv(netdev);
3293                 pi->adapter = adapter;
3294                 pi->port_id = i;
3295                 netif_carrier_off(netdev);
3296                 netdev->irq = pdev->irq;
3297                 netdev->mem_start = mmio_start;
3298                 netdev->mem_end = mmio_start + mmio_len - 1;
3299                 netdev->hw_features = NETIF_F_SG | NETIF_F_IP_CSUM |
3300                         NETIF_F_TSO | NETIF_F_RXCSUM | NETIF_F_HW_VLAN_CTAG_RX;
3301                 netdev->features |= netdev->hw_features |
3302                                     NETIF_F_HW_VLAN_CTAG_TX;
3303                 netdev->vlan_features |= netdev->features & VLAN_FEAT;
3304                 if (pci_using_dac)
3305                         netdev->features |= NETIF_F_HIGHDMA;
3306
3307                 netdev->netdev_ops = &cxgb_netdev_ops;
3308                 netdev->ethtool_ops = &cxgb_ethtool_ops;
3309         }
3310
3311         pci_set_drvdata(pdev, adapter);
3312         if (t3_prep_adapter(adapter, ai, 1) < 0) {
3313                 err = -ENODEV;
3314                 goto out_free_dev;
3315         }
3316
3317         /*
3318          * The card is now ready to go.  If any errors occur during device
3319          * registration we do not fail the whole card but rather proceed only
3320          * with the ports we manage to register successfully.  However we must
3321          * register at least one net device.
3322          */
3323         for_each_port(adapter, i) {
3324                 err = register_netdev(adapter->port[i]);
3325                 if (err)
3326                         dev_warn(&pdev->dev,
3327                                  "cannot register net device %s, skipping\n",
3328                                  adapter->port[i]->name);
3329                 else {
3330                         /*
3331                          * Change the name we use for messages to the name of
3332                          * the first successfully registered interface.
3333                          */
3334                         if (!adapter->registered_device_map)
3335                                 adapter->name = adapter->port[i]->name;
3336
3337                         __set_bit(i, &adapter->registered_device_map);
3338                 }
3339         }
3340         if (!adapter->registered_device_map) {
3341                 dev_err(&pdev->dev, "could not register any net devices\n");
3342                 goto out_free_dev;
3343         }
3344
3345         for_each_port(adapter, i)
3346                 cxgb3_init_iscsi_mac(adapter->port[i]);
3347
3348         /* Driver's ready. Reflect it on LEDs */
3349         t3_led_ready(adapter);
3350
3351         if (is_offload(adapter)) {
3352                 __set_bit(OFFLOAD_DEVMAP_BIT, &adapter->registered_device_map);
3353                 cxgb3_adapter_ofld(adapter);
3354         }
3355
3356         /* See what interrupts we'll be using */
3357         if (msi > 1 && cxgb_enable_msix(adapter) == 0)
3358                 adapter->flags |= USING_MSIX;
3359         else if (msi > 0 && pci_enable_msi(pdev) == 0)
3360                 adapter->flags |= USING_MSI;
3361
3362         set_nqsets(adapter);
3363
3364         err = sysfs_create_group(&adapter->port[0]->dev.kobj,
3365                                  &cxgb3_attr_group);
3366
3367         print_port_info(adapter, ai);
3368         return 0;
3369
3370 out_free_dev:
3371         iounmap(adapter->regs);
3372         for (i = ai->nports0 + ai->nports1 - 1; i >= 0; --i)
3373                 if (adapter->port[i])
3374                         free_netdev(adapter->port[i]);
3375
3376 out_free_adapter_nofail:
3377         kfree_skb(adapter->nofail_skb);
3378
3379 out_free_adapter:
3380         kfree(adapter);
3381
3382 out_release_regions:
3383         pci_release_regions(pdev);
3384 out_disable_device:
3385         pci_disable_device(pdev);
3386 out:
3387         return err;
3388 }
3389
3390 static void remove_one(struct pci_dev *pdev)
3391 {
3392         struct adapter *adapter = pci_get_drvdata(pdev);
3393
3394         if (adapter) {
3395                 int i;
3396
3397                 t3_sge_stop(adapter);
3398                 sysfs_remove_group(&adapter->port[0]->dev.kobj,
3399                                    &cxgb3_attr_group);
3400
3401                 if (is_offload(adapter)) {
3402                         cxgb3_adapter_unofld(adapter);
3403                         if (test_bit(OFFLOAD_DEVMAP_BIT,
3404                                      &adapter->open_device_map))
3405                                 offload_close(&adapter->tdev);
3406                 }
3407
3408                 for_each_port(adapter, i)
3409                     if (test_bit(i, &adapter->registered_device_map))
3410                         unregister_netdev(adapter->port[i]);
3411
3412                 t3_stop_sge_timers(adapter);
3413                 t3_free_sge_resources(adapter);
3414                 cxgb_disable_msi(adapter);
3415
3416                 for_each_port(adapter, i)
3417                         if (adapter->port[i])
3418                                 free_netdev(adapter->port[i]);
3419
3420                 iounmap(adapter->regs);
3421                 if (adapter->nofail_skb)
3422                         kfree_skb(adapter->nofail_skb);
3423                 kfree(adapter);
3424                 pci_release_regions(pdev);
3425                 pci_disable_device(pdev);
3426         }
3427 }
3428
3429 static struct pci_driver driver = {
3430         .name = DRV_NAME,
3431         .id_table = cxgb3_pci_tbl,
3432         .probe = init_one,
3433         .remove = remove_one,
3434         .err_handler = &t3_err_handler,
3435 };
3436
3437 static int __init cxgb3_init_module(void)
3438 {
3439         int ret;
3440
3441         cxgb3_offload_init();
3442
3443         ret = pci_register_driver(&driver);
3444         return ret;
3445 }
3446
3447 static void __exit cxgb3_cleanup_module(void)
3448 {
3449         pci_unregister_driver(&driver);
3450         if (cxgb3_wq)
3451                 destroy_workqueue(cxgb3_wq);
3452 }
3453
3454 module_init(cxgb3_init_module);
3455 module_exit(cxgb3_cleanup_module);