drivers/net/ethernet/cavium/liquidio/lio_core.c

   1 /**********************************************************************
   2  * Author: Cavium, Inc.
   3  *
   4  * Contact: support@cavium.com
   5  *          Please include "LiquidIO" in the subject.
   6  *
   7  * Copyright (c) 2003-2016 Cavium, Inc.
   8  *
   9  * This file is free software; you can redistribute it and/or modify
  10  * it under the terms of the GNU General Public License, Version 2, as
  11  * published by the Free Software Foundation.
  12  *
  13  * This file is distributed in the hope that it will be useful, but
  14  * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty
  15  * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or
  16  * NONINFRINGEMENT.  See the GNU General Public License for more details.
  17  ***********************************************************************/
  18 #include <linux/pci.h>
  19 #include <linux/if_vlan.h>
  20 #include "liquidio_common.h"
  21 #include "octeon_droq.h"
  22 #include "octeon_iq.h"
  23 #include "response_manager.h"
  24 #include "octeon_device.h"
  25 #include "octeon_nic.h"
  26 #include "octeon_main.h"
  27 #include "octeon_network.h"
  28
  29 /* OOM task polling interval */
  30 #define LIO_OOM_POLL_INTERVAL_MS 250
  31
  32 #define OCTNIC_MAX_SG  MAX_SKB_FRAGS
  33
  34 /**
  35  * \brief Callback for getting interface configuration
  36  * @param status status of request
  37  * @param buf pointer to resp structure
  38  */
  39 void lio_if_cfg_callback(struct octeon_device *oct,
  40                          u32 status __attribute__((unused)), void *buf)
  41 {
  42         struct octeon_soft_command *sc = (struct octeon_soft_command *)buf;
  43         struct liquidio_if_cfg_context *ctx;
  44         struct liquidio_if_cfg_resp *resp;
  45
  46         resp = (struct liquidio_if_cfg_resp *)sc->virtrptr;
  47         ctx = (struct liquidio_if_cfg_context *)sc->ctxptr;
  48
  49         oct = lio_get_device(ctx->octeon_id);
  50         if (resp->status)
  51                 dev_err(&oct->pci_dev->dev, "nic if cfg instruction failed. Status: %llx\n",
  52                         CVM_CAST64(resp->status));
  53         WRITE_ONCE(ctx->cond, 1);
  54
  55         snprintf(oct->fw_info.liquidio_firmware_version, 32, "%s",
  56                  resp->cfg_info.liquidio_firmware_version);
  57
  58         /* This barrier is required to be sure that the response has been
  59          * written fully before waking up the handler
  60          */
  61         wmb();
  62
  63         wake_up_interruptible(&ctx->wc);
  64 }
  65
  66 /**
  67  * \brief Delete gather lists
  68  * @param lio per-network private data
  69  */
  70 void lio_delete_glists(struct lio *lio)
  71 {
  72         struct octnic_gather *g;
  73         int i;
  74
  75         kfree(lio->glist_lock);
  76         lio->glist_lock = NULL;
  77
  78         if (!lio->glist)
  79                 return;
  80
  81         for (i = 0; i < lio->oct_dev->num_iqs; i++) {
  82                 do {
  83                         g = (struct octnic_gather *)
  84                             lio_list_delete_head(&lio->glist[i]);
  85                         kfree(g);
  86                 } while (g);
  87
  88                 if (lio->glists_virt_base && lio->glists_virt_base[i] &&
  89                     lio->glists_dma_base && lio->glists_dma_base[i]) {
  90                         lio_dma_free(lio->oct_dev,
  91                                      lio->glist_entry_size * lio->tx_qsize,
  92                                      lio->glists_virt_base[i],
  93                                      lio->glists_dma_base[i]);
  94                 }
  95         }
  96
  97         kfree(lio->glists_virt_base);
  98         lio->glists_virt_base = NULL;
  99
 100         kfree(lio->glists_dma_base);
 101         lio->glists_dma_base = NULL;
 102
 103         kfree(lio->glist);
 104         lio->glist = NULL;
 105 }
 106
 107 /**
 108  * \brief Setup gather lists
 109  * @param lio per-network private data
 110  */
 111 int lio_setup_glists(struct octeon_device *oct, struct lio *lio, int num_iqs)
 112 {
 113         struct octnic_gather *g;
 114         int i, j;
 115
 116         lio->glist_lock =
 117             kcalloc(num_iqs, sizeof(*lio->glist_lock), GFP_KERNEL);
 118         if (!lio->glist_lock)
 119                 return -ENOMEM;
 120
 121         lio->glist =
 122             kcalloc(num_iqs, sizeof(*lio->glist), GFP_KERNEL);
 123         if (!lio->glist) {
 124                 kfree(lio->glist_lock);
 125                 lio->glist_lock = NULL;
 126                 return -ENOMEM;
 127         }
 128
 129         lio->glist_entry_size =
 130                 ROUNDUP8((ROUNDUP4(OCTNIC_MAX_SG) >> 2) * OCT_SG_ENTRY_SIZE);
 131
 132         /* allocate memory to store virtual and dma base address of
 133          * per glist consistent memory
 134          */
 135         lio->glists_virt_base = kcalloc(num_iqs, sizeof(*lio->glists_virt_base),
 136                                         GFP_KERNEL);
 137         lio->glists_dma_base = kcalloc(num_iqs, sizeof(*lio->glists_dma_base),
 138                                        GFP_KERNEL);
 139
 140         if (!lio->glists_virt_base || !lio->glists_dma_base) {
 141                 lio_delete_glists(lio);
 142                 return -ENOMEM;
 143         }
 144
 145         for (i = 0; i < num_iqs; i++) {
 146                 int numa_node = dev_to_node(&oct->pci_dev->dev);
 147
 148                 spin_lock_init(&lio->glist_lock[i]);
 149
 150                 INIT_LIST_HEAD(&lio->glist[i]);
 151
 152                 lio->glists_virt_base[i] =
 153                         lio_dma_alloc(oct,
 154                                       lio->glist_entry_size * lio->tx_qsize,
 155                                       &lio->glists_dma_base[i]);
 156
 157                 if (!lio->glists_virt_base[i]) {
 158                         lio_delete_glists(lio);
 159                         return -ENOMEM;
 160                 }
 161
 162                 for (j = 0; j < lio->tx_qsize; j++) {
 163                         g = kzalloc_node(sizeof(*g), GFP_KERNEL,
 164                                          numa_node);
 165                         if (!g)
 166                                 g = kzalloc(sizeof(*g), GFP_KERNEL);
 167                         if (!g)
 168                                 break;
 169
 170                         g->sg = lio->glists_virt_base[i] +
 171                                 (j * lio->glist_entry_size);
 172
 173                         g->sg_dma_ptr = lio->glists_dma_base[i] +
 174                                         (j * lio->glist_entry_size);
 175
 176                         list_add_tail(&g->list, &lio->glist[i]);
 177                 }
 178
 179                 if (j != lio->tx_qsize) {
 180                         lio_delete_glists(lio);
 181                         return -ENOMEM;
 182                 }
 183         }
 184
 185         return 0;
 186 }
 187
 188 int liquidio_set_feature(struct net_device *netdev, int cmd, u16 param1)
 189 {
 190         struct lio *lio = GET_LIO(netdev);
 191         struct octeon_device *oct = lio->oct_dev;
 192         struct octnic_ctrl_pkt nctrl;
 193         int ret = 0;
 194
 195         memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
 196
 197         nctrl.ncmd.u64 = 0;
 198         nctrl.ncmd.s.cmd = cmd;
 199         nctrl.ncmd.s.param1 = param1;
 200         nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
 201         nctrl.wait_time = 100;
 202         nctrl.netpndev = (u64)netdev;
 203         nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
 204
 205         ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
 206         if (ret < 0) {
 207                 dev_err(&oct->pci_dev->dev, "Feature change failed in core (ret: 0x%x)\n",
 208                         ret);
 209         }
 210         return ret;
 211 }
 212
 213 void octeon_report_tx_completion_to_bql(void *txq, unsigned int pkts_compl,
 214                                         unsigned int bytes_compl)
 215 {
 216         struct netdev_queue *netdev_queue = txq;
 217
 218         netdev_tx_completed_queue(netdev_queue, pkts_compl, bytes_compl);
 219 }
 220
 221 void octeon_update_tx_completion_counters(void *buf, int reqtype,
 222                                           unsigned int *pkts_compl,
 223                                           unsigned int *bytes_compl)
 224 {
 225         struct octnet_buf_free_info *finfo;
 226         struct sk_buff *skb = NULL;
 227         struct octeon_soft_command *sc;
 228
 229         switch (reqtype) {
 230         case REQTYPE_NORESP_NET:
 231         case REQTYPE_NORESP_NET_SG:
 232                 finfo = buf;
 233                 skb = finfo->skb;
 234                 break;
 235
 236         case REQTYPE_RESP_NET_SG:
 237         case REQTYPE_RESP_NET:
 238                 sc = buf;
 239                 skb = sc->callback_arg;
 240                 break;
 241
 242         default:
 243                 return;
 244         }
 245
 246         (*pkts_compl)++;
 247         *bytes_compl += skb->len;
 248 }
 249
 250 int octeon_report_sent_bytes_to_bql(void *buf, int reqtype)
 251 {
 252         struct octnet_buf_free_info *finfo;
 253         struct sk_buff *skb;
 254         struct octeon_soft_command *sc;
 255         struct netdev_queue *txq;
 256
 257         switch (reqtype) {
 258         case REQTYPE_NORESP_NET:
 259         case REQTYPE_NORESP_NET_SG:
 260                 finfo = buf;
 261                 skb = finfo->skb;
 262                 break;
 263
 264         case REQTYPE_RESP_NET_SG:
 265         case REQTYPE_RESP_NET:
 266                 sc = buf;
 267                 skb = sc->callback_arg;
 268                 break;
 269
 270         default:
 271                 return 0;
 272         }
 273
 274         txq = netdev_get_tx_queue(skb->dev, skb_get_queue_mapping(skb));
 275         netdev_tx_sent_queue(txq, skb->len);
 276
 277         return netif_xmit_stopped(txq);
 278 }
 279
 280 void liquidio_link_ctrl_cmd_completion(void *nctrl_ptr)
 281 {
 282         struct octnic_ctrl_pkt *nctrl = (struct octnic_ctrl_pkt *)nctrl_ptr;
 283         struct net_device *netdev = (struct net_device *)nctrl->netpndev;
 284         struct lio *lio = GET_LIO(netdev);
 285         struct octeon_device *oct = lio->oct_dev;
 286         u8 *mac;
 287
 288         if (nctrl->completion && nctrl->response_code) {
 289                 /* Signal whoever is interested that the response code from the
 290                  * firmware has arrived.
 291                  */
 292                 WRITE_ONCE(*nctrl->response_code, nctrl->status);
 293                 complete(nctrl->completion);
 294         }
 295
 296         if (nctrl->status)
 297                 return;
 298
 299         switch (nctrl->ncmd.s.cmd) {
 300         case OCTNET_CMD_CHANGE_DEVFLAGS:
 301         case OCTNET_CMD_SET_MULTI_LIST:
 302         case OCTNET_CMD_SET_UC_LIST:
 303                 break;
 304
 305         case OCTNET_CMD_CHANGE_MACADDR:
 306                 mac = ((u8 *)&nctrl->udd[0]) + 2;
 307                 if (nctrl->ncmd.s.param1) {
 308                         /* vfidx is 0 based, but vf_num (param1) is 1 based */
 309                         int vfidx = nctrl->ncmd.s.param1 - 1;
 310                         bool mac_is_admin_assigned = nctrl->ncmd.s.param2;
 311
 312                         if (mac_is_admin_assigned)
 313                                 netif_info(lio, probe, lio->netdev,
 314                                            "MAC Address %pM is configured for VF %d\n",
 315                                            mac, vfidx);
 316                 } else {
 317                         netif_info(lio, probe, lio->netdev,
 318                                    " MACAddr changed to %pM\n",
 319                                    mac);
 320                 }
 321                 break;
 322
 323         case OCTNET_CMD_GPIO_ACCESS:
 324                 netif_info(lio, probe, lio->netdev, "LED Flashing visual identification\n");
 325
 326                 break;
 327
 328         case OCTNET_CMD_ID_ACTIVE:
 329                 netif_info(lio, probe, lio->netdev, "LED Flashing visual identification\n");
 330
 331                 break;
 332
 333         case OCTNET_CMD_LRO_ENABLE:
 334                 dev_info(&oct->pci_dev->dev, "%s LRO Enabled\n", netdev->name);
 335                 break;
 336
 337         case OCTNET_CMD_LRO_DISABLE:
 338                 dev_info(&oct->pci_dev->dev, "%s LRO Disabled\n",
 339                          netdev->name);
 340                 break;
 341
 342         case OCTNET_CMD_VERBOSE_ENABLE:
 343                 dev_info(&oct->pci_dev->dev, "%s Firmware debug enabled\n",
 344                          netdev->name);
 345                 break;
 346
 347         case OCTNET_CMD_VERBOSE_DISABLE:
 348                 dev_info(&oct->pci_dev->dev, "%s Firmware debug disabled\n",
 349                          netdev->name);
 350                 break;
 351
 352         case OCTNET_CMD_VLAN_FILTER_CTL:
 353                 if (nctrl->ncmd.s.param1)
 354                         dev_info(&oct->pci_dev->dev,
 355                                  "%s VLAN filter enabled\n", netdev->name);
 356                 else
 357                         dev_info(&oct->pci_dev->dev,
 358                                  "%s VLAN filter disabled\n", netdev->name);
 359                 break;
 360
 361         case OCTNET_CMD_ADD_VLAN_FILTER:
 362                 dev_info(&oct->pci_dev->dev, "%s VLAN filter %d added\n",
 363                          netdev->name, nctrl->ncmd.s.param1);
 364                 break;
 365
 366         case OCTNET_CMD_DEL_VLAN_FILTER:
 367                 dev_info(&oct->pci_dev->dev, "%s VLAN filter %d removed\n",
 368                          netdev->name, nctrl->ncmd.s.param1);
 369                 break;
 370
 371         case OCTNET_CMD_SET_SETTINGS:
 372                 dev_info(&oct->pci_dev->dev, "%s settings changed\n",
 373                          netdev->name);
 374
 375                 break;
 376
 377         /* Case to handle "OCTNET_CMD_TNL_RX_CSUM_CTL"
 378          * Command passed by NIC driver
 379          */
 380         case OCTNET_CMD_TNL_RX_CSUM_CTL:
 381                 if (nctrl->ncmd.s.param1 == OCTNET_CMD_RXCSUM_ENABLE) {
 382                         netif_info(lio, probe, lio->netdev,
 383                                    "RX Checksum Offload Enabled\n");
 384                 } else if (nctrl->ncmd.s.param1 ==
 385                            OCTNET_CMD_RXCSUM_DISABLE) {
 386                         netif_info(lio, probe, lio->netdev,
 387                                    "RX Checksum Offload Disabled\n");
 388                 }
 389                 break;
 390
 391                 /* Case to handle "OCTNET_CMD_TNL_TX_CSUM_CTL"
 392                  * Command passed by NIC driver
 393                  */
 394         case OCTNET_CMD_TNL_TX_CSUM_CTL:
 395                 if (nctrl->ncmd.s.param1 == OCTNET_CMD_TXCSUM_ENABLE) {
 396                         netif_info(lio, probe, lio->netdev,
 397                                    "TX Checksum Offload Enabled\n");
 398                 } else if (nctrl->ncmd.s.param1 ==
 399                            OCTNET_CMD_TXCSUM_DISABLE) {
 400                         netif_info(lio, probe, lio->netdev,
 401                                    "TX Checksum Offload Disabled\n");
 402                 }
 403                 break;
 404
 405                 /* Case to handle "OCTNET_CMD_VXLAN_PORT_CONFIG"
 406                  * Command passed by NIC driver
 407                  */
 408         case OCTNET_CMD_VXLAN_PORT_CONFIG:
 409                 if (nctrl->ncmd.s.more == OCTNET_CMD_VXLAN_PORT_ADD) {
 410                         netif_info(lio, probe, lio->netdev,
 411                                    "VxLAN Destination UDP PORT:%d ADDED\n",
 412                                    nctrl->ncmd.s.param1);
 413                 } else if (nctrl->ncmd.s.more ==
 414                            OCTNET_CMD_VXLAN_PORT_DEL) {
 415                         netif_info(lio, probe, lio->netdev,
 416                                    "VxLAN Destination UDP PORT:%d DELETED\n",
 417                                    nctrl->ncmd.s.param1);
 418                 }
 419                 break;
 420
 421         case OCTNET_CMD_SET_FLOW_CTL:
 422                 netif_info(lio, probe, lio->netdev, "Set RX/TX flow control parameters\n");
 423                 break;
 424
 425         case OCTNET_CMD_QUEUE_COUNT_CTL:
 426                 netif_info(lio, probe, lio->netdev, "Queue count updated to %d\n",
 427                            nctrl->ncmd.s.param1);
 428                 break;
 429
 430         default:
 431                 dev_err(&oct->pci_dev->dev, "%s Unknown cmd %d\n", __func__,
 432                         nctrl->ncmd.s.cmd);
 433         }
 434 }
 435
 436 void octeon_pf_changed_vf_macaddr(struct octeon_device *oct, u8 *mac)
 437 {
 438         bool macaddr_changed = false;
 439         struct net_device *netdev;
 440         struct lio *lio;
 441
 442         rtnl_lock();
 443
 444         netdev = oct->props[0].netdev;
 445         lio = GET_LIO(netdev);
 446
 447         lio->linfo.macaddr_is_admin_asgnd = true;
 448
 449         if (!ether_addr_equal(netdev->dev_addr, mac)) {
 450                 macaddr_changed = true;
 451                 ether_addr_copy(netdev->dev_addr, mac);
 452                 ether_addr_copy(((u8 *)&lio->linfo.hw_addr) + 2, mac);
 453                 call_netdevice_notifiers(NETDEV_CHANGEADDR, netdev);
 454         }
 455
 456         rtnl_unlock();
 457
 458         if (macaddr_changed)
 459                 dev_info(&oct->pci_dev->dev,
 460                          "PF changed VF's MAC address to %pM\n", mac);
 461
 462         /* no need to notify the firmware of the macaddr change because
 463          * the PF did that already
 464          */
 465 }
 466
 467 static void octnet_poll_check_rxq_oom_status(struct work_struct *work)
 468 {
 469         struct cavium_wk *wk = (struct cavium_wk *)work;
 470         struct lio *lio = (struct lio *)wk->ctxptr;
 471         struct octeon_device *oct = lio->oct_dev;
 472         struct octeon_droq *droq;
 473         int q, q_no = 0;
 474
 475         if (ifstate_check(lio, LIO_IFSTATE_RUNNING)) {
 476                 for (q = 0; q < lio->linfo.num_rxpciq; q++) {
 477                         q_no = lio->linfo.rxpciq[q].s.q_no;
 478                         droq = oct->droq[q_no];
 479                         if (!droq)
 480                                 continue;
 481                         octeon_droq_check_oom(droq);
 482                 }
 483         }
 484         queue_delayed_work(lio->rxq_status_wq.wq,
 485                            &lio->rxq_status_wq.wk.work,
 486                            msecs_to_jiffies(LIO_OOM_POLL_INTERVAL_MS));
 487 }
 488
 489 int setup_rx_oom_poll_fn(struct net_device *netdev)
 490 {
 491         struct lio *lio = GET_LIO(netdev);
 492         struct octeon_device *oct = lio->oct_dev;
 493
 494         lio->rxq_status_wq.wq = alloc_workqueue("rxq-oom-status",
 495                                                 WQ_MEM_RECLAIM, 0);
 496         if (!lio->rxq_status_wq.wq) {
 497                 dev_err(&oct->pci_dev->dev, "unable to create cavium rxq oom status wq\n");
 498                 return -ENOMEM;
 499         }
 500         INIT_DELAYED_WORK(&lio->rxq_status_wq.wk.work,
 501                           octnet_poll_check_rxq_oom_status);
 502         lio->rxq_status_wq.wk.ctxptr = lio;
 503         queue_delayed_work(lio->rxq_status_wq.wq,
 504                            &lio->rxq_status_wq.wk.work,
 505                            msecs_to_jiffies(LIO_OOM_POLL_INTERVAL_MS));
 506         return 0;
 507 }
 508
 509 void cleanup_rx_oom_poll_fn(struct net_device *netdev)
 510 {
 511         struct lio *lio = GET_LIO(netdev);
 512
 513         if (lio->rxq_status_wq.wq) {
 514                 cancel_delayed_work_sync(&lio->rxq_status_wq.wk.work);
 515                 flush_workqueue(lio->rxq_status_wq.wq);
 516                 destroy_workqueue(lio->rxq_status_wq.wq);
 517         }
 518 }
 519
 520 /* Runs in interrupt context. */
 521 static void lio_update_txq_status(struct octeon_device *oct, int iq_num)
 522 {
 523         struct octeon_instr_queue *iq = oct->instr_queue[iq_num];
 524         struct net_device *netdev;
 525         struct lio *lio;
 526
 527         netdev = oct->props[iq->ifidx].netdev;
 528
 529         /* This is needed because the first IQ does not have
 530          * a netdev associated with it.
 531          */
 532         if (!netdev)
 533                 return;
 534
 535         lio = GET_LIO(netdev);
 536         if (__netif_subqueue_stopped(netdev, iq->q_index) &&
 537             lio->linfo.link.s.link_up &&
 538             (!octnet_iq_is_full(oct, iq_num))) {
 539                 netif_wake_subqueue(netdev, iq->q_index);
 540                 INCR_INSTRQUEUE_PKT_COUNT(lio->oct_dev, iq_num,
 541                                           tx_restart, 1);
 542         }
 543 }
 544
 545 /**
 546  * \brief Setup output queue
 547  * @param oct octeon device
 548  * @param q_no which queue
 549  * @param num_descs how many descriptors
 550  * @param desc_size size of each descriptor
 551  * @param app_ctx application context
 552  */
 553 static int octeon_setup_droq(struct octeon_device *oct, int q_no, int num_descs,
 554                              int desc_size, void *app_ctx)
 555 {
 556         int ret_val;
 557
 558         dev_dbg(&oct->pci_dev->dev, "Creating Droq: %d\n", q_no);
 559         /* droq creation and local register settings. */
 560         ret_val = octeon_create_droq(oct, q_no, num_descs, desc_size, app_ctx);
 561         if (ret_val < 0)
 562                 return ret_val;
 563
 564         if (ret_val == 1) {
 565                 dev_dbg(&oct->pci_dev->dev, "Using default droq %d\n", q_no);
 566                 return 0;
 567         }
 568
 569         /* Enable the droq queues */
 570         octeon_set_droq_pkt_op(oct, q_no, 1);
 571
 572         /* Send Credit for Octeon Output queues. Credits are always
 573          * sent after the output queue is enabled.
 574          */
 575         writel(oct->droq[q_no]->max_count, oct->droq[q_no]->pkts_credit_reg);
 576
 577         return ret_val;
 578 }
 579
 580 /** Routine to push packets arriving on Octeon interface upto network layer.
 581  * @param oct_id   - octeon device id.
 582  * @param skbuff   - skbuff struct to be passed to network layer.
 583  * @param len      - size of total data received.
 584  * @param rh       - Control header associated with the packet
 585  * @param param    - additional control data with the packet
 586  * @param arg      - farg registered in droq_ops
 587  */
 588 static void
 589 liquidio_push_packet(u32 octeon_id __attribute__((unused)),
 590                      void *skbuff,
 591                      u32 len,
 592                      union octeon_rh *rh,
 593                      void *param,
 594                      void *arg)
 595 {
 596         struct net_device *netdev = (struct net_device *)arg;
 597         struct octeon_droq *droq =
 598             container_of(param, struct octeon_droq, napi);
 599         struct sk_buff *skb = (struct sk_buff *)skbuff;
 600         struct skb_shared_hwtstamps *shhwtstamps;
 601         struct napi_struct *napi = param;
 602         u16 vtag = 0;
 603         u32 r_dh_off;
 604         u64 ns;
 605
 606         if (netdev) {
 607                 struct lio *lio = GET_LIO(netdev);
 608                 struct octeon_device *oct = lio->oct_dev;
 609
 610                 /* Do not proceed if the interface is not in RUNNING state. */
 611                 if (!ifstate_check(lio, LIO_IFSTATE_RUNNING)) {
 612                         recv_buffer_free(skb);
 613                         droq->stats.rx_dropped++;
 614                         return;
 615                 }
 616
 617                 skb->dev = netdev;
 618
 619                 skb_record_rx_queue(skb, droq->q_no);
 620                 if (likely(len > MIN_SKB_SIZE)) {
 621                         struct octeon_skb_page_info *pg_info;
 622                         unsigned char *va;
 623
 624                         pg_info = ((struct octeon_skb_page_info *)(skb->cb));
 625                         if (pg_info->page) {
 626                                 /* For Paged allocation use the frags */
 627                                 va = page_address(pg_info->page) +
 628                                         pg_info->page_offset;
 629                                 memcpy(skb->data, va, MIN_SKB_SIZE);
 630                                 skb_put(skb, MIN_SKB_SIZE);
 631                                 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags,
 632                                                 pg_info->page,
 633                                                 pg_info->page_offset +
 634                                                 MIN_SKB_SIZE,
 635                                                 len - MIN_SKB_SIZE,
 636                                                 LIO_RXBUFFER_SZ);
 637                         }
 638                 } else {
 639                         struct octeon_skb_page_info *pg_info =
 640                                 ((struct octeon_skb_page_info *)(skb->cb));
 641                         skb_copy_to_linear_data(skb, page_address(pg_info->page)
 642                                                 + pg_info->page_offset, len);
 643                         skb_put(skb, len);
 644                         put_page(pg_info->page);
 645                 }
 646
 647                 r_dh_off = (rh->r_dh.len - 1) * BYTES_PER_DHLEN_UNIT;
 648
 649                 if (oct->ptp_enable) {
 650                         if (rh->r_dh.has_hwtstamp) {
 651                                 /* timestamp is included from the hardware at
 652                                  * the beginning of the packet.
 653                                  */
 654                                 if (ifstate_check
 655                                         (lio,
 656                                          LIO_IFSTATE_RX_TIMESTAMP_ENABLED)) {
 657                                         /* Nanoseconds are in the first 64-bits
 658                                          * of the packet.
 659                                          */
 660                                         memcpy(&ns, (skb->data + r_dh_off),
 661                                                sizeof(ns));
 662                                         r_dh_off -= BYTES_PER_DHLEN_UNIT;
 663                                         shhwtstamps = skb_hwtstamps(skb);
 664                                         shhwtstamps->hwtstamp =
 665                                                 ns_to_ktime(ns +
 666                                                             lio->ptp_adjust);
 667                                 }
 668                         }
 669                 }
 670
 671                 if (rh->r_dh.has_hash) {
 672                         __be32 *hash_be = (__be32 *)(skb->data + r_dh_off);
 673                         u32 hash = be32_to_cpu(*hash_be);
 674
 675                         skb_set_hash(skb, hash, PKT_HASH_TYPE_L4);
 676                         r_dh_off -= BYTES_PER_DHLEN_UNIT;
 677                 }
 678
 679                 skb_pull(skb, rh->r_dh.len * BYTES_PER_DHLEN_UNIT);
 680                 skb->protocol = eth_type_trans(skb, skb->dev);
 681
 682                 if ((netdev->features & NETIF_F_RXCSUM) &&
 683                     (((rh->r_dh.encap_on) &&
 684                       (rh->r_dh.csum_verified & CNNIC_TUN_CSUM_VERIFIED)) ||
 685                      (!(rh->r_dh.encap_on) &&
 686                       (rh->r_dh.csum_verified & CNNIC_CSUM_VERIFIED))))
 687                         /* checksum has already been verified */
 688                         skb->ip_summed = CHECKSUM_UNNECESSARY;
 689                 else
 690                         skb->ip_summed = CHECKSUM_NONE;
 691
 692                 /* Setting Encapsulation field on basis of status received
 693                  * from the firmware
 694                  */
 695                 if (rh->r_dh.encap_on) {
 696                         skb->encapsulation = 1;
 697                         skb->csum_level = 1;
 698                         droq->stats.rx_vxlan++;
 699                 }
 700
 701                 /* inbound VLAN tag */
 702                 if ((netdev->features & NETIF_F_HW_VLAN_CTAG_RX) &&
 703                     rh->r_dh.vlan) {
 704                         u16 priority = rh->r_dh.priority;
 705                         u16 vid = rh->r_dh.vlan;
 706
 707                         vtag = (priority << VLAN_PRIO_SHIFT) | vid;
 708                         __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vtag);
 709                 }
 710
 711                 napi_gro_receive(napi, skb);
 712
 713                 droq->stats.rx_bytes_received += len -
 714                         rh->r_dh.len * BYTES_PER_DHLEN_UNIT;
 715                 droq->stats.rx_pkts_received++;
 716         } else {
 717                 recv_buffer_free(skb);
 718         }
 719 }
 720
 721 /**
 722  * \brief wrapper for calling napi_schedule
 723  * @param param parameters to pass to napi_schedule
 724  *
 725  * Used when scheduling on different CPUs
 726  */
 727 static void napi_schedule_wrapper(void *param)
 728 {
 729         struct napi_struct *napi = param;
 730
 731         napi_schedule(napi);
 732 }
 733
 734 /**
 735  * \brief callback when receive interrupt occurs and we are in NAPI mode
 736  * @param arg pointer to octeon output queue
 737  */
 738 static void liquidio_napi_drv_callback(void *arg)
 739 {
 740         struct octeon_device *oct;
 741         struct octeon_droq *droq = arg;
 742         int this_cpu = smp_processor_id();
 743
 744         oct = droq->oct_dev;
 745
 746         if (OCTEON_CN23XX_PF(oct) || OCTEON_CN23XX_VF(oct) ||
 747             droq->cpu_id == this_cpu) {
 748                 napi_schedule_irqoff(&droq->napi);
 749         } else {
 750                 call_single_data_t *csd = &droq->csd;
 751
 752                 csd->func = napi_schedule_wrapper;
 753                 csd->info = &droq->napi;
 754                 csd->flags = 0;
 755
 756                 smp_call_function_single_async(droq->cpu_id, csd);
 757         }
 758 }
 759
 760 /**
 761  * \brief Entry point for NAPI polling
 762  * @param napi NAPI structure
 763  * @param budget maximum number of items to process
 764  */
 765 static int liquidio_napi_poll(struct napi_struct *napi, int budget)
 766 {
 767         struct octeon_instr_queue *iq;
 768         struct octeon_device *oct;
 769         struct octeon_droq *droq;
 770         int tx_done = 0, iq_no;
 771         int work_done;
 772
 773         droq = container_of(napi, struct octeon_droq, napi);
 774         oct = droq->oct_dev;
 775         iq_no = droq->q_no;
 776
 777         /* Handle Droq descriptors */
 778         work_done = octeon_droq_process_poll_pkts(oct, droq, budget);
 779
 780         /* Flush the instruction queue */
 781         iq = oct->instr_queue[iq_no];
 782         if (iq) {
 783                 /* TODO: move this check to inside octeon_flush_iq,
 784                  * once check_db_timeout is removed
 785                  */
 786                 if (atomic_read(&iq->instr_pending))
 787                         /* Process iq buffers with in the budget limits */
 788                         tx_done = octeon_flush_iq(oct, iq, budget);
 789                 else
 790                         tx_done = 1;
 791                 /* Update iq read-index rather than waiting for next interrupt.
 792                  * Return back if tx_done is false.
 793                  */
 794                 /* sub-queue status update */
 795                 lio_update_txq_status(oct, iq_no);
 796         } else {
 797                 dev_err(&oct->pci_dev->dev, "%s:  iq (%d) num invalid\n",
 798                         __func__, iq_no);
 799         }
 800
 801 #define MAX_REG_CNT  2000000U
 802         /* force enable interrupt if reg cnts are high to avoid wraparound */
 803         if ((work_done < budget && tx_done) ||
 804             (iq && iq->pkt_in_done >= MAX_REG_CNT) ||
 805             (droq->pkt_count >= MAX_REG_CNT)) {
 806                 tx_done = 1;
 807                 napi_complete_done(napi, work_done);
 808
 809                 octeon_enable_irq(droq->oct_dev, droq->q_no);
 810                 return 0;
 811         }
 812
 813         return (!tx_done) ? (budget) : (work_done);
 814 }
 815
 816 /**
 817  * \brief Setup input and output queues
 818  * @param octeon_dev octeon device
 819  * @param ifidx Interface index
 820  *
 821  * Note: Queues are with respect to the octeon device. Thus
 822  * an input queue is for egress packets, and output queues
 823  * are for ingress packets.
 824  */
 825 int liquidio_setup_io_queues(struct octeon_device *octeon_dev, int ifidx,
 826                              u32 num_iqs, u32 num_oqs)
 827 {
 828         struct octeon_droq_ops droq_ops;
 829         struct net_device *netdev;
 830         struct octeon_droq *droq;
 831         struct napi_struct *napi;
 832         int cpu_id_modulus;
 833         int num_tx_descs;
 834         struct lio *lio;
 835         int retval = 0;
 836         int q, q_no;
 837         int cpu_id;
 838
 839         netdev = octeon_dev->props[ifidx].netdev;
 840
 841         lio = GET_LIO(netdev);
 842
 843         memset(&droq_ops, 0, sizeof(struct octeon_droq_ops));
 844
 845         droq_ops.fptr = liquidio_push_packet;
 846         droq_ops.farg = netdev;
 847
 848         droq_ops.poll_mode = 1;
 849         droq_ops.napi_fn = liquidio_napi_drv_callback;
 850         cpu_id = 0;
 851         cpu_id_modulus = num_present_cpus();
 852
 853         /* set up DROQs. */
 854         for (q = 0; q < num_oqs; q++) {
 855                 q_no = lio->linfo.rxpciq[q].s.q_no;
 856                 dev_dbg(&octeon_dev->pci_dev->dev,
 857                         "%s index:%d linfo.rxpciq.s.q_no:%d\n",
 858                         __func__, q, q_no);
 859                 retval = octeon_setup_droq(
 860                     octeon_dev, q_no,
 861                     CFG_GET_NUM_RX_DESCS_NIC_IF(octeon_get_conf(octeon_dev),
 862                                                 lio->ifidx),
 863                     CFG_GET_NUM_RX_BUF_SIZE_NIC_IF(octeon_get_conf(octeon_dev),
 864                                                    lio->ifidx),
 865                     NULL);
 866                 if (retval) {
 867                         dev_err(&octeon_dev->pci_dev->dev,
 868                                 "%s : Runtime DROQ(RxQ) creation failed.\n",
 869                                 __func__);
 870                         return 1;
 871                 }
 872
 873                 droq = octeon_dev->droq[q_no];
 874                 napi = &droq->napi;
 875                 dev_dbg(&octeon_dev->pci_dev->dev, "netif_napi_add netdev:%llx oct:%llx\n",
 876                         (u64)netdev, (u64)octeon_dev);
 877                 netif_napi_add(netdev, napi, liquidio_napi_poll, 64);
 878
 879                 /* designate a CPU for this droq */
 880                 droq->cpu_id = cpu_id;
 881                 cpu_id++;
 882                 if (cpu_id >= cpu_id_modulus)
 883                         cpu_id = 0;
 884
 885                 octeon_register_droq_ops(octeon_dev, q_no, &droq_ops);
 886         }
 887
 888         if (OCTEON_CN23XX_PF(octeon_dev) || OCTEON_CN23XX_VF(octeon_dev)) {
 889                 /* 23XX PF/VF can send/recv control messages (via the first
 890                  * PF/VF-owned droq) from the firmware even if the ethX
 891                  * interface is down, so that's why poll_mode must be off
 892                  * for the first droq.
 893                  */
 894                 octeon_dev->droq[0]->ops.poll_mode = 0;
 895         }
 896
 897         /* set up IQs. */
 898         for (q = 0; q < num_iqs; q++) {
 899                 num_tx_descs = CFG_GET_NUM_TX_DESCS_NIC_IF(
 900                     octeon_get_conf(octeon_dev), lio->ifidx);
 901                 retval = octeon_setup_iq(octeon_dev, ifidx, q,
 902                                          lio->linfo.txpciq[q], num_tx_descs,
 903                                          netdev_get_tx_queue(netdev, q));
 904                 if (retval) {
 905                         dev_err(&octeon_dev->pci_dev->dev,
 906                                 " %s : Runtime IQ(TxQ) creation failed.\n",
 907                                 __func__);
 908                         return 1;
 909                 }
 910
 911                 /* XPS */
 912                 if (!OCTEON_CN23XX_VF(octeon_dev) && octeon_dev->msix_on &&
 913                     octeon_dev->ioq_vector) {
 914                         struct octeon_ioq_vector    *ioq_vector;
 915
 916                         ioq_vector = &octeon_dev->ioq_vector[q];
 917                         netif_set_xps_queue(netdev,
 918                                             &ioq_vector->affinity_mask,
 919                                             ioq_vector->iq_index);
 920                 }
 921         }
 922
 923         return 0;
 924 }
 925
 926 static
 927 int liquidio_schedule_msix_droq_pkt_handler(struct octeon_droq *droq, u64 ret)
 928 {
 929         struct octeon_device *oct = droq->oct_dev;
 930         struct octeon_device_priv *oct_priv =
 931             (struct octeon_device_priv *)oct->priv;
 932
 933         if (droq->ops.poll_mode) {
 934                 droq->ops.napi_fn(droq);
 935         } else {
 936                 if (ret & MSIX_PO_INT) {
 937                         if (OCTEON_CN23XX_VF(oct))
 938                                 dev_err(&oct->pci_dev->dev,
 939                                         "should not come here should not get rx when poll mode = 0 for vf\n");
 940                         tasklet_schedule(&oct_priv->droq_tasklet);
 941                         return 1;
 942                 }
 943                 /* this will be flushed periodically by check iq db */
 944                 if (ret & MSIX_PI_INT)
 945                         return 0;
 946         }
 947
 948         return 0;
 949 }
 950
 951 irqreturn_t
 952 liquidio_msix_intr_handler(int irq __attribute__((unused)), void *dev)
 953 {
 954         struct octeon_ioq_vector *ioq_vector = (struct octeon_ioq_vector *)dev;
 955         struct octeon_device *oct = ioq_vector->oct_dev;
 956         struct octeon_droq *droq = oct->droq[ioq_vector->droq_index];
 957         u64 ret;
 958
 959         ret = oct->fn_list.msix_interrupt_handler(ioq_vector);
 960
 961         if (ret & MSIX_PO_INT || ret & MSIX_PI_INT)
 962                 liquidio_schedule_msix_droq_pkt_handler(droq, ret);
 963
 964         return IRQ_HANDLED;
 965 }
 966
 967 /**
 968  * \brief Droq packet processor sceduler
 969  * @param oct octeon device
 970  */
 971 static void liquidio_schedule_droq_pkt_handlers(struct octeon_device *oct)
 972 {
 973         struct octeon_device_priv *oct_priv =
 974                 (struct octeon_device_priv *)oct->priv;
 975         struct octeon_droq *droq;
 976         u64 oq_no;
 977
 978         if (oct->int_status & OCT_DEV_INTR_PKT_DATA) {
 979                 for (oq_no = 0; oq_no < MAX_OCTEON_OUTPUT_QUEUES(oct);
 980                      oq_no++) {
 981                         if (!(oct->droq_intr & BIT_ULL(oq_no)))
 982                                 continue;
 983
 984                         droq = oct->droq[oq_no];
 985
 986                         if (droq->ops.poll_mode) {
 987                                 droq->ops.napi_fn(droq);
 988                                 oct_priv->napi_mask |= BIT_ULL(oq_no);
 989                         } else {
 990                                 tasklet_schedule(&oct_priv->droq_tasklet);
 991                         }
 992                 }
 993         }
 994 }
 995
 996 /**
 997  * \brief Interrupt handler for octeon
 998  * @param irq unused
 999  * @param dev octeon device
1000  */
1001 static
1002 irqreturn_t liquidio_legacy_intr_handler(int irq __attribute__((unused)),
1003                                          void *dev)
1004 {
1005         struct octeon_device *oct = (struct octeon_device *)dev;
1006         irqreturn_t ret;
1007
1008         /* Disable our interrupts for the duration of ISR */
1009         oct->fn_list.disable_interrupt(oct, OCTEON_ALL_INTR);
1010
1011         ret = oct->fn_list.process_interrupt_regs(oct);
1012
1013         if (ret == IRQ_HANDLED)
1014                 liquidio_schedule_droq_pkt_handlers(oct);
1015
1016         /* Re-enable our interrupts  */
1017         if (!(atomic_read(&oct->status) == OCT_DEV_IN_RESET))
1018                 oct->fn_list.enable_interrupt(oct, OCTEON_ALL_INTR);
1019
1020         return ret;
1021 }
1022
1023 /**
1024  * \brief Setup interrupt for octeon device
1025  * @param oct octeon device
1026  *
1027  *  Enable interrupt in Octeon device as given in the PCI interrupt mask.
1028  */
1029 int octeon_setup_interrupt(struct octeon_device *oct, u32 num_ioqs)
1030 {
1031         struct msix_entry *msix_entries;
1032         char *queue_irq_names = NULL;
1033         int i, num_interrupts = 0;
1034         int num_alloc_ioq_vectors;
1035         char *aux_irq_name = NULL;
1036         int num_ioq_vectors;
1037         int irqret, err;
1038
1039         if (oct->msix_on) {
1040                 oct->num_msix_irqs = num_ioqs;
1041                 if (OCTEON_CN23XX_PF(oct)) {
1042                         num_interrupts = MAX_IOQ_INTERRUPTS_PER_PF + 1;
1043
1044                         /* one non ioq interrupt for handling
1045                          * sli_mac_pf_int_sum
1046                          */
1047                         oct->num_msix_irqs += 1;
1048                 } else if (OCTEON_CN23XX_VF(oct)) {
1049                         num_interrupts = MAX_IOQ_INTERRUPTS_PER_VF;
1050                 }
1051
1052                 /* allocate storage for the names assigned to each irq */
1053                 oct->irq_name_storage =
1054                         kcalloc(num_interrupts, INTRNAMSIZ, GFP_KERNEL);
1055                 if (!oct->irq_name_storage) {
1056                         dev_err(&oct->pci_dev->dev, "Irq name storage alloc failed...\n");
1057                         return -ENOMEM;
1058                 }
1059
1060                 queue_irq_names = oct->irq_name_storage;
1061
1062                 if (OCTEON_CN23XX_PF(oct))
1063                         aux_irq_name = &queue_irq_names
1064                                 [IRQ_NAME_OFF(MAX_IOQ_INTERRUPTS_PER_PF)];
1065
1066                 oct->msix_entries = kcalloc(oct->num_msix_irqs,
1067                                             sizeof(struct msix_entry),
1068                                             GFP_KERNEL);
1069                 if (!oct->msix_entries) {
1070                         dev_err(&oct->pci_dev->dev, "Memory Alloc failed...\n");
1071                         kfree(oct->irq_name_storage);
1072                         oct->irq_name_storage = NULL;
1073                         return -ENOMEM;
1074                 }
1075
1076                 msix_entries = (struct msix_entry *)oct->msix_entries;
1077
1078                 /*Assumption is that pf msix vectors start from pf srn to pf to
1079                  * trs and not from 0. if not change this code
1080                  */
1081                 if (OCTEON_CN23XX_PF(oct)) {
1082                         for (i = 0; i < oct->num_msix_irqs - 1; i++)
1083                                 msix_entries[i].entry =
1084                                         oct->sriov_info.pf_srn + i;
1085
1086                         msix_entries[oct->num_msix_irqs - 1].entry =
1087                                 oct->sriov_info.trs;
1088                 } else if (OCTEON_CN23XX_VF(oct)) {
1089                         for (i = 0; i < oct->num_msix_irqs; i++)
1090                                 msix_entries[i].entry = i;
1091                 }
1092                 num_alloc_ioq_vectors = pci_enable_msix_range(
1093                                                 oct->pci_dev, msix_entries,
1094                                                 oct->num_msix_irqs,
1095                                                 oct->num_msix_irqs);
1096                 if (num_alloc_ioq_vectors < 0) {
1097                         dev_err(&oct->pci_dev->dev, "unable to Allocate MSI-X interrupts\n");
1098                         kfree(oct->msix_entries);
1099                         oct->msix_entries = NULL;
1100                         kfree(oct->irq_name_storage);
1101                         oct->irq_name_storage = NULL;
1102                         return num_alloc_ioq_vectors;
1103                 }
1104
1105                 dev_dbg(&oct->pci_dev->dev, "OCTEON: Enough MSI-X interrupts are allocated...\n");
1106
1107                 num_ioq_vectors = oct->num_msix_irqs;
1108                 /** For PF, there is one non-ioq interrupt handler */
1109                 if (OCTEON_CN23XX_PF(oct)) {
1110                         num_ioq_vectors -= 1;
1111
1112                         snprintf(aux_irq_name, INTRNAMSIZ,
1113                                  "LiquidIO%u-pf%u-aux", oct->octeon_id,
1114                                  oct->pf_num);
1115                         irqret = request_irq(
1116                                         msix_entries[num_ioq_vectors].vector,
1117                                         liquidio_legacy_intr_handler, 0,
1118                                         aux_irq_name, oct);
1119                         if (irqret) {
1120                                 dev_err(&oct->pci_dev->dev,
1121                                         "Request_irq failed for MSIX interrupt Error: %d\n",
1122                                         irqret);
1123                                 pci_disable_msix(oct->pci_dev);
1124                                 kfree(oct->msix_entries);
1125                                 kfree(oct->irq_name_storage);
1126                                 oct->irq_name_storage = NULL;
1127                                 oct->msix_entries = NULL;
1128                                 return irqret;
1129                         }
1130                 }
1131                 for (i = 0 ; i < num_ioq_vectors ; i++) {
1132                         if (OCTEON_CN23XX_PF(oct))
1133                                 snprintf(&queue_irq_names[IRQ_NAME_OFF(i)],
1134                                          INTRNAMSIZ, "LiquidIO%u-pf%u-rxtx-%u",
1135                                          oct->octeon_id, oct->pf_num, i);
1136
1137                         if (OCTEON_CN23XX_VF(oct))
1138                                 snprintf(&queue_irq_names[IRQ_NAME_OFF(i)],
1139                                          INTRNAMSIZ, "LiquidIO%u-vf%u-rxtx-%u",
1140                                          oct->octeon_id, oct->vf_num, i);
1141
1142                         irqret = request_irq(msix_entries[i].vector,
1143                                              liquidio_msix_intr_handler, 0,
1144                                              &queue_irq_names[IRQ_NAME_OFF(i)],
1145                                              &oct->ioq_vector[i]);
1146
1147                         if (irqret) {
1148                                 dev_err(&oct->pci_dev->dev,
1149                                         "Request_irq failed for MSIX interrupt Error: %d\n",
1150                                         irqret);
1151                                 /** Freeing the non-ioq irq vector here . */
1152                                 free_irq(msix_entries[num_ioq_vectors].vector,
1153                                          oct);
1154
1155                                 while (i) {
1156                                         i--;
1157                                         /** clearing affinity mask. */
1158                                         irq_set_affinity_hint(
1159                                                       msix_entries[i].vector,
1160                                                       NULL);
1161                                         free_irq(msix_entries[i].vector,
1162                                                  &oct->ioq_vector[i]);
1163                                 }
1164                                 pci_disable_msix(oct->pci_dev);
1165                                 kfree(oct->msix_entries);
1166                                 kfree(oct->irq_name_storage);
1167                                 oct->irq_name_storage = NULL;
1168                                 oct->msix_entries = NULL;
1169                                 return irqret;
1170                         }
1171                         oct->ioq_vector[i].vector = msix_entries[i].vector;
1172                         /* assign the cpu mask for this msix interrupt vector */
1173                         irq_set_affinity_hint(msix_entries[i].vector,
1174                                               &oct->ioq_vector[i].affinity_mask
1175                                               );
1176                 }
1177                 dev_dbg(&oct->pci_dev->dev, "OCTEON[%d]: MSI-X enabled\n",
1178                         oct->octeon_id);
1179         } else {
1180                 err = pci_enable_msi(oct->pci_dev);
1181                 if (err)
1182                         dev_warn(&oct->pci_dev->dev, "Reverting to legacy interrupts. Error: %d\n",
1183                                  err);
1184                 else
1185                         oct->flags |= LIO_FLAG_MSI_ENABLED;
1186
1187                 /* allocate storage for the names assigned to the irq */
1188                 oct->irq_name_storage = kcalloc(1, INTRNAMSIZ, GFP_KERNEL);
1189                 if (!oct->irq_name_storage)
1190                         return -ENOMEM;
1191
1192                 queue_irq_names = oct->irq_name_storage;
1193
1194                 if (OCTEON_CN23XX_PF(oct))
1195                         snprintf(&queue_irq_names[IRQ_NAME_OFF(0)], INTRNAMSIZ,
1196                                  "LiquidIO%u-pf%u-rxtx-%u",
1197                                  oct->octeon_id, oct->pf_num, 0);
1198
1199                 if (OCTEON_CN23XX_VF(oct))
1200                         snprintf(&queue_irq_names[IRQ_NAME_OFF(0)], INTRNAMSIZ,
1201                                  "LiquidIO%u-vf%u-rxtx-%u",
1202                                  oct->octeon_id, oct->vf_num, 0);
1203
1204                 irqret = request_irq(oct->pci_dev->irq,
1205                                      liquidio_legacy_intr_handler,
1206                                      IRQF_SHARED,
1207                                      &queue_irq_names[IRQ_NAME_OFF(0)], oct);
1208                 if (irqret) {
1209                         if (oct->flags & LIO_FLAG_MSI_ENABLED)
1210                                 pci_disable_msi(oct->pci_dev);
1211                         dev_err(&oct->pci_dev->dev, "Request IRQ failed with code: %d\n",
1212                                 irqret);
1213                         kfree(oct->irq_name_storage);
1214                         oct->irq_name_storage = NULL;
1215                         return irqret;
1216                 }
1217         }
1218         return 0;
1219 }
1220
1221 static void liquidio_change_mtu_completion(struct octeon_device *oct,
1222                                            u32 status, void *buf)
1223 {
1224         struct octeon_soft_command *sc = (struct octeon_soft_command *)buf;
1225         struct liquidio_if_cfg_context *ctx;
1226
1227         ctx  = (struct liquidio_if_cfg_context *)sc->ctxptr;
1228
1229         if (status) {
1230                 dev_err(&oct->pci_dev->dev, "MTU change failed. Status: %llx\n",
1231                         CVM_CAST64(status));
1232                 WRITE_ONCE(ctx->cond, LIO_CHANGE_MTU_FAIL);
1233         } else {
1234                 WRITE_ONCE(ctx->cond, LIO_CHANGE_MTU_SUCCESS);
1235         }
1236
1237         /* This barrier is required to be sure that the response has been
1238          * written fully before waking up the handler
1239          */
1240         wmb();
1241
1242         wake_up_interruptible(&ctx->wc);
1243 }
1244
1245 /**
1246  * \brief Net device change_mtu
1247  * @param netdev network device
1248  */
1249 int liquidio_change_mtu(struct net_device *netdev, int new_mtu)
1250 {
1251         struct lio *lio = GET_LIO(netdev);
1252         struct octeon_device *oct = lio->oct_dev;
1253         struct liquidio_if_cfg_context *ctx;
1254         struct octeon_soft_command *sc;
1255         union octnet_cmd *ncmd;
1256         int ctx_size;
1257         int ret = 0;
1258
1259         ctx_size = sizeof(struct liquidio_if_cfg_context);
1260         sc = (struct octeon_soft_command *)
1261                 octeon_alloc_soft_command(oct, OCTNET_CMD_SIZE, 16, ctx_size);
1262
1263         ncmd = (union octnet_cmd *)sc->virtdptr;
1264         ctx  = (struct liquidio_if_cfg_context *)sc->ctxptr;
1265
1266         WRITE_ONCE(ctx->cond, 0);
1267         ctx->octeon_id = lio_get_device_id(oct);
1268         init_waitqueue_head(&ctx->wc);
1269
1270         ncmd->u64 = 0;
1271         ncmd->s.cmd = OCTNET_CMD_CHANGE_MTU;
1272         ncmd->s.param1 = new_mtu;
1273
1274         octeon_swap_8B_data((u64 *)ncmd, (OCTNET_CMD_SIZE >> 3));
1275
1276         sc->iq_no = lio->linfo.txpciq[0].s.q_no;
1277
1278         octeon_prepare_soft_command(oct, sc, OPCODE_NIC,
1279                                     OPCODE_NIC_CMD, 0, 0, 0);
1280
1281         sc->callback = liquidio_change_mtu_completion;
1282         sc->callback_arg = sc;
1283         sc->wait_time = 100;
1284
1285         ret = octeon_send_soft_command(oct, sc);
1286         if (ret == IQ_SEND_FAILED) {
1287                 netif_info(lio, rx_err, lio->netdev, "Failed to change MTU\n");
1288                 return -EINVAL;
1289         }
1290         /* Sleep on a wait queue till the cond flag indicates that the
1291          * response arrived or timed-out.
1292          */
1293         if (sleep_cond(&ctx->wc, &ctx->cond) == -EINTR ||
1294             ctx->cond == LIO_CHANGE_MTU_FAIL) {
1295                 octeon_free_soft_command(oct, sc);
1296                 return -EINVAL;
1297         }
1298
1299         netdev->mtu = new_mtu;
1300         lio->mtu = new_mtu;
1301
1302         octeon_free_soft_command(oct, sc);
1303         return 0;
1304 }
1305
1306 int lio_wait_for_clean_oq(struct octeon_device *oct)
1307 {
1308         int retry = 100, pending_pkts = 0;
1309         int idx;
1310
1311         do {
1312                 pending_pkts = 0;
1313
1314                 for (idx = 0; idx < MAX_OCTEON_OUTPUT_QUEUES(oct); idx++) {
1315                         if (!(oct->io_qmask.oq & BIT_ULL(idx)))
1316                                 continue;
1317                         pending_pkts +=
1318                                 atomic_read(&oct->droq[idx]->pkts_pending);
1319                 }
1320
1321                 if (pending_pkts > 0)
1322                         schedule_timeout_uninterruptible(1);
1323
1324         } while (retry-- && pending_pkts);
1325
1326         return pending_pkts;
1327 }
1328
1329 static void
1330 octnet_nic_stats_callback(struct octeon_device *oct_dev,
1331                           u32 status, void *ptr)
1332 {
1333         struct octeon_soft_command *sc = (struct octeon_soft_command *)ptr;
1334         struct oct_nic_stats_resp *resp =
1335             (struct oct_nic_stats_resp *)sc->virtrptr;
1336         struct oct_nic_stats_ctrl *ctrl =
1337             (struct oct_nic_stats_ctrl *)sc->ctxptr;
1338         struct nic_rx_stats *rsp_rstats = &resp->stats.fromwire;
1339         struct nic_tx_stats *rsp_tstats = &resp->stats.fromhost;
1340         struct nic_rx_stats *rstats = &oct_dev->link_stats.fromwire;
1341         struct nic_tx_stats *tstats = &oct_dev->link_stats.fromhost;
1342
1343         if (status != OCTEON_REQUEST_TIMEOUT && !resp->status) {
1344                 octeon_swap_8B_data((u64 *)&resp->stats,
1345                                     (sizeof(struct oct_link_stats)) >> 3);
1346
1347                 /* RX link-level stats */
1348                 rstats->total_rcvd = rsp_rstats->total_rcvd;
1349                 rstats->bytes_rcvd = rsp_rstats->bytes_rcvd;
1350                 rstats->total_bcst = rsp_rstats->total_bcst;
1351                 rstats->total_mcst = rsp_rstats->total_mcst;
1352                 rstats->runts      = rsp_rstats->runts;
1353                 rstats->ctl_rcvd   = rsp_rstats->ctl_rcvd;
1354                 /* Accounts for over/under-run of buffers */
1355                 rstats->fifo_err  = rsp_rstats->fifo_err;
1356                 rstats->dmac_drop = rsp_rstats->dmac_drop;
1357                 rstats->fcs_err   = rsp_rstats->fcs_err;
1358                 rstats->jabber_err = rsp_rstats->jabber_err;
1359                 rstats->l2_err    = rsp_rstats->l2_err;
1360                 rstats->frame_err = rsp_rstats->frame_err;
1361                 rstats->red_drops = rsp_rstats->red_drops;
1362
1363                 /* RX firmware stats */
1364                 rstats->fw_total_rcvd = rsp_rstats->fw_total_rcvd;
1365                 rstats->fw_total_fwd = rsp_rstats->fw_total_fwd;
1366                 rstats->fw_total_mcast = rsp_rstats->fw_total_mcast;
1367                 rstats->fw_total_bcast = rsp_rstats->fw_total_bcast;
1368                 rstats->fw_err_pko = rsp_rstats->fw_err_pko;
1369                 rstats->fw_err_link = rsp_rstats->fw_err_link;
1370                 rstats->fw_err_drop = rsp_rstats->fw_err_drop;
1371                 rstats->fw_rx_vxlan = rsp_rstats->fw_rx_vxlan;
1372                 rstats->fw_rx_vxlan_err = rsp_rstats->fw_rx_vxlan_err;
1373
1374                 /* Number of packets that are LROed      */
1375                 rstats->fw_lro_pkts = rsp_rstats->fw_lro_pkts;
1376                 /* Number of octets that are LROed       */
1377                 rstats->fw_lro_octs = rsp_rstats->fw_lro_octs;
1378                 /* Number of LRO packets formed          */
1379                 rstats->fw_total_lro = rsp_rstats->fw_total_lro;
1380                 /* Number of times lRO of packet aborted */
1381                 rstats->fw_lro_aborts = rsp_rstats->fw_lro_aborts;
1382                 rstats->fw_lro_aborts_port = rsp_rstats->fw_lro_aborts_port;
1383                 rstats->fw_lro_aborts_seq = rsp_rstats->fw_lro_aborts_seq;
1384                 rstats->fw_lro_aborts_tsval = rsp_rstats->fw_lro_aborts_tsval;
1385                 rstats->fw_lro_aborts_timer = rsp_rstats->fw_lro_aborts_timer;
1386                 /* intrmod: packet forward rate */
1387                 rstats->fwd_rate = rsp_rstats->fwd_rate;
1388
1389                 /* TX link-level stats */
1390                 tstats->total_pkts_sent = rsp_tstats->total_pkts_sent;
1391                 tstats->total_bytes_sent = rsp_tstats->total_bytes_sent;
1392                 tstats->mcast_pkts_sent = rsp_tstats->mcast_pkts_sent;
1393                 tstats->bcast_pkts_sent = rsp_tstats->bcast_pkts_sent;
1394                 tstats->ctl_sent = rsp_tstats->ctl_sent;
1395                 /* Packets sent after one collision*/
1396                 tstats->one_collision_sent = rsp_tstats->one_collision_sent;
1397                 /* Packets sent after multiple collision*/
1398                 tstats->multi_collision_sent = rsp_tstats->multi_collision_sent;
1399                 /* Packets not sent due to max collisions */
1400                 tstats->max_collision_fail = rsp_tstats->max_collision_fail;
1401                 /* Packets not sent due to max deferrals */
1402                 tstats->max_deferral_fail = rsp_tstats->max_deferral_fail;
1403                 /* Accounts for over/under-run of buffers */
1404                 tstats->fifo_err = rsp_tstats->fifo_err;
1405                 tstats->runts = rsp_tstats->runts;
1406                 /* Total number of collisions detected */
1407                 tstats->total_collisions = rsp_tstats->total_collisions;
1408
1409                 /* firmware stats */
1410                 tstats->fw_total_sent = rsp_tstats->fw_total_sent;
1411                 tstats->fw_total_fwd = rsp_tstats->fw_total_fwd;
1412                 tstats->fw_total_mcast_sent = rsp_tstats->fw_total_mcast_sent;
1413                 tstats->fw_total_bcast_sent = rsp_tstats->fw_total_bcast_sent;
1414                 tstats->fw_err_pko = rsp_tstats->fw_err_pko;
1415                 tstats->fw_err_pki = rsp_tstats->fw_err_pki;
1416                 tstats->fw_err_link = rsp_tstats->fw_err_link;
1417                 tstats->fw_err_drop = rsp_tstats->fw_err_drop;
1418                 tstats->fw_tso = rsp_tstats->fw_tso;
1419                 tstats->fw_tso_fwd = rsp_tstats->fw_tso_fwd;
1420                 tstats->fw_err_tso = rsp_tstats->fw_err_tso;
1421                 tstats->fw_tx_vxlan = rsp_tstats->fw_tx_vxlan;
1422
1423                 resp->status = 1;
1424         } else {
1425                 resp->status = -1;
1426         }
1427         complete(&ctrl->complete);
1428 }
1429
1430 int octnet_get_link_stats(struct net_device *netdev)
1431 {
1432         struct lio *lio = GET_LIO(netdev);
1433         struct octeon_device *oct_dev = lio->oct_dev;
1434         struct octeon_soft_command *sc;
1435         struct oct_nic_stats_ctrl *ctrl;
1436         struct oct_nic_stats_resp *resp;
1437         int retval;
1438
1439         /* Alloc soft command */
1440         sc = (struct octeon_soft_command *)
1441                 octeon_alloc_soft_command(oct_dev,
1442                                           0,
1443                                           sizeof(struct oct_nic_stats_resp),
1444                                           sizeof(struct octnic_ctrl_pkt));
1445
1446         if (!sc)
1447                 return -ENOMEM;
1448
1449         resp = (struct oct_nic_stats_resp *)sc->virtrptr;
1450         memset(resp, 0, sizeof(struct oct_nic_stats_resp));
1451
1452         ctrl = (struct oct_nic_stats_ctrl *)sc->ctxptr;
1453         memset(ctrl, 0, sizeof(struct oct_nic_stats_ctrl));
1454         ctrl->netdev = netdev;
1455         init_completion(&ctrl->complete);
1456
1457         sc->iq_no = lio->linfo.txpciq[0].s.q_no;
1458
1459         octeon_prepare_soft_command(oct_dev, sc, OPCODE_NIC,
1460                                     OPCODE_NIC_PORT_STATS, 0, 0, 0);
1461
1462         sc->callback = octnet_nic_stats_callback;
1463         sc->callback_arg = sc;
1464         sc->wait_time = 500;    /*in milli seconds*/
1465
1466         retval = octeon_send_soft_command(oct_dev, sc);
1467         if (retval == IQ_SEND_FAILED) {
1468                 octeon_free_soft_command(oct_dev, sc);
1469                 return -EINVAL;
1470         }
1471
1472         wait_for_completion_timeout(&ctrl->complete, msecs_to_jiffies(1000));
1473
1474         if (resp->status != 1) {
1475                 octeon_free_soft_command(oct_dev, sc);
1476
1477                 return -EINVAL;
1478         }
1479
1480         octeon_free_soft_command(oct_dev, sc);
1481
1482         return 0;
1483 }
1484
1485 static void liquidio_nic_seapi_ctl_callback(struct octeon_device *oct,
1486                                             u32 status,
1487                                             void *buf)
1488 {
1489         struct liquidio_nic_seapi_ctl_context *ctx;
1490         struct octeon_soft_command *sc = buf;
1491
1492         ctx = sc->ctxptr;
1493
1494         oct = lio_get_device(ctx->octeon_id);
1495         if (status) {
1496                 dev_err(&oct->pci_dev->dev, "%s: instruction failed. Status: %llx\n",
1497                         __func__,
1498                         CVM_CAST64(status));
1499         }
1500         ctx->status = status;
1501         complete(&ctx->complete);
1502 }
1503
1504 int liquidio_set_speed(struct lio *lio, int speed)
1505 {
1506         struct liquidio_nic_seapi_ctl_context *ctx;
1507         struct octeon_device *oct = lio->oct_dev;
1508         struct oct_nic_seapi_resp *resp;
1509         struct octeon_soft_command *sc;
1510         union octnet_cmd *ncmd;
1511         u32 ctx_size;
1512         int retval;
1513         u32 var;
1514
1515         if (oct->speed_setting == speed)
1516                 return 0;
1517
1518         if (!OCTEON_CN23XX_PF(oct)) {
1519                 dev_err(&oct->pci_dev->dev, "%s: SET SPEED only for PF\n",
1520                         __func__);
1521                 return -EOPNOTSUPP;
1522         }
1523
1524         ctx_size = sizeof(struct liquidio_nic_seapi_ctl_context);
1525         sc = octeon_alloc_soft_command(oct, OCTNET_CMD_SIZE,
1526                                        sizeof(struct oct_nic_seapi_resp),
1527                                        ctx_size);
1528         if (!sc)
1529                 return -ENOMEM;
1530
1531         ncmd = sc->virtdptr;
1532         ctx  = sc->ctxptr;
1533         resp = sc->virtrptr;
1534         memset(resp, 0, sizeof(struct oct_nic_seapi_resp));
1535
1536         ctx->octeon_id = lio_get_device_id(oct);
1537         ctx->status = 0;
1538         init_completion(&ctx->complete);
1539
1540         ncmd->u64 = 0;
1541         ncmd->s.cmd = SEAPI_CMD_SPEED_SET;
1542         ncmd->s.param1 = speed;
1543
1544         octeon_swap_8B_data((u64 *)ncmd, (OCTNET_CMD_SIZE >> 3));
1545
1546         sc->iq_no = lio->linfo.txpciq[0].s.q_no;
1547
1548         octeon_prepare_soft_command(oct, sc, OPCODE_NIC,
1549                                     OPCODE_NIC_UBOOT_CTL, 0, 0, 0);
1550
1551         sc->callback = liquidio_nic_seapi_ctl_callback;
1552         sc->callback_arg = sc;
1553         sc->wait_time = 5000;
1554
1555         retval = octeon_send_soft_command(oct, sc);
1556         if (retval == IQ_SEND_FAILED) {
1557                 dev_info(&oct->pci_dev->dev, "Failed to send soft command\n");
1558                 retval = -EBUSY;
1559         } else {
1560                 /* Wait for response or timeout */
1561                 if (wait_for_completion_timeout(&ctx->complete,
1562                                                 msecs_to_jiffies(10000)) == 0) {
1563                         dev_err(&oct->pci_dev->dev, "%s: sc timeout\n",
1564                                 __func__);
1565                         octeon_free_soft_command(oct, sc);
1566                         return -EINTR;
1567                 }
1568
1569                 retval = resp->status;
1570
1571                 if (retval) {
1572                         dev_err(&oct->pci_dev->dev, "%s failed, retval=%d\n",
1573                                 __func__, retval);
1574                         octeon_free_soft_command(oct, sc);
1575                         return -EIO;
1576                 }
1577
1578                 var = be32_to_cpu((__force __be32)resp->speed);
1579                 if (var != speed) {
1580                         dev_err(&oct->pci_dev->dev,
1581                                 "%s: setting failed speed= %x, expect %x\n",
1582                                 __func__, var, speed);
1583                 }
1584
1585                 oct->speed_setting = var;
1586         }
1587
1588         octeon_free_soft_command(oct, sc);
1589
1590         return retval;
1591 }
1592
1593 int liquidio_get_speed(struct lio *lio)
1594 {
1595         struct liquidio_nic_seapi_ctl_context *ctx;
1596         struct octeon_device *oct = lio->oct_dev;
1597         struct oct_nic_seapi_resp *resp;
1598         struct octeon_soft_command *sc;
1599         union octnet_cmd *ncmd;
1600         u32 ctx_size;
1601         int retval;
1602
1603         ctx_size = sizeof(struct liquidio_nic_seapi_ctl_context);
1604         sc = octeon_alloc_soft_command(oct, OCTNET_CMD_SIZE,
1605                                        sizeof(struct oct_nic_seapi_resp),
1606                                        ctx_size);
1607         if (!sc)
1608                 return -ENOMEM;
1609
1610         ncmd = sc->virtdptr;
1611         ctx  = sc->ctxptr;
1612         resp = sc->virtrptr;
1613         memset(resp, 0, sizeof(struct oct_nic_seapi_resp));
1614
1615         ctx->octeon_id = lio_get_device_id(oct);
1616         ctx->status = 0;
1617         init_completion(&ctx->complete);
1618
1619         ncmd->u64 = 0;
1620         ncmd->s.cmd = SEAPI_CMD_SPEED_GET;
1621
1622         octeon_swap_8B_data((u64 *)ncmd, (OCTNET_CMD_SIZE >> 3));
1623
1624         sc->iq_no = lio->linfo.txpciq[0].s.q_no;
1625
1626         octeon_prepare_soft_command(oct, sc, OPCODE_NIC,
1627                                     OPCODE_NIC_UBOOT_CTL, 0, 0, 0);
1628
1629         sc->callback = liquidio_nic_seapi_ctl_callback;
1630         sc->callback_arg = sc;
1631         sc->wait_time = 5000;
1632
1633         retval = octeon_send_soft_command(oct, sc);
1634         if (retval == IQ_SEND_FAILED) {
1635                 dev_info(&oct->pci_dev->dev, "Failed to send soft command\n");
1636                 oct->no_speed_setting = 1;
1637                 oct->speed_setting = 25;
1638
1639                 retval = -EBUSY;
1640         } else {
1641                 if (wait_for_completion_timeout(&ctx->complete,
1642                                                 msecs_to_jiffies(10000)) == 0) {
1643                         dev_err(&oct->pci_dev->dev, "%s: sc timeout\n",
1644                                 __func__);
1645
1646                         oct->speed_setting = 25;
1647                         oct->no_speed_setting = 1;
1648
1649                         octeon_free_soft_command(oct, sc);
1650
1651                         return -EINTR;
1652                 }
1653                 retval = resp->status;
1654                 if (retval) {
1655                         dev_err(&oct->pci_dev->dev,
1656                                 "%s failed retval=%d\n", __func__, retval);
1657                         oct->no_speed_setting = 1;
1658                         oct->speed_setting = 25;
1659                         octeon_free_soft_command(oct, sc);
1660                         retval = -EIO;
1661                 } else {
1662                         u32 var;
1663
1664                         var = be32_to_cpu((__force __be32)resp->speed);
1665                         oct->speed_setting = var;
1666                         if (var == 0xffff) {
1667                                 oct->no_speed_setting = 1;
1668                                 /* unable to access boot variables
1669                                  * get the default value based on the NIC type
1670                                  */
1671                                 oct->speed_setting = 25;
1672                         }
1673                 }
1674         }
1675
1676         octeon_free_soft_command(oct, sc);
1677
1678         return retval;
1679 }