drivers/net/ethernet/intel/i40e/i40e_main.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright(c) 2013 - 2018 Intel Corporation. */
   3
   4 #include <linux/etherdevice.h>
   5 #include <linux/of_net.h>
   6 #include <linux/pci.h>
   7 #include <linux/bpf.h>
   8
   9 /* Local includes */
  10 #include "i40e.h"
  11 #include "i40e_diag.h"
  12 #include <net/udp_tunnel.h>
  13 /* All i40e tracepoints are defined by the include below, which
  14  * must be included exactly once across the whole kernel with
  15  * CREATE_TRACE_POINTS defined
  16  */
  17 #define CREATE_TRACE_POINTS
  18 #include "i40e_trace.h"
  19
  20 const char i40e_driver_name[] = "i40e";
  21 static const char i40e_driver_string[] =
  22                         "Intel(R) Ethernet Connection XL710 Network Driver";
  23
  24 #define DRV_KERN "-k"
  25
  26 #define DRV_VERSION_MAJOR 2
  27 #define DRV_VERSION_MINOR 3
  28 #define DRV_VERSION_BUILD 2
  29 #define DRV_VERSION __stringify(DRV_VERSION_MAJOR) "." \
  30              __stringify(DRV_VERSION_MINOR) "." \
  31              __stringify(DRV_VERSION_BUILD)    DRV_KERN
  32 const char i40e_driver_version_str[] = DRV_VERSION;
  33 static const char i40e_copyright[] = "Copyright (c) 2013 - 2014 Intel Corporation.";
  34
  35 /* a bit of forward declarations */
  36 static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi);
  37 static void i40e_handle_reset_warning(struct i40e_pf *pf, bool lock_acquired);
  38 static int i40e_add_vsi(struct i40e_vsi *vsi);
  39 static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi);
  40 static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit);
  41 static int i40e_setup_misc_vector(struct i40e_pf *pf);
  42 static void i40e_determine_queue_usage(struct i40e_pf *pf);
  43 static int i40e_setup_pf_filter_control(struct i40e_pf *pf);
  44 static void i40e_prep_for_reset(struct i40e_pf *pf, bool lock_acquired);
  45 static void i40e_reset_and_rebuild(struct i40e_pf *pf, bool reinit,
  46                                    bool lock_acquired);
  47 static int i40e_reset(struct i40e_pf *pf);
  48 static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired);
  49 static void i40e_fdir_sb_setup(struct i40e_pf *pf);
  50 static int i40e_veb_get_bw_info(struct i40e_veb *veb);
  51 static int i40e_get_capabilities(struct i40e_pf *pf,
  52                                  enum i40e_admin_queue_opc list_type);
  53
  54
  55 /* i40e_pci_tbl - PCI Device ID Table
  56  *
  57  * Last entry must be all 0s
  58  *
  59  * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
  60  *   Class, Class Mask, private data (not used) }
  61  */
  62 static const struct pci_device_id i40e_pci_tbl[] = {
  63         {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_XL710), 0},
  64         {PCI_VDEVICE(INTEL, I40E_DEV_ID_QEMU), 0},
  65         {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_B), 0},
  66         {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_C), 0},
  67         {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_A), 0},
  68         {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_B), 0},
  69         {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_C), 0},
  70         {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T), 0},
  71         {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T4), 0},
  72         {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_X722), 0},
  73         {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_X722), 0},
  74         {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_X722), 0},
  75         {PCI_VDEVICE(INTEL, I40E_DEV_ID_1G_BASE_T_X722), 0},
  76         {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T_X722), 0},
  77         {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_I_X722), 0},
  78         {PCI_VDEVICE(INTEL, I40E_DEV_ID_20G_KR2), 0},
  79         {PCI_VDEVICE(INTEL, I40E_DEV_ID_20G_KR2_A), 0},
  80         {PCI_VDEVICE(INTEL, I40E_DEV_ID_25G_B), 0},
  81         {PCI_VDEVICE(INTEL, I40E_DEV_ID_25G_SFP28), 0},
  82         /* required last entry */
  83         {0, }
  84 };
  85 MODULE_DEVICE_TABLE(pci, i40e_pci_tbl);
  86
  87 #define I40E_MAX_VF_COUNT 128
  88 static int debug = -1;
  89 module_param(debug, uint, 0);
  90 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all), Debug mask (0x8XXXXXXX)");
  91
  92 MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
  93 MODULE_DESCRIPTION("Intel(R) Ethernet Connection XL710 Network Driver");
  94 MODULE_LICENSE("GPL");
  95 MODULE_VERSION(DRV_VERSION);
  96
  97 static struct workqueue_struct *i40e_wq;
  98
  99 static void netdev_hw_addr_refcnt(struct i40e_mac_filter *f,
 100                                   struct net_device *netdev, int delta)
 101 {
 102         struct netdev_hw_addr *ha;
 103
 104         if (!f || !netdev)
 105                 return;
 106
 107         netdev_for_each_mc_addr(ha, netdev) {
 108                 if (ether_addr_equal(ha->addr, f->macaddr)) {
 109                         ha->refcount += delta;
 110                         if (ha->refcount <= 0)
 111                                 ha->refcount = 1;
 112                         break;
 113                 }
 114         }
 115 }
 116
 117 /**
 118  * i40e_allocate_dma_mem_d - OS specific memory alloc for shared code
 119  * @hw:   pointer to the HW structure
 120  * @mem:  ptr to mem struct to fill out
 121  * @size: size of memory requested
 122  * @alignment: what to align the allocation to
 123  **/
 124 int i40e_allocate_dma_mem_d(struct i40e_hw *hw, struct i40e_dma_mem *mem,
 125                             u64 size, u32 alignment)
 126 {
 127         struct i40e_pf *pf = (struct i40e_pf *)hw->back;
 128
 129         mem->size = ALIGN(size, alignment);
 130         mem->va = dma_zalloc_coherent(&pf->pdev->dev, mem->size,
 131                                       &mem->pa, GFP_KERNEL);
 132         if (!mem->va)
 133                 return -ENOMEM;
 134
 135         return 0;
 136 }
 137
 138 /**
 139  * i40e_free_dma_mem_d - OS specific memory free for shared code
 140  * @hw:   pointer to the HW structure
 141  * @mem:  ptr to mem struct to free
 142  **/
 143 int i40e_free_dma_mem_d(struct i40e_hw *hw, struct i40e_dma_mem *mem)
 144 {
 145         struct i40e_pf *pf = (struct i40e_pf *)hw->back;
 146
 147         dma_free_coherent(&pf->pdev->dev, mem->size, mem->va, mem->pa);
 148         mem->va = NULL;
 149         mem->pa = 0;
 150         mem->size = 0;
 151
 152         return 0;
 153 }
 154
 155 /**
 156  * i40e_allocate_virt_mem_d - OS specific memory alloc for shared code
 157  * @hw:   pointer to the HW structure
 158  * @mem:  ptr to mem struct to fill out
 159  * @size: size of memory requested
 160  **/
 161 int i40e_allocate_virt_mem_d(struct i40e_hw *hw, struct i40e_virt_mem *mem,
 162                              u32 size)
 163 {
 164         mem->size = size;
 165         mem->va = kzalloc(size, GFP_KERNEL);
 166
 167         if (!mem->va)
 168                 return -ENOMEM;
 169
 170         return 0;
 171 }
 172
 173 /**
 174  * i40e_free_virt_mem_d - OS specific memory free for shared code
 175  * @hw:   pointer to the HW structure
 176  * @mem:  ptr to mem struct to free
 177  **/
 178 int i40e_free_virt_mem_d(struct i40e_hw *hw, struct i40e_virt_mem *mem)
 179 {
 180         /* it's ok to kfree a NULL pointer */
 181         kfree(mem->va);
 182         mem->va = NULL;
 183         mem->size = 0;
 184
 185         return 0;
 186 }
 187
 188 /**
 189  * i40e_get_lump - find a lump of free generic resource
 190  * @pf: board private structure
 191  * @pile: the pile of resource to search
 192  * @needed: the number of items needed
 193  * @id: an owner id to stick on the items assigned
 194  *
 195  * Returns the base item index of the lump, or negative for error
 196  **/
 197 static int i40e_get_lump(struct i40e_pf *pf, struct i40e_lump_tracking *pile,
 198                          u16 needed, u16 id)
 199 {
 200         int ret = -ENOMEM;
 201         int i, j;
 202
 203         if (!pile || needed == 0 || id >= I40E_PILE_VALID_BIT) {
 204                 dev_info(&pf->pdev->dev,
 205                          "param err: pile=%s needed=%d id=0x%04x\n",
 206                          pile ? "<valid>" : "<null>", needed, id);
 207                 return -EINVAL;
 208         }
 209
 210         /* Allocate last queue in the pile for FDIR VSI queue
 211          * so it doesn't fragment the qp_pile
 212          */
 213         if (pile == pf->qp_pile && pf->vsi[id]->type == I40E_VSI_FDIR) {
 214                 if (pile->list[pile->num_entries - 1] & I40E_PILE_VALID_BIT) {
 215                         dev_err(&pf->pdev->dev,
 216                                 "Cannot allocate queue %d for I40E_VSI_FDIR\n",
 217                                 pile->num_entries - 1);
 218                         return -ENOMEM;
 219                 }
 220                 pile->list[pile->num_entries - 1] = id | I40E_PILE_VALID_BIT;
 221                 return pile->num_entries - 1;
 222         }
 223
 224         i = 0;
 225         while (i < pile->num_entries) {
 226                 /* skip already allocated entries */
 227                 if (pile->list[i] & I40E_PILE_VALID_BIT) {
 228                         i++;
 229                         continue;
 230                 }
 231
 232                 /* do we have enough in this lump? */
 233                 for (j = 0; (j < needed) && ((i+j) < pile->num_entries); j++) {
 234                         if (pile->list[i+j] & I40E_PILE_VALID_BIT)
 235                                 break;
 236                 }
 237
 238                 if (j == needed) {
 239                         /* there was enough, so assign it to the requestor */
 240                         for (j = 0; j < needed; j++)
 241                                 pile->list[i+j] = id | I40E_PILE_VALID_BIT;
 242                         ret = i;
 243                         break;
 244                 }
 245
 246                 /* not enough, so skip over it and continue looking */
 247                 i += j;
 248         }
 249
 250         return ret;
 251 }
 252
 253 /**
 254  * i40e_put_lump - return a lump of generic resource
 255  * @pile: the pile of resource to search
 256  * @index: the base item index
 257  * @id: the owner id of the items assigned
 258  *
 259  * Returns the count of items in the lump
 260  **/
 261 static int i40e_put_lump(struct i40e_lump_tracking *pile, u16 index, u16 id)
 262 {
 263         int valid_id = (id | I40E_PILE_VALID_BIT);
 264         int count = 0;
 265         u16 i;
 266
 267         if (!pile || index >= pile->num_entries)
 268                 return -EINVAL;
 269
 270         for (i = index;
 271              i < pile->num_entries && pile->list[i] == valid_id;
 272              i++) {
 273                 pile->list[i] = 0;
 274                 count++;
 275         }
 276
 277
 278         return count;
 279 }
 280
 281 /**
 282  * i40e_find_vsi_from_id - searches for the vsi with the given id
 283  * @pf: the pf structure to search for the vsi
 284  * @id: id of the vsi it is searching for
 285  **/
 286 struct i40e_vsi *i40e_find_vsi_from_id(struct i40e_pf *pf, u16 id)
 287 {
 288         int i;
 289
 290         for (i = 0; i < pf->num_alloc_vsi; i++)
 291                 if (pf->vsi[i] && (pf->vsi[i]->id == id))
 292                         return pf->vsi[i];
 293
 294         return NULL;
 295 }
 296
 297 /**
 298  * i40e_service_event_schedule - Schedule the service task to wake up
 299  * @pf: board private structure
 300  *
 301  * If not already scheduled, this puts the task into the work queue
 302  **/
 303 void i40e_service_event_schedule(struct i40e_pf *pf)
 304 {
 305         if (!test_bit(__I40E_DOWN, pf->state) &&
 306             !test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
 307                 queue_work(i40e_wq, &pf->service_task);
 308 }
 309
 310 /**
 311  * i40e_tx_timeout - Respond to a Tx Hang
 312  * @netdev: network interface device structure
 313  *
 314  * If any port has noticed a Tx timeout, it is likely that the whole
 315  * device is munged, not just the one netdev port, so go for the full
 316  * reset.
 317  **/
 318 static void i40e_tx_timeout(struct net_device *netdev)
 319 {
 320         struct i40e_netdev_priv *np = netdev_priv(netdev);
 321         struct i40e_vsi *vsi = np->vsi;
 322         struct i40e_pf *pf = vsi->back;
 323         struct i40e_ring *tx_ring = NULL;
 324         unsigned int i, hung_queue = 0;
 325         u32 head, val;
 326
 327         pf->tx_timeout_count++;
 328
 329         /* find the stopped queue the same way the stack does */
 330         for (i = 0; i < netdev->num_tx_queues; i++) {
 331                 struct netdev_queue *q;
 332                 unsigned long trans_start;
 333
 334                 q = netdev_get_tx_queue(netdev, i);
 335                 trans_start = q->trans_start;
 336                 if (netif_xmit_stopped(q) &&
 337                     time_after(jiffies,
 338                                (trans_start + netdev->watchdog_timeo))) {
 339                         hung_queue = i;
 340                         break;
 341                 }
 342         }
 343
 344         if (i == netdev->num_tx_queues) {
 345                 netdev_info(netdev, "tx_timeout: no netdev hung queue found\n");
 346         } else {
 347                 /* now that we have an index, find the tx_ring struct */
 348                 for (i = 0; i < vsi->num_queue_pairs; i++) {
 349                         if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc) {
 350                                 if (hung_queue ==
 351                                     vsi->tx_rings[i]->queue_index) {
 352                                         tx_ring = vsi->tx_rings[i];
 353                                         break;
 354                                 }
 355                         }
 356                 }
 357         }
 358
 359         if (time_after(jiffies, (pf->tx_timeout_last_recovery + HZ*20)))
 360                 pf->tx_timeout_recovery_level = 1;  /* reset after some time */
 361         else if (time_before(jiffies,
 362                       (pf->tx_timeout_last_recovery + netdev->watchdog_timeo)))
 363                 return;   /* don't do any new action before the next timeout */
 364
 365         /* don't kick off another recovery if one is already pending */
 366         if (test_and_set_bit(__I40E_TIMEOUT_RECOVERY_PENDING, pf->state))
 367                 return;
 368
 369         if (tx_ring) {
 370                 head = i40e_get_head(tx_ring);
 371                 /* Read interrupt register */
 372                 if (pf->flags & I40E_FLAG_MSIX_ENABLED)
 373                         val = rd32(&pf->hw,
 374                              I40E_PFINT_DYN_CTLN(tx_ring->q_vector->v_idx +
 375                                                 tx_ring->vsi->base_vector - 1));
 376                 else
 377                         val = rd32(&pf->hw, I40E_PFINT_DYN_CTL0);
 378
 379                 netdev_info(netdev, "tx_timeout: VSI_seid: %d, Q %d, NTC: 0x%x, HWB: 0x%x, NTU: 0x%x, TAIL: 0x%x, INT: 0x%x\n",
 380                             vsi->seid, hung_queue, tx_ring->next_to_clean,
 381                             head, tx_ring->next_to_use,
 382                             readl(tx_ring->tail), val);
 383         }
 384
 385         pf->tx_timeout_last_recovery = jiffies;
 386         netdev_info(netdev, "tx_timeout recovery level %d, hung_queue %d\n",
 387                     pf->tx_timeout_recovery_level, hung_queue);
 388
 389         switch (pf->tx_timeout_recovery_level) {
 390         case 1:
 391                 set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
 392                 break;
 393         case 2:
 394                 set_bit(__I40E_CORE_RESET_REQUESTED, pf->state);
 395                 break;
 396         case 3:
 397                 set_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state);
 398                 break;
 399         default:
 400                 netdev_err(netdev, "tx_timeout recovery unsuccessful, device is in non-recoverable state.\n");
 401                 set_bit(__I40E_DOWN_REQUESTED, pf->state);
 402                 set_bit(__I40E_VSI_DOWN_REQUESTED, vsi->state);
 403                 break;
 404         }
 405
 406         i40e_service_event_schedule(pf);
 407         pf->tx_timeout_recovery_level++;
 408 }
 409
 410 /**
 411  * i40e_get_vsi_stats_struct - Get System Network Statistics
 412  * @vsi: the VSI we care about
 413  *
 414  * Returns the address of the device statistics structure.
 415  * The statistics are actually updated from the service task.
 416  **/
 417 struct rtnl_link_stats64 *i40e_get_vsi_stats_struct(struct i40e_vsi *vsi)
 418 {
 419         return &vsi->net_stats;
 420 }
 421
 422 /**
 423  * i40e_get_netdev_stats_struct_tx - populate stats from a Tx ring
 424  * @ring: Tx ring to get statistics from
 425  * @stats: statistics entry to be updated
 426  **/
 427 static void i40e_get_netdev_stats_struct_tx(struct i40e_ring *ring,
 428                                             struct rtnl_link_stats64 *stats)
 429 {
 430         u64 bytes, packets;
 431         unsigned int start;
 432
 433         do {
 434                 start = u64_stats_fetch_begin_irq(&ring->syncp);
 435                 packets = ring->stats.packets;
 436                 bytes   = ring->stats.bytes;
 437         } while (u64_stats_fetch_retry_irq(&ring->syncp, start));
 438
 439         stats->tx_packets += packets;
 440         stats->tx_bytes   += bytes;
 441 }
 442
 443 /**
 444  * i40e_get_netdev_stats_struct - Get statistics for netdev interface
 445  * @netdev: network interface device structure
 446  * @stats: data structure to store statistics
 447  *
 448  * Returns the address of the device statistics structure.
 449  * The statistics are actually updated from the service task.
 450  **/
 451 static void i40e_get_netdev_stats_struct(struct net_device *netdev,
 452                                   struct rtnl_link_stats64 *stats)
 453 {
 454         struct i40e_netdev_priv *np = netdev_priv(netdev);
 455         struct i40e_vsi *vsi = np->vsi;
 456         struct rtnl_link_stats64 *vsi_stats = i40e_get_vsi_stats_struct(vsi);
 457         struct i40e_ring *ring;
 458         int i;
 459
 460         if (test_bit(__I40E_VSI_DOWN, vsi->state))
 461                 return;
 462
 463         if (!vsi->tx_rings)
 464                 return;
 465
 466         rcu_read_lock();
 467         for (i = 0; i < vsi->num_queue_pairs; i++) {
 468                 u64 bytes, packets;
 469                 unsigned int start;
 470
 471                 ring = READ_ONCE(vsi->tx_rings[i]);
 472                 if (!ring)
 473                         continue;
 474                 i40e_get_netdev_stats_struct_tx(ring, stats);
 475
 476                 if (i40e_enabled_xdp_vsi(vsi)) {
 477                         ring = READ_ONCE(vsi->xdp_rings[i]);
 478                         if (!ring)
 479                                 continue;
 480                         i40e_get_netdev_stats_struct_tx(ring, stats);
 481                 }
 482
 483                 ring = READ_ONCE(vsi->rx_rings[i]);
 484                 if (!ring)
 485                         continue;
 486                 do {
 487                         start   = u64_stats_fetch_begin_irq(&ring->syncp);
 488                         packets = ring->stats.packets;
 489                         bytes   = ring->stats.bytes;
 490                 } while (u64_stats_fetch_retry_irq(&ring->syncp, start));
 491
 492                 stats->rx_packets += packets;
 493                 stats->rx_bytes   += bytes;
 494
 495         }
 496         rcu_read_unlock();
 497
 498         /* following stats updated by i40e_watchdog_subtask() */
 499         stats->multicast        = vsi_stats->multicast;
 500         stats->tx_errors        = vsi_stats->tx_errors;
 501         stats->tx_dropped       = vsi_stats->tx_dropped;
 502         stats->rx_errors        = vsi_stats->rx_errors;
 503         stats->rx_dropped       = vsi_stats->rx_dropped;
 504         stats->rx_crc_errors    = vsi_stats->rx_crc_errors;
 505         stats->rx_length_errors = vsi_stats->rx_length_errors;
 506 }
 507
 508 /**
 509  * i40e_vsi_reset_stats - Resets all stats of the given vsi
 510  * @vsi: the VSI to have its stats reset
 511  **/
 512 void i40e_vsi_reset_stats(struct i40e_vsi *vsi)
 513 {
 514         struct rtnl_link_stats64 *ns;
 515         int i;
 516
 517         if (!vsi)
 518                 return;
 519
 520         ns = i40e_get_vsi_stats_struct(vsi);
 521         memset(ns, 0, sizeof(*ns));
 522         memset(&vsi->net_stats_offsets, 0, sizeof(vsi->net_stats_offsets));
 523         memset(&vsi->eth_stats, 0, sizeof(vsi->eth_stats));
 524         memset(&vsi->eth_stats_offsets, 0, sizeof(vsi->eth_stats_offsets));
 525         if (vsi->rx_rings && vsi->rx_rings[0]) {
 526                 for (i = 0; i < vsi->num_queue_pairs; i++) {
 527                         memset(&vsi->rx_rings[i]->stats, 0,
 528                                sizeof(vsi->rx_rings[i]->stats));
 529                         memset(&vsi->rx_rings[i]->rx_stats, 0,
 530                                sizeof(vsi->rx_rings[i]->rx_stats));
 531                         memset(&vsi->tx_rings[i]->stats, 0,
 532                                sizeof(vsi->tx_rings[i]->stats));
 533                         memset(&vsi->tx_rings[i]->tx_stats, 0,
 534                                sizeof(vsi->tx_rings[i]->tx_stats));
 535                 }
 536         }
 537         vsi->stat_offsets_loaded = false;
 538 }
 539
 540 /**
 541  * i40e_pf_reset_stats - Reset all of the stats for the given PF
 542  * @pf: the PF to be reset
 543  **/
 544 void i40e_pf_reset_stats(struct i40e_pf *pf)
 545 {
 546         int i;
 547
 548         memset(&pf->stats, 0, sizeof(pf->stats));
 549         memset(&pf->stats_offsets, 0, sizeof(pf->stats_offsets));
 550         pf->stat_offsets_loaded = false;
 551
 552         for (i = 0; i < I40E_MAX_VEB; i++) {
 553                 if (pf->veb[i]) {
 554                         memset(&pf->veb[i]->stats, 0,
 555                                sizeof(pf->veb[i]->stats));
 556                         memset(&pf->veb[i]->stats_offsets, 0,
 557                                sizeof(pf->veb[i]->stats_offsets));
 558                         pf->veb[i]->stat_offsets_loaded = false;
 559                 }
 560         }
 561         pf->hw_csum_rx_error = 0;
 562 }
 563
 564 /**
 565  * i40e_stat_update48 - read and update a 48 bit stat from the chip
 566  * @hw: ptr to the hardware info
 567  * @hireg: the high 32 bit reg to read
 568  * @loreg: the low 32 bit reg to read
 569  * @offset_loaded: has the initial offset been loaded yet
 570  * @offset: ptr to current offset value
 571  * @stat: ptr to the stat
 572  *
 573  * Since the device stats are not reset at PFReset, they likely will not
 574  * be zeroed when the driver starts.  We'll save the first values read
 575  * and use them as offsets to be subtracted from the raw values in order
 576  * to report stats that count from zero.  In the process, we also manage
 577  * the potential roll-over.
 578  **/
 579 static void i40e_stat_update48(struct i40e_hw *hw, u32 hireg, u32 loreg,
 580                                bool offset_loaded, u64 *offset, u64 *stat)
 581 {
 582         u64 new_data;
 583
 584         if (hw->device_id == I40E_DEV_ID_QEMU) {
 585                 new_data = rd32(hw, loreg);
 586                 new_data |= ((u64)(rd32(hw, hireg) & 0xFFFF)) << 32;
 587         } else {
 588                 new_data = rd64(hw, loreg);
 589         }
 590         if (!offset_loaded)
 591                 *offset = new_data;
 592         if (likely(new_data >= *offset))
 593                 *stat = new_data - *offset;
 594         else
 595                 *stat = (new_data + BIT_ULL(48)) - *offset;
 596         *stat &= 0xFFFFFFFFFFFFULL;
 597 }
 598
 599 /**
 600  * i40e_stat_update32 - read and update a 32 bit stat from the chip
 601  * @hw: ptr to the hardware info
 602  * @reg: the hw reg to read
 603  * @offset_loaded: has the initial offset been loaded yet
 604  * @offset: ptr to current offset value
 605  * @stat: ptr to the stat
 606  **/
 607 static void i40e_stat_update32(struct i40e_hw *hw, u32 reg,
 608                                bool offset_loaded, u64 *offset, u64 *stat)
 609 {
 610         u32 new_data;
 611
 612         new_data = rd32(hw, reg);
 613         if (!offset_loaded)
 614                 *offset = new_data;
 615         if (likely(new_data >= *offset))
 616                 *stat = (u32)(new_data - *offset);
 617         else
 618                 *stat = (u32)((new_data + BIT_ULL(32)) - *offset);
 619 }
 620
 621 /**
 622  * i40e_stat_update_and_clear32 - read and clear hw reg, update a 32 bit stat
 623  * @hw: ptr to the hardware info
 624  * @reg: the hw reg to read and clear
 625  * @stat: ptr to the stat
 626  **/
 627 static void i40e_stat_update_and_clear32(struct i40e_hw *hw, u32 reg, u64 *stat)
 628 {
 629         u32 new_data = rd32(hw, reg);
 630
 631         wr32(hw, reg, 1); /* must write a nonzero value to clear register */
 632         *stat += new_data;
 633 }
 634
 635 /**
 636  * i40e_update_eth_stats - Update VSI-specific ethernet statistics counters.
 637  * @vsi: the VSI to be updated
 638  **/
 639 void i40e_update_eth_stats(struct i40e_vsi *vsi)
 640 {
 641         int stat_idx = le16_to_cpu(vsi->info.stat_counter_idx);
 642         struct i40e_pf *pf = vsi->back;
 643         struct i40e_hw *hw = &pf->hw;
 644         struct i40e_eth_stats *oes;
 645         struct i40e_eth_stats *es;     /* device's eth stats */
 646
 647         es = &vsi->eth_stats;
 648         oes = &vsi->eth_stats_offsets;
 649
 650         /* Gather up the stats that the hw collects */
 651         i40e_stat_update32(hw, I40E_GLV_TEPC(stat_idx),
 652                            vsi->stat_offsets_loaded,
 653                            &oes->tx_errors, &es->tx_errors);
 654         i40e_stat_update32(hw, I40E_GLV_RDPC(stat_idx),
 655                            vsi->stat_offsets_loaded,
 656                            &oes->rx_discards, &es->rx_discards);
 657         i40e_stat_update32(hw, I40E_GLV_RUPP(stat_idx),
 658                            vsi->stat_offsets_loaded,
 659                            &oes->rx_unknown_protocol, &es->rx_unknown_protocol);
 660         i40e_stat_update32(hw, I40E_GLV_TEPC(stat_idx),
 661                            vsi->stat_offsets_loaded,
 662                            &oes->tx_errors, &es->tx_errors);
 663
 664         i40e_stat_update48(hw, I40E_GLV_GORCH(stat_idx),
 665                            I40E_GLV_GORCL(stat_idx),
 666                            vsi->stat_offsets_loaded,
 667                            &oes->rx_bytes, &es->rx_bytes);
 668         i40e_stat_update48(hw, I40E_GLV_UPRCH(stat_idx),
 669                            I40E_GLV_UPRCL(stat_idx),
 670                            vsi->stat_offsets_loaded,
 671                            &oes->rx_unicast, &es->rx_unicast);
 672         i40e_stat_update48(hw, I40E_GLV_MPRCH(stat_idx),
 673                            I40E_GLV_MPRCL(stat_idx),
 674                            vsi->stat_offsets_loaded,
 675                            &oes->rx_multicast, &es->rx_multicast);
 676         i40e_stat_update48(hw, I40E_GLV_BPRCH(stat_idx),
 677                            I40E_GLV_BPRCL(stat_idx),
 678                            vsi->stat_offsets_loaded,
 679                            &oes->rx_broadcast, &es->rx_broadcast);
 680
 681         i40e_stat_update48(hw, I40E_GLV_GOTCH(stat_idx),
 682                            I40E_GLV_GOTCL(stat_idx),
 683                            vsi->stat_offsets_loaded,
 684                            &oes->tx_bytes, &es->tx_bytes);
 685         i40e_stat_update48(hw, I40E_GLV_UPTCH(stat_idx),
 686                            I40E_GLV_UPTCL(stat_idx),
 687                            vsi->stat_offsets_loaded,
 688                            &oes->tx_unicast, &es->tx_unicast);
 689         i40e_stat_update48(hw, I40E_GLV_MPTCH(stat_idx),
 690                            I40E_GLV_MPTCL(stat_idx),
 691                            vsi->stat_offsets_loaded,
 692                            &oes->tx_multicast, &es->tx_multicast);
 693         i40e_stat_update48(hw, I40E_GLV_BPTCH(stat_idx),
 694                            I40E_GLV_BPTCL(stat_idx),
 695                            vsi->stat_offsets_loaded,
 696                            &oes->tx_broadcast, &es->tx_broadcast);
 697         vsi->stat_offsets_loaded = true;
 698 }
 699
 700 /**
 701  * i40e_update_veb_stats - Update Switch component statistics
 702  * @veb: the VEB being updated
 703  **/
 704 static void i40e_update_veb_stats(struct i40e_veb *veb)
 705 {
 706         struct i40e_pf *pf = veb->pf;
 707         struct i40e_hw *hw = &pf->hw;
 708         struct i40e_eth_stats *oes;
 709         struct i40e_eth_stats *es;     /* device's eth stats */
 710         struct i40e_veb_tc_stats *veb_oes;
 711         struct i40e_veb_tc_stats *veb_es;
 712         int i, idx = 0;
 713
 714         idx = veb->stats_idx;
 715         es = &veb->stats;
 716         oes = &veb->stats_offsets;
 717         veb_es = &veb->tc_stats;
 718         veb_oes = &veb->tc_stats_offsets;
 719
 720         /* Gather up the stats that the hw collects */
 721         i40e_stat_update32(hw, I40E_GLSW_TDPC(idx),
 722                            veb->stat_offsets_loaded,
 723                            &oes->tx_discards, &es->tx_discards);
 724         if (hw->revision_id > 0)
 725                 i40e_stat_update32(hw, I40E_GLSW_RUPP(idx),
 726                                    veb->stat_offsets_loaded,
 727                                    &oes->rx_unknown_protocol,
 728                                    &es->rx_unknown_protocol);
 729         i40e_stat_update48(hw, I40E_GLSW_GORCH(idx), I40E_GLSW_GORCL(idx),
 730                            veb->stat_offsets_loaded,
 731                            &oes->rx_bytes, &es->rx_bytes);
 732         i40e_stat_update48(hw, I40E_GLSW_UPRCH(idx), I40E_GLSW_UPRCL(idx),
 733                            veb->stat_offsets_loaded,
 734                            &oes->rx_unicast, &es->rx_unicast);
 735         i40e_stat_update48(hw, I40E_GLSW_MPRCH(idx), I40E_GLSW_MPRCL(idx),
 736                            veb->stat_offsets_loaded,
 737                            &oes->rx_multicast, &es->rx_multicast);
 738         i40e_stat_update48(hw, I40E_GLSW_BPRCH(idx), I40E_GLSW_BPRCL(idx),
 739                            veb->stat_offsets_loaded,
 740                            &oes->rx_broadcast, &es->rx_broadcast);
 741
 742         i40e_stat_update48(hw, I40E_GLSW_GOTCH(idx), I40E_GLSW_GOTCL(idx),
 743                            veb->stat_offsets_loaded,
 744                            &oes->tx_bytes, &es->tx_bytes);
 745         i40e_stat_update48(hw, I40E_GLSW_UPTCH(idx), I40E_GLSW_UPTCL(idx),
 746                            veb->stat_offsets_loaded,
 747                            &oes->tx_unicast, &es->tx_unicast);
 748         i40e_stat_update48(hw, I40E_GLSW_MPTCH(idx), I40E_GLSW_MPTCL(idx),
 749                            veb->stat_offsets_loaded,
 750                            &oes->tx_multicast, &es->tx_multicast);
 751         i40e_stat_update48(hw, I40E_GLSW_BPTCH(idx), I40E_GLSW_BPTCL(idx),
 752                            veb->stat_offsets_loaded,
 753                            &oes->tx_broadcast, &es->tx_broadcast);
 754         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
 755                 i40e_stat_update48(hw, I40E_GLVEBTC_RPCH(i, idx),
 756                                    I40E_GLVEBTC_RPCL(i, idx),
 757                                    veb->stat_offsets_loaded,
 758                                    &veb_oes->tc_rx_packets[i],
 759                                    &veb_es->tc_rx_packets[i]);
 760                 i40e_stat_update48(hw, I40E_GLVEBTC_RBCH(i, idx),
 761                                    I40E_GLVEBTC_RBCL(i, idx),
 762                                    veb->stat_offsets_loaded,
 763                                    &veb_oes->tc_rx_bytes[i],
 764                                    &veb_es->tc_rx_bytes[i]);
 765                 i40e_stat_update48(hw, I40E_GLVEBTC_TPCH(i, idx),
 766                                    I40E_GLVEBTC_TPCL(i, idx),
 767                                    veb->stat_offsets_loaded,
 768                                    &veb_oes->tc_tx_packets[i],
 769                                    &veb_es->tc_tx_packets[i]);
 770                 i40e_stat_update48(hw, I40E_GLVEBTC_TBCH(i, idx),
 771                                    I40E_GLVEBTC_TBCL(i, idx),
 772                                    veb->stat_offsets_loaded,
 773                                    &veb_oes->tc_tx_bytes[i],
 774                                    &veb_es->tc_tx_bytes[i]);
 775         }
 776         veb->stat_offsets_loaded = true;
 777 }
 778
 779 /**
 780  * i40e_update_vsi_stats - Update the vsi statistics counters.
 781  * @vsi: the VSI to be updated
 782  *
 783  * There are a few instances where we store the same stat in a
 784  * couple of different structs.  This is partly because we have
 785  * the netdev stats that need to be filled out, which is slightly
 786  * different from the "eth_stats" defined by the chip and used in
 787  * VF communications.  We sort it out here.
 788  **/
 789 static void i40e_update_vsi_stats(struct i40e_vsi *vsi)
 790 {
 791         struct i40e_pf *pf = vsi->back;
 792         struct rtnl_link_stats64 *ons;
 793         struct rtnl_link_stats64 *ns;   /* netdev stats */
 794         struct i40e_eth_stats *oes;
 795         struct i40e_eth_stats *es;     /* device's eth stats */
 796         u64 tx_restart, tx_busy;
 797         struct i40e_ring *p;
 798         u64 rx_page, rx_buf;
 799         u64 bytes, packets;
 800         unsigned int start;
 801         u64 tx_linearize;
 802         u64 tx_force_wb;
 803         u64 rx_p, rx_b;
 804         u64 tx_p, tx_b;
 805         u16 q;
 806
 807         if (test_bit(__I40E_VSI_DOWN, vsi->state) ||
 808             test_bit(__I40E_CONFIG_BUSY, pf->state))
 809                 return;
 810
 811         ns = i40e_get_vsi_stats_struct(vsi);
 812         ons = &vsi->net_stats_offsets;
 813         es = &vsi->eth_stats;
 814         oes = &vsi->eth_stats_offsets;
 815
 816         /* Gather up the netdev and vsi stats that the driver collects
 817          * on the fly during packet processing
 818          */
 819         rx_b = rx_p = 0;
 820         tx_b = tx_p = 0;
 821         tx_restart = tx_busy = tx_linearize = tx_force_wb = 0;
 822         rx_page = 0;
 823         rx_buf = 0;
 824         rcu_read_lock();
 825         for (q = 0; q < vsi->num_queue_pairs; q++) {
 826                 /* locate Tx ring */
 827                 p = READ_ONCE(vsi->tx_rings[q]);
 828                 if (!p)
 829                         continue;
 830
 831                 do {
 832                         start = u64_stats_fetch_begin_irq(&p->syncp);
 833                         packets = p->stats.packets;
 834                         bytes = p->stats.bytes;
 835                 } while (u64_stats_fetch_retry_irq(&p->syncp, start));
 836                 tx_b += bytes;
 837                 tx_p += packets;
 838                 tx_restart += p->tx_stats.restart_queue;
 839                 tx_busy += p->tx_stats.tx_busy;
 840                 tx_linearize += p->tx_stats.tx_linearize;
 841                 tx_force_wb += p->tx_stats.tx_force_wb;
 842
 843                 /* locate Rx ring */
 844                 p = READ_ONCE(vsi->rx_rings[q]);
 845                 if (!p)
 846                         continue;
 847
 848                 do {
 849                         start = u64_stats_fetch_begin_irq(&p->syncp);
 850                         packets = p->stats.packets;
 851                         bytes = p->stats.bytes;
 852                 } while (u64_stats_fetch_retry_irq(&p->syncp, start));
 853                 rx_b += bytes;
 854                 rx_p += packets;
 855                 rx_buf += p->rx_stats.alloc_buff_failed;
 856                 rx_page += p->rx_stats.alloc_page_failed;
 857         }
 858         rcu_read_unlock();
 859         vsi->tx_restart = tx_restart;
 860         vsi->tx_busy = tx_busy;
 861         vsi->tx_linearize = tx_linearize;
 862         vsi->tx_force_wb = tx_force_wb;
 863         vsi->rx_page_failed = rx_page;
 864         vsi->rx_buf_failed = rx_buf;
 865
 866         ns->rx_packets = rx_p;
 867         ns->rx_bytes = rx_b;
 868         ns->tx_packets = tx_p;
 869         ns->tx_bytes = tx_b;
 870
 871         /* update netdev stats from eth stats */
 872         i40e_update_eth_stats(vsi);
 873         ons->tx_errors = oes->tx_errors;
 874         ns->tx_errors = es->tx_errors;
 875         ons->multicast = oes->rx_multicast;
 876         ns->multicast = es->rx_multicast;
 877         ons->rx_dropped = oes->rx_discards;
 878         ns->rx_dropped = es->rx_discards;
 879         ons->tx_dropped = oes->tx_discards;
 880         ns->tx_dropped = es->tx_discards;
 881
 882         /* pull in a couple PF stats if this is the main vsi */
 883         if (vsi == pf->vsi[pf->lan_vsi]) {
 884                 ns->rx_crc_errors = pf->stats.crc_errors;
 885                 ns->rx_errors = pf->stats.crc_errors + pf->stats.illegal_bytes;
 886                 ns->rx_length_errors = pf->stats.rx_length_errors;
 887         }
 888 }
 889
 890 /**
 891  * i40e_update_pf_stats - Update the PF statistics counters.
 892  * @pf: the PF to be updated
 893  **/
 894 static void i40e_update_pf_stats(struct i40e_pf *pf)
 895 {
 896         struct i40e_hw_port_stats *osd = &pf->stats_offsets;
 897         struct i40e_hw_port_stats *nsd = &pf->stats;
 898         struct i40e_hw *hw = &pf->hw;
 899         u32 val;
 900         int i;
 901
 902         i40e_stat_update48(hw, I40E_GLPRT_GORCH(hw->port),
 903                            I40E_GLPRT_GORCL(hw->port),
 904                            pf->stat_offsets_loaded,
 905                            &osd->eth.rx_bytes, &nsd->eth.rx_bytes);
 906         i40e_stat_update48(hw, I40E_GLPRT_GOTCH(hw->port),
 907                            I40E_GLPRT_GOTCL(hw->port),
 908                            pf->stat_offsets_loaded,
 909                            &osd->eth.tx_bytes, &nsd->eth.tx_bytes);
 910         i40e_stat_update32(hw, I40E_GLPRT_RDPC(hw->port),
 911                            pf->stat_offsets_loaded,
 912                            &osd->eth.rx_discards,
 913                            &nsd->eth.rx_discards);
 914         i40e_stat_update48(hw, I40E_GLPRT_UPRCH(hw->port),
 915                            I40E_GLPRT_UPRCL(hw->port),
 916                            pf->stat_offsets_loaded,
 917                            &osd->eth.rx_unicast,
 918                            &nsd->eth.rx_unicast);
 919         i40e_stat_update48(hw, I40E_GLPRT_MPRCH(hw->port),
 920                            I40E_GLPRT_MPRCL(hw->port),
 921                            pf->stat_offsets_loaded,
 922                            &osd->eth.rx_multicast,
 923                            &nsd->eth.rx_multicast);
 924         i40e_stat_update48(hw, I40E_GLPRT_BPRCH(hw->port),
 925                            I40E_GLPRT_BPRCL(hw->port),
 926                            pf->stat_offsets_loaded,
 927                            &osd->eth.rx_broadcast,
 928                            &nsd->eth.rx_broadcast);
 929         i40e_stat_update48(hw, I40E_GLPRT_UPTCH(hw->port),
 930                            I40E_GLPRT_UPTCL(hw->port),
 931                            pf->stat_offsets_loaded,
 932                            &osd->eth.tx_unicast,
 933                            &nsd->eth.tx_unicast);
 934         i40e_stat_update48(hw, I40E_GLPRT_MPTCH(hw->port),
 935                            I40E_GLPRT_MPTCL(hw->port),
 936                            pf->stat_offsets_loaded,
 937                            &osd->eth.tx_multicast,
 938                            &nsd->eth.tx_multicast);
 939         i40e_stat_update48(hw, I40E_GLPRT_BPTCH(hw->port),
 940                            I40E_GLPRT_BPTCL(hw->port),
 941                            pf->stat_offsets_loaded,
 942                            &osd->eth.tx_broadcast,
 943                            &nsd->eth.tx_broadcast);
 944
 945         i40e_stat_update32(hw, I40E_GLPRT_TDOLD(hw->port),
 946                            pf->stat_offsets_loaded,
 947                            &osd->tx_dropped_link_down,
 948                            &nsd->tx_dropped_link_down);
 949
 950         i40e_stat_update32(hw, I40E_GLPRT_CRCERRS(hw->port),
 951                            pf->stat_offsets_loaded,
 952                            &osd->crc_errors, &nsd->crc_errors);
 953
 954         i40e_stat_update32(hw, I40E_GLPRT_ILLERRC(hw->port),
 955                            pf->stat_offsets_loaded,
 956                            &osd->illegal_bytes, &nsd->illegal_bytes);
 957
 958         i40e_stat_update32(hw, I40E_GLPRT_MLFC(hw->port),
 959                            pf->stat_offsets_loaded,
 960                            &osd->mac_local_faults,
 961                            &nsd->mac_local_faults);
 962         i40e_stat_update32(hw, I40E_GLPRT_MRFC(hw->port),
 963                            pf->stat_offsets_loaded,
 964                            &osd->mac_remote_faults,
 965                            &nsd->mac_remote_faults);
 966
 967         i40e_stat_update32(hw, I40E_GLPRT_RLEC(hw->port),
 968                            pf->stat_offsets_loaded,
 969                            &osd->rx_length_errors,
 970                            &nsd->rx_length_errors);
 971
 972         i40e_stat_update32(hw, I40E_GLPRT_LXONRXC(hw->port),
 973                            pf->stat_offsets_loaded,
 974                            &osd->link_xon_rx, &nsd->link_xon_rx);
 975         i40e_stat_update32(hw, I40E_GLPRT_LXONTXC(hw->port),
 976                            pf->stat_offsets_loaded,
 977                            &osd->link_xon_tx, &nsd->link_xon_tx);
 978         i40e_stat_update32(hw, I40E_GLPRT_LXOFFRXC(hw->port),
 979                            pf->stat_offsets_loaded,
 980                            &osd->link_xoff_rx, &nsd->link_xoff_rx);
 981         i40e_stat_update32(hw, I40E_GLPRT_LXOFFTXC(hw->port),
 982                            pf->stat_offsets_loaded,
 983                            &osd->link_xoff_tx, &nsd->link_xoff_tx);
 984
 985         for (i = 0; i < 8; i++) {
 986                 i40e_stat_update32(hw, I40E_GLPRT_PXOFFRXC(hw->port, i),
 987                                    pf->stat_offsets_loaded,
 988                                    &osd->priority_xoff_rx[i],
 989                                    &nsd->priority_xoff_rx[i]);
 990                 i40e_stat_update32(hw, I40E_GLPRT_PXONRXC(hw->port, i),
 991                                    pf->stat_offsets_loaded,
 992                                    &osd->priority_xon_rx[i],
 993                                    &nsd->priority_xon_rx[i]);
 994                 i40e_stat_update32(hw, I40E_GLPRT_PXONTXC(hw->port, i),
 995                                    pf->stat_offsets_loaded,
 996                                    &osd->priority_xon_tx[i],
 997                                    &nsd->priority_xon_tx[i]);
 998                 i40e_stat_update32(hw, I40E_GLPRT_PXOFFTXC(hw->port, i),
 999                                    pf->stat_offsets_loaded,
1000                                    &osd->priority_xoff_tx[i],
1001                                    &nsd->priority_xoff_tx[i]);
1002                 i40e_stat_update32(hw,
1003                                    I40E_GLPRT_RXON2OFFCNT(hw->port, i),
1004                                    pf->stat_offsets_loaded,
1005                                    &osd->priority_xon_2_xoff[i],
1006                                    &nsd->priority_xon_2_xoff[i]);
1007         }
1008
1009         i40e_stat_update48(hw, I40E_GLPRT_PRC64H(hw->port),
1010                            I40E_GLPRT_PRC64L(hw->port),
1011                            pf->stat_offsets_loaded,
1012                            &osd->rx_size_64, &nsd->rx_size_64);
1013         i40e_stat_update48(hw, I40E_GLPRT_PRC127H(hw->port),
1014                            I40E_GLPRT_PRC127L(hw->port),
1015                            pf->stat_offsets_loaded,
1016                            &osd->rx_size_127, &nsd->rx_size_127);
1017         i40e_stat_update48(hw, I40E_GLPRT_PRC255H(hw->port),
1018                            I40E_GLPRT_PRC255L(hw->port),
1019                            pf->stat_offsets_loaded,
1020                            &osd->rx_size_255, &nsd->rx_size_255);
1021         i40e_stat_update48(hw, I40E_GLPRT_PRC511H(hw->port),
1022                            I40E_GLPRT_PRC511L(hw->port),
1023                            pf->stat_offsets_loaded,
1024                            &osd->rx_size_511, &nsd->rx_size_511);
1025         i40e_stat_update48(hw, I40E_GLPRT_PRC1023H(hw->port),
1026                            I40E_GLPRT_PRC1023L(hw->port),
1027                            pf->stat_offsets_loaded,
1028                            &osd->rx_size_1023, &nsd->rx_size_1023);
1029         i40e_stat_update48(hw, I40E_GLPRT_PRC1522H(hw->port),
1030                            I40E_GLPRT_PRC1522L(hw->port),
1031                            pf->stat_offsets_loaded,
1032                            &osd->rx_size_1522, &nsd->rx_size_1522);
1033         i40e_stat_update48(hw, I40E_GLPRT_PRC9522H(hw->port),
1034                            I40E_GLPRT_PRC9522L(hw->port),
1035                            pf->stat_offsets_loaded,
1036                            &osd->rx_size_big, &nsd->rx_size_big);
1037
1038         i40e_stat_update48(hw, I40E_GLPRT_PTC64H(hw->port),
1039                            I40E_GLPRT_PTC64L(hw->port),
1040                            pf->stat_offsets_loaded,
1041                            &osd->tx_size_64, &nsd->tx_size_64);
1042         i40e_stat_update48(hw, I40E_GLPRT_PTC127H(hw->port),
1043                            I40E_GLPRT_PTC127L(hw->port),
1044                            pf->stat_offsets_loaded,
1045                            &osd->tx_size_127, &nsd->tx_size_127);
1046         i40e_stat_update48(hw, I40E_GLPRT_PTC255H(hw->port),
1047                            I40E_GLPRT_PTC255L(hw->port),
1048                            pf->stat_offsets_loaded,
1049                            &osd->tx_size_255, &nsd->tx_size_255);
1050         i40e_stat_update48(hw, I40E_GLPRT_PTC511H(hw->port),
1051                            I40E_GLPRT_PTC511L(hw->port),
1052                            pf->stat_offsets_loaded,
1053                            &osd->tx_size_511, &nsd->tx_size_511);
1054         i40e_stat_update48(hw, I40E_GLPRT_PTC1023H(hw->port),
1055                            I40E_GLPRT_PTC1023L(hw->port),
1056                            pf->stat_offsets_loaded,
1057                            &osd->tx_size_1023, &nsd->tx_size_1023);
1058         i40e_stat_update48(hw, I40E_GLPRT_PTC1522H(hw->port),
1059                            I40E_GLPRT_PTC1522L(hw->port),
1060                            pf->stat_offsets_loaded,
1061                            &osd->tx_size_1522, &nsd->tx_size_1522);
1062         i40e_stat_update48(hw, I40E_GLPRT_PTC9522H(hw->port),
1063                            I40E_GLPRT_PTC9522L(hw->port),
1064                            pf->stat_offsets_loaded,
1065                            &osd->tx_size_big, &nsd->tx_size_big);
1066
1067         i40e_stat_update32(hw, I40E_GLPRT_RUC(hw->port),
1068                            pf->stat_offsets_loaded,
1069                            &osd->rx_undersize, &nsd->rx_undersize);
1070         i40e_stat_update32(hw, I40E_GLPRT_RFC(hw->port),
1071                            pf->stat_offsets_loaded,
1072                            &osd->rx_fragments, &nsd->rx_fragments);
1073         i40e_stat_update32(hw, I40E_GLPRT_ROC(hw->port),
1074                            pf->stat_offsets_loaded,
1075                            &osd->rx_oversize, &nsd->rx_oversize);
1076         i40e_stat_update32(hw, I40E_GLPRT_RJC(hw->port),
1077                            pf->stat_offsets_loaded,
1078                            &osd->rx_jabber, &nsd->rx_jabber);
1079
1080         /* FDIR stats */
1081         i40e_stat_update_and_clear32(hw,
1082                         I40E_GLQF_PCNT(I40E_FD_ATR_STAT_IDX(hw->pf_id)),
1083                         &nsd->fd_atr_match);
1084         i40e_stat_update_and_clear32(hw,
1085                         I40E_GLQF_PCNT(I40E_FD_SB_STAT_IDX(hw->pf_id)),
1086                         &nsd->fd_sb_match);
1087         i40e_stat_update_and_clear32(hw,
1088                         I40E_GLQF_PCNT(I40E_FD_ATR_TUNNEL_STAT_IDX(hw->pf_id)),
1089                         &nsd->fd_atr_tunnel_match);
1090
1091         val = rd32(hw, I40E_PRTPM_EEE_STAT);
1092         nsd->tx_lpi_status =
1093                        (val & I40E_PRTPM_EEE_STAT_TX_LPI_STATUS_MASK) >>
1094                         I40E_PRTPM_EEE_STAT_TX_LPI_STATUS_SHIFT;
1095         nsd->rx_lpi_status =
1096                        (val & I40E_PRTPM_EEE_STAT_RX_LPI_STATUS_MASK) >>
1097                         I40E_PRTPM_EEE_STAT_RX_LPI_STATUS_SHIFT;
1098         i40e_stat_update32(hw, I40E_PRTPM_TLPIC,
1099                            pf->stat_offsets_loaded,
1100                            &osd->tx_lpi_count, &nsd->tx_lpi_count);
1101         i40e_stat_update32(hw, I40E_PRTPM_RLPIC,
1102                            pf->stat_offsets_loaded,
1103                            &osd->rx_lpi_count, &nsd->rx_lpi_count);
1104
1105         if (pf->flags & I40E_FLAG_FD_SB_ENABLED &&
1106             !test_bit(__I40E_FD_SB_AUTO_DISABLED, pf->state))
1107                 nsd->fd_sb_status = true;
1108         else
1109                 nsd->fd_sb_status = false;
1110
1111         if (pf->flags & I40E_FLAG_FD_ATR_ENABLED &&
1112             !test_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state))
1113                 nsd->fd_atr_status = true;
1114         else
1115                 nsd->fd_atr_status = false;
1116
1117         pf->stat_offsets_loaded = true;
1118 }
1119
1120 /**
1121  * i40e_update_stats - Update the various statistics counters.
1122  * @vsi: the VSI to be updated
1123  *
1124  * Update the various stats for this VSI and its related entities.
1125  **/
1126 void i40e_update_stats(struct i40e_vsi *vsi)
1127 {
1128         struct i40e_pf *pf = vsi->back;
1129
1130         if (vsi == pf->vsi[pf->lan_vsi])
1131                 i40e_update_pf_stats(pf);
1132
1133         i40e_update_vsi_stats(vsi);
1134 }
1135
1136 /**
1137  * i40e_find_filter - Search VSI filter list for specific mac/vlan filter
1138  * @vsi: the VSI to be searched
1139  * @macaddr: the MAC address
1140  * @vlan: the vlan
1141  *
1142  * Returns ptr to the filter object or NULL
1143  **/
1144 static struct i40e_mac_filter *i40e_find_filter(struct i40e_vsi *vsi,
1145                                                 const u8 *macaddr, s16 vlan)
1146 {
1147         struct i40e_mac_filter *f;
1148         u64 key;
1149
1150         if (!vsi || !macaddr)
1151                 return NULL;
1152
1153         key = i40e_addr_to_hkey(macaddr);
1154         hash_for_each_possible(vsi->mac_filter_hash, f, hlist, key) {
1155                 if ((ether_addr_equal(macaddr, f->macaddr)) &&
1156                     (vlan == f->vlan))
1157                         return f;
1158         }
1159         return NULL;
1160 }
1161
1162 /**
1163  * i40e_find_mac - Find a mac addr in the macvlan filters list
1164  * @vsi: the VSI to be searched
1165  * @macaddr: the MAC address we are searching for
1166  *
1167  * Returns the first filter with the provided MAC address or NULL if
1168  * MAC address was not found
1169  **/
1170 struct i40e_mac_filter *i40e_find_mac(struct i40e_vsi *vsi, const u8 *macaddr)
1171 {
1172         struct i40e_mac_filter *f;
1173         u64 key;
1174
1175         if (!vsi || !macaddr)
1176                 return NULL;
1177
1178         key = i40e_addr_to_hkey(macaddr);
1179         hash_for_each_possible(vsi->mac_filter_hash, f, hlist, key) {
1180                 if ((ether_addr_equal(macaddr, f->macaddr)))
1181                         return f;
1182         }
1183         return NULL;
1184 }
1185
1186 /**
1187  * i40e_is_vsi_in_vlan - Check if VSI is in vlan mode
1188  * @vsi: the VSI to be searched
1189  *
1190  * Returns true if VSI is in vlan mode or false otherwise
1191  **/
1192 bool i40e_is_vsi_in_vlan(struct i40e_vsi *vsi)
1193 {
1194         /* If we have a PVID, always operate in VLAN mode */
1195         if (vsi->info.pvid)
1196                 return true;
1197
1198         /* We need to operate in VLAN mode whenever we have any filters with
1199          * a VLAN other than I40E_VLAN_ALL. We could check the table each
1200          * time, incurring search cost repeatedly. However, we can notice two
1201          * things:
1202          *
1203          * 1) the only place where we can gain a VLAN filter is in
1204          *    i40e_add_filter.
1205          *
1206          * 2) the only place where filters are actually removed is in
1207          *    i40e_sync_filters_subtask.
1208          *
1209          * Thus, we can simply use a boolean value, has_vlan_filters which we
1210          * will set to true when we add a VLAN filter in i40e_add_filter. Then
1211          * we have to perform the full search after deleting filters in
1212          * i40e_sync_filters_subtask, but we already have to search
1213          * filters here and can perform the check at the same time. This
1214          * results in avoiding embedding a loop for VLAN mode inside another
1215          * loop over all the filters, and should maintain correctness as noted
1216          * above.
1217          */
1218         return vsi->has_vlan_filter;
1219 }
1220
1221 /**
1222  * i40e_correct_mac_vlan_filters - Correct non-VLAN filters if necessary
1223  * @vsi: the VSI to configure
1224  * @tmp_add_list: list of filters ready to be added
1225  * @tmp_del_list: list of filters ready to be deleted
1226  * @vlan_filters: the number of active VLAN filters
1227  *
1228  * Update VLAN=0 and VLAN=-1 (I40E_VLAN_ANY) filters properly so that they
1229  * behave as expected. If we have any active VLAN filters remaining or about
1230  * to be added then we need to update non-VLAN filters to be marked as VLAN=0
1231  * so that they only match against untagged traffic. If we no longer have any
1232  * active VLAN filters, we need to make all non-VLAN filters marked as VLAN=-1
1233  * so that they match against both tagged and untagged traffic. In this way,
1234  * we ensure that we correctly receive the desired traffic. This ensures that
1235  * when we have an active VLAN we will receive only untagged traffic and
1236  * traffic matching active VLANs. If we have no active VLANs then we will
1237  * operate in non-VLAN mode and receive all traffic, tagged or untagged.
1238  *
1239  * Finally, in a similar fashion, this function also corrects filters when
1240  * there is an active PVID assigned to this VSI.
1241  *
1242  * In case of memory allocation failure return -ENOMEM. Otherwise, return 0.
1243  *
1244  * This function is only expected to be called from within
1245  * i40e_sync_vsi_filters.
1246  *
1247  * NOTE: This function expects to be called while under the
1248  * mac_filter_hash_lock
1249  */
1250 static int i40e_correct_mac_vlan_filters(struct i40e_vsi *vsi,
1251                                          struct hlist_head *tmp_add_list,
1252                                          struct hlist_head *tmp_del_list,
1253                                          int vlan_filters)
1254 {
1255         s16 pvid = le16_to_cpu(vsi->info.pvid);
1256         struct i40e_mac_filter *f, *add_head;
1257         struct i40e_new_mac_filter *new;
1258         struct hlist_node *h;
1259         int bkt, new_vlan;
1260
1261         /* To determine if a particular filter needs to be replaced we
1262          * have the three following conditions:
1263          *
1264          * a) if we have a PVID assigned, then all filters which are
1265          *    not marked as VLAN=PVID must be replaced with filters that
1266          *    are.
1267          * b) otherwise, if we have any active VLANS, all filters
1268          *    which are marked as VLAN=-1 must be replaced with
1269          *    filters marked as VLAN=0
1270          * c) finally, if we do not have any active VLANS, all filters
1271          *    which are marked as VLAN=0 must be replaced with filters
1272          *    marked as VLAN=-1
1273          */
1274
1275         /* Update the filters about to be added in place */
1276         hlist_for_each_entry(new, tmp_add_list, hlist) {
1277                 if (pvid && new->f->vlan != pvid)
1278                         new->f->vlan = pvid;
1279                 else if (vlan_filters && new->f->vlan == I40E_VLAN_ANY)
1280                         new->f->vlan = 0;
1281                 else if (!vlan_filters && new->f->vlan == 0)
1282                         new->f->vlan = I40E_VLAN_ANY;
1283         }
1284
1285         /* Update the remaining active filters */
1286         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
1287                 /* Combine the checks for whether a filter needs to be changed
1288                  * and then determine the new VLAN inside the if block, in
1289                  * order to avoid duplicating code for adding the new filter
1290                  * then deleting the old filter.
1291                  */
1292                 if ((pvid && f->vlan != pvid) ||
1293                     (vlan_filters && f->vlan == I40E_VLAN_ANY) ||
1294                     (!vlan_filters && f->vlan == 0)) {
1295                         /* Determine the new vlan we will be adding */
1296                         if (pvid)
1297                                 new_vlan = pvid;
1298                         else if (vlan_filters)
1299                                 new_vlan = 0;
1300                         else
1301                                 new_vlan = I40E_VLAN_ANY;
1302
1303                         /* Create the new filter */
1304                         add_head = i40e_add_filter(vsi, f->macaddr, new_vlan);
1305                         if (!add_head)
1306                                 return -ENOMEM;
1307
1308                         /* Create a temporary i40e_new_mac_filter */
1309                         new = kzalloc(sizeof(*new), GFP_ATOMIC);
1310                         if (!new)
1311                                 return -ENOMEM;
1312
1313                         new->f = add_head;
1314                         new->state = add_head->state;
1315
1316                         /* Add the new filter to the tmp list */
1317                         hlist_add_head(&new->hlist, tmp_add_list);
1318
1319                         /* Put the original filter into the delete list */
1320                         f->state = I40E_FILTER_REMOVE;
1321                         hash_del(&f->hlist);
1322                         hlist_add_head(&f->hlist, tmp_del_list);
1323                 }
1324         }
1325
1326         vsi->has_vlan_filter = !!vlan_filters;
1327
1328         return 0;
1329 }
1330
1331 /**
1332  * i40e_rm_default_mac_filter - Remove the default MAC filter set by NVM
1333  * @vsi: the PF Main VSI - inappropriate for any other VSI
1334  * @macaddr: the MAC address
1335  *
1336  * Remove whatever filter the firmware set up so the driver can manage
1337  * its own filtering intelligently.
1338  **/
1339 static void i40e_rm_default_mac_filter(struct i40e_vsi *vsi, u8 *macaddr)
1340 {
1341         struct i40e_aqc_remove_macvlan_element_data element;
1342         struct i40e_pf *pf = vsi->back;
1343
1344         /* Only appropriate for the PF main VSI */
1345         if (vsi->type != I40E_VSI_MAIN)
1346                 return;
1347
1348         memset(&element, 0, sizeof(element));
1349         ether_addr_copy(element.mac_addr, macaddr);
1350         element.vlan_tag = 0;
1351         /* Ignore error returns, some firmware does it this way... */
1352         element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;
1353         i40e_aq_remove_macvlan(&pf->hw, vsi->seid, &element, 1, NULL);
1354
1355         memset(&element, 0, sizeof(element));
1356         ether_addr_copy(element.mac_addr, macaddr);
1357         element.vlan_tag = 0;
1358         /* ...and some firmware does it this way. */
1359         element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH |
1360                         I40E_AQC_MACVLAN_DEL_IGNORE_VLAN;
1361         i40e_aq_remove_macvlan(&pf->hw, vsi->seid, &element, 1, NULL);
1362 }
1363
1364 /**
1365  * i40e_add_filter - Add a mac/vlan filter to the VSI
1366  * @vsi: the VSI to be searched
1367  * @macaddr: the MAC address
1368  * @vlan: the vlan
1369  *
1370  * Returns ptr to the filter object or NULL when no memory available.
1371  *
1372  * NOTE: This function is expected to be called with mac_filter_hash_lock
1373  * being held.
1374  **/
1375 struct i40e_mac_filter *i40e_add_filter(struct i40e_vsi *vsi,
1376                                         const u8 *macaddr, s16 vlan)
1377 {
1378         struct i40e_mac_filter *f;
1379         u64 key;
1380
1381         if (!vsi || !macaddr)
1382                 return NULL;
1383
1384         f = i40e_find_filter(vsi, macaddr, vlan);
1385         if (!f) {
1386                 f = kzalloc(sizeof(*f), GFP_ATOMIC);
1387                 if (!f)
1388                         return NULL;
1389
1390                 /* Update the boolean indicating if we need to function in
1391                  * VLAN mode.
1392                  */
1393                 if (vlan >= 0)
1394                         vsi->has_vlan_filter = true;
1395
1396                 ether_addr_copy(f->macaddr, macaddr);
1397                 f->vlan = vlan;
1398                 f->state = I40E_FILTER_NEW;
1399                 INIT_HLIST_NODE(&f->hlist);
1400
1401                 key = i40e_addr_to_hkey(macaddr);
1402                 hash_add(vsi->mac_filter_hash, &f->hlist, key);
1403
1404                 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
1405                 set_bit(__I40E_MACVLAN_SYNC_PENDING, vsi->back->state);
1406         }
1407
1408         /* If we're asked to add a filter that has been marked for removal, it
1409          * is safe to simply restore it to active state. __i40e_del_filter
1410          * will have simply deleted any filters which were previously marked
1411          * NEW or FAILED, so if it is currently marked REMOVE it must have
1412          * previously been ACTIVE. Since we haven't yet run the sync filters
1413          * task, just restore this filter to the ACTIVE state so that the
1414          * sync task leaves it in place
1415          */
1416         if (f->state == I40E_FILTER_REMOVE)
1417                 f->state = I40E_FILTER_ACTIVE;
1418
1419         return f;
1420 }
1421
1422 /**
1423  * __i40e_del_filter - Remove a specific filter from the VSI
1424  * @vsi: VSI to remove from
1425  * @f: the filter to remove from the list
1426  *
1427  * This function should be called instead of i40e_del_filter only if you know
1428  * the exact filter you will remove already, such as via i40e_find_filter or
1429  * i40e_find_mac.
1430  *
1431  * NOTE: This function is expected to be called with mac_filter_hash_lock
1432  * being held.
1433  * ANOTHER NOTE: This function MUST be called from within the context of
1434  * the "safe" variants of any list iterators, e.g. list_for_each_entry_safe()
1435  * instead of list_for_each_entry().
1436  **/
1437 void __i40e_del_filter(struct i40e_vsi *vsi, struct i40e_mac_filter *f)
1438 {
1439         if (!f)
1440                 return;
1441
1442         /* If the filter was never added to firmware then we can just delete it
1443          * directly and we don't want to set the status to remove or else an
1444          * admin queue command will unnecessarily fire.
1445          */
1446         if ((f->state == I40E_FILTER_FAILED) ||
1447             (f->state == I40E_FILTER_NEW)) {
1448                 hash_del(&f->hlist);
1449                 kfree(f);
1450         } else {
1451                 f->state = I40E_FILTER_REMOVE;
1452         }
1453
1454         vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
1455         set_bit(__I40E_MACVLAN_SYNC_PENDING, vsi->back->state);
1456 }
1457
1458 /**
1459  * i40e_del_filter - Remove a MAC/VLAN filter from the VSI
1460  * @vsi: the VSI to be searched
1461  * @macaddr: the MAC address
1462  * @vlan: the VLAN
1463  *
1464  * NOTE: This function is expected to be called with mac_filter_hash_lock
1465  * being held.
1466  * ANOTHER NOTE: This function MUST be called from within the context of
1467  * the "safe" variants of any list iterators, e.g. list_for_each_entry_safe()
1468  * instead of list_for_each_entry().
1469  **/
1470 void i40e_del_filter(struct i40e_vsi *vsi, const u8 *macaddr, s16 vlan)
1471 {
1472         struct i40e_mac_filter *f;
1473
1474         if (!vsi || !macaddr)
1475                 return;
1476
1477         f = i40e_find_filter(vsi, macaddr, vlan);
1478         __i40e_del_filter(vsi, f);
1479 }
1480
1481 /**
1482  * i40e_add_mac_filter - Add a MAC filter for all active VLANs
1483  * @vsi: the VSI to be searched
1484  * @macaddr: the mac address to be filtered
1485  *
1486  * If we're not in VLAN mode, just add the filter to I40E_VLAN_ANY. Otherwise,
1487  * go through all the macvlan filters and add a macvlan filter for each
1488  * unique vlan that already exists. If a PVID has been assigned, instead only
1489  * add the macaddr to that VLAN.
1490  *
1491  * Returns last filter added on success, else NULL
1492  **/
1493 struct i40e_mac_filter *i40e_add_mac_filter(struct i40e_vsi *vsi,
1494                                             const u8 *macaddr)
1495 {
1496         struct i40e_mac_filter *f, *add = NULL;
1497         struct hlist_node *h;
1498         int bkt;
1499
1500         if (vsi->info.pvid)
1501                 return i40e_add_filter(vsi, macaddr,
1502                                        le16_to_cpu(vsi->info.pvid));
1503
1504         if (!i40e_is_vsi_in_vlan(vsi))
1505                 return i40e_add_filter(vsi, macaddr, I40E_VLAN_ANY);
1506
1507         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
1508                 if (f->state == I40E_FILTER_REMOVE)
1509                         continue;
1510                 add = i40e_add_filter(vsi, macaddr, f->vlan);
1511                 if (!add)
1512                         return NULL;
1513         }
1514
1515         return add;
1516 }
1517
1518 /**
1519  * i40e_del_mac_filter - Remove a MAC filter from all VLANs
1520  * @vsi: the VSI to be searched
1521  * @macaddr: the mac address to be removed
1522  *
1523  * Removes a given MAC address from a VSI regardless of what VLAN it has been
1524  * associated with.
1525  *
1526  * Returns 0 for success, or error
1527  **/
1528 int i40e_del_mac_filter(struct i40e_vsi *vsi, const u8 *macaddr)
1529 {
1530         struct i40e_mac_filter *f;
1531         struct hlist_node *h;
1532         bool found = false;
1533         int bkt;
1534
1535         WARN(!spin_is_locked(&vsi->mac_filter_hash_lock),
1536              "Missing mac_filter_hash_lock\n");
1537         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
1538                 if (ether_addr_equal(macaddr, f->macaddr)) {
1539                         __i40e_del_filter(vsi, f);
1540                         found = true;
1541                 }
1542         }
1543
1544         if (found)
1545                 return 0;
1546         else
1547                 return -ENOENT;
1548 }
1549
1550 /**
1551  * i40e_set_mac - NDO callback to set mac address
1552  * @netdev: network interface device structure
1553  * @p: pointer to an address structure
1554  *
1555  * Returns 0 on success, negative on failure
1556  **/
1557 static int i40e_set_mac(struct net_device *netdev, void *p)
1558 {
1559         struct i40e_netdev_priv *np = netdev_priv(netdev);
1560         struct i40e_vsi *vsi = np->vsi;
1561         struct i40e_pf *pf = vsi->back;
1562         struct i40e_hw *hw = &pf->hw;
1563         struct sockaddr *addr = p;
1564
1565         if (!is_valid_ether_addr(addr->sa_data))
1566                 return -EADDRNOTAVAIL;
1567
1568         if (ether_addr_equal(netdev->dev_addr, addr->sa_data)) {
1569                 netdev_info(netdev, "already using mac address %pM\n",
1570                             addr->sa_data);
1571                 return 0;
1572         }
1573
1574         if (test_bit(__I40E_VSI_DOWN, vsi->back->state) ||
1575             test_bit(__I40E_RESET_RECOVERY_PENDING, vsi->back->state))
1576                 return -EADDRNOTAVAIL;
1577
1578         if (ether_addr_equal(hw->mac.addr, addr->sa_data))
1579                 netdev_info(netdev, "returning to hw mac address %pM\n",
1580                             hw->mac.addr);
1581         else
1582                 netdev_info(netdev, "set new mac address %pM\n", addr->sa_data);
1583
1584         /* Copy the address first, so that we avoid a possible race with
1585          * .set_rx_mode().
1586          * - Remove old address from MAC filter
1587          * - Copy new address
1588          * - Add new address to MAC filter
1589          */
1590         spin_lock_bh(&vsi->mac_filter_hash_lock);
1591         i40e_del_mac_filter(vsi, netdev->dev_addr);
1592         ether_addr_copy(netdev->dev_addr, addr->sa_data);
1593         i40e_add_mac_filter(vsi, netdev->dev_addr);
1594         spin_unlock_bh(&vsi->mac_filter_hash_lock);
1595
1596         if (vsi->type == I40E_VSI_MAIN) {
1597                 i40e_status ret;
1598
1599                 ret = i40e_aq_mac_address_write(&vsi->back->hw,
1600                                                 I40E_AQC_WRITE_TYPE_LAA_WOL,
1601                                                 addr->sa_data, NULL);
1602                 if (ret)
1603                         netdev_info(netdev, "Ignoring error from firmware on LAA update, status %s, AQ ret %s\n",
1604                                     i40e_stat_str(hw, ret),
1605                                     i40e_aq_str(hw, hw->aq.asq_last_status));
1606         }
1607
1608         /* schedule our worker thread which will take care of
1609          * applying the new filter changes
1610          */
1611         i40e_service_event_schedule(vsi->back);
1612         return 0;
1613 }
1614
1615 /**
1616  * i40e_config_rss_aq - Prepare for RSS using AQ commands
1617  * @vsi: vsi structure
1618  * @seed: RSS hash seed
1619  **/
1620 static int i40e_config_rss_aq(struct i40e_vsi *vsi, const u8 *seed,
1621                               u8 *lut, u16 lut_size)
1622 {
1623         struct i40e_pf *pf = vsi->back;
1624         struct i40e_hw *hw = &pf->hw;
1625         int ret = 0;
1626
1627         if (seed) {
1628                 struct i40e_aqc_get_set_rss_key_data *seed_dw =
1629                         (struct i40e_aqc_get_set_rss_key_data *)seed;
1630                 ret = i40e_aq_set_rss_key(hw, vsi->id, seed_dw);
1631                 if (ret) {
1632                         dev_info(&pf->pdev->dev,
1633                                  "Cannot set RSS key, err %s aq_err %s\n",
1634                                  i40e_stat_str(hw, ret),
1635                                  i40e_aq_str(hw, hw->aq.asq_last_status));
1636                         return ret;
1637                 }
1638         }
1639         if (lut) {
1640                 bool pf_lut = vsi->type == I40E_VSI_MAIN ? true : false;
1641
1642                 ret = i40e_aq_set_rss_lut(hw, vsi->id, pf_lut, lut, lut_size);
1643                 if (ret) {
1644                         dev_info(&pf->pdev->dev,
1645                                  "Cannot set RSS lut, err %s aq_err %s\n",
1646                                  i40e_stat_str(hw, ret),
1647                                  i40e_aq_str(hw, hw->aq.asq_last_status));
1648                         return ret;
1649                 }
1650         }
1651         return ret;
1652 }
1653
1654 /**
1655  * i40e_vsi_config_rss - Prepare for VSI(VMDq) RSS if used
1656  * @vsi: VSI structure
1657  **/
1658 static int i40e_vsi_config_rss(struct i40e_vsi *vsi)
1659 {
1660         struct i40e_pf *pf = vsi->back;
1661         u8 seed[I40E_HKEY_ARRAY_SIZE];
1662         u8 *lut;
1663         int ret;
1664
1665         if (!(pf->hw_features & I40E_HW_RSS_AQ_CAPABLE))
1666                 return 0;
1667         if (!vsi->rss_size)
1668                 vsi->rss_size = min_t(int, pf->alloc_rss_size,
1669                                       vsi->num_queue_pairs);
1670         if (!vsi->rss_size)
1671                 return -EINVAL;
1672         lut = kzalloc(vsi->rss_table_size, GFP_KERNEL);
1673         if (!lut)
1674                 return -ENOMEM;
1675
1676         /* Use the user configured hash keys and lookup table if there is one,
1677          * otherwise use default
1678          */
1679         if (vsi->rss_lut_user)
1680                 memcpy(lut, vsi->rss_lut_user, vsi->rss_table_size);
1681         else
1682                 i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, vsi->rss_size);
1683         if (vsi->rss_hkey_user)
1684                 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE);
1685         else
1686                 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE);
1687         ret = i40e_config_rss_aq(vsi, seed, lut, vsi->rss_table_size);
1688         kfree(lut);
1689         return ret;
1690 }
1691
1692 /**
1693  * i40e_vsi_setup_queue_map_mqprio - Prepares mqprio based tc_config
1694  * @vsi: the VSI being configured,
1695  * @ctxt: VSI context structure
1696  * @enabled_tc: number of traffic classes to enable
1697  *
1698  * Prepares VSI tc_config to have queue configurations based on MQPRIO options.
1699  **/
1700 static int i40e_vsi_setup_queue_map_mqprio(struct i40e_vsi *vsi,
1701                                            struct i40e_vsi_context *ctxt,
1702                                            u8 enabled_tc)
1703 {
1704         u16 qcount = 0, max_qcount, qmap, sections = 0;
1705         int i, override_q, pow, num_qps, ret;
1706         u8 netdev_tc = 0, offset = 0;
1707
1708         if (vsi->type != I40E_VSI_MAIN)
1709                 return -EINVAL;
1710         sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;
1711         sections |= I40E_AQ_VSI_PROP_SCHED_VALID;
1712         vsi->tc_config.numtc = vsi->mqprio_qopt.qopt.num_tc;
1713         vsi->tc_config.enabled_tc = enabled_tc ? enabled_tc : 1;
1714         num_qps = vsi->mqprio_qopt.qopt.count[0];
1715
1716         /* find the next higher power-of-2 of num queue pairs */
1717         pow = ilog2(num_qps);
1718         if (!is_power_of_2(num_qps))
1719                 pow++;
1720         qmap = (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
1721                 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT);
1722
1723         /* Setup queue offset/count for all TCs for given VSI */
1724         max_qcount = vsi->mqprio_qopt.qopt.count[0];
1725         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
1726                 /* See if the given TC is enabled for the given VSI */
1727                 if (vsi->tc_config.enabled_tc & BIT(i)) {
1728                         offset = vsi->mqprio_qopt.qopt.offset[i];
1729                         qcount = vsi->mqprio_qopt.qopt.count[i];
1730                         if (qcount > max_qcount)
1731                                 max_qcount = qcount;
1732                         vsi->tc_config.tc_info[i].qoffset = offset;
1733                         vsi->tc_config.tc_info[i].qcount = qcount;
1734                         vsi->tc_config.tc_info[i].netdev_tc = netdev_tc++;
1735                 } else {
1736                         /* TC is not enabled so set the offset to
1737                          * default queue and allocate one queue
1738                          * for the given TC.
1739                          */
1740                         vsi->tc_config.tc_info[i].qoffset = 0;
1741                         vsi->tc_config.tc_info[i].qcount = 1;
1742                         vsi->tc_config.tc_info[i].netdev_tc = 0;
1743                 }
1744         }
1745
1746         /* Set actual Tx/Rx queue pairs */
1747         vsi->num_queue_pairs = offset + qcount;
1748
1749         /* Setup queue TC[0].qmap for given VSI context */
1750         ctxt->info.tc_mapping[0] = cpu_to_le16(qmap);
1751         ctxt->info.mapping_flags |= cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG);
1752         ctxt->info.queue_mapping[0] = cpu_to_le16(vsi->base_queue);
1753         ctxt->info.valid_sections |= cpu_to_le16(sections);
1754
1755         /* Reconfigure RSS for main VSI with max queue count */
1756         vsi->rss_size = max_qcount;
1757         ret = i40e_vsi_config_rss(vsi);
1758         if (ret) {
1759                 dev_info(&vsi->back->pdev->dev,
1760                          "Failed to reconfig rss for num_queues (%u)\n",
1761                          max_qcount);
1762                 return ret;
1763         }
1764         vsi->reconfig_rss = true;
1765         dev_dbg(&vsi->back->pdev->dev,
1766                 "Reconfigured rss with num_queues (%u)\n", max_qcount);
1767
1768         /* Find queue count available for channel VSIs and starting offset
1769          * for channel VSIs
1770          */
1771         override_q = vsi->mqprio_qopt.qopt.count[0];
1772         if (override_q && override_q < vsi->num_queue_pairs) {
1773                 vsi->cnt_q_avail = vsi->num_queue_pairs - override_q;
1774                 vsi->next_base_queue = override_q;
1775         }
1776         return 0;
1777 }
1778
1779 /**
1780  * i40e_vsi_setup_queue_map - Setup a VSI queue map based on enabled_tc
1781  * @vsi: the VSI being setup
1782  * @ctxt: VSI context structure
1783  * @enabled_tc: Enabled TCs bitmap
1784  * @is_add: True if called before Add VSI
1785  *
1786  * Setup VSI queue mapping for enabled traffic classes.
1787  **/
1788 static void i40e_vsi_setup_queue_map(struct i40e_vsi *vsi,
1789                                      struct i40e_vsi_context *ctxt,
1790                                      u8 enabled_tc,
1791                                      bool is_add)
1792 {
1793         struct i40e_pf *pf = vsi->back;
1794         u16 num_tc_qps = 0;
1795         u16 sections = 0;
1796         u8 netdev_tc = 0;
1797         u16 numtc = 1;
1798         u16 qcount;
1799         u8 offset;
1800         u16 qmap;
1801         int i;
1802
1803         sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;
1804         offset = 0;
1805
1806         if (vsi->type == I40E_VSI_MAIN) {
1807                 /* This code helps add more queue to the VSI if we have
1808                  * more cores than RSS can support, the higher cores will
1809                  * be served by ATR or other filters. Furthermore, the
1810                  * non-zero req_queue_pairs says that user requested a new
1811                  * queue count via ethtool's set_channels, so use this
1812                  * value for queues distribution across traffic classes
1813                  * We need at least one queue pair for the interface
1814                  * to be usable as we see in else statement.
1815                  */
1816                 if (vsi->req_queue_pairs > 0)
1817                         vsi->num_queue_pairs = vsi->req_queue_pairs;
1818                 else if (pf->flags & I40E_FLAG_MSIX_ENABLED)
1819                         vsi->num_queue_pairs = pf->num_lan_msix;
1820                 else
1821                         vsi->num_queue_pairs = 1;
1822         }
1823
1824         /* Number of queues per enabled TC */
1825         if (vsi->type == I40E_VSI_MAIN)
1826                 num_tc_qps = vsi->num_queue_pairs;
1827         else
1828                 num_tc_qps = vsi->alloc_queue_pairs;
1829         if (enabled_tc && (vsi->back->flags & I40E_FLAG_DCB_ENABLED)) {
1830                 /* Find numtc from enabled TC bitmap */
1831                 for (i = 0, numtc = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
1832                         if (enabled_tc & BIT(i)) /* TC is enabled */
1833                                 numtc++;
1834                 }
1835                 if (!numtc) {
1836                         dev_warn(&pf->pdev->dev, "DCB is enabled but no TC enabled, forcing TC0\n");
1837                         numtc = 1;
1838                 }
1839                 num_tc_qps = num_tc_qps / numtc;
1840                 num_tc_qps = min_t(int, num_tc_qps,
1841                                    i40e_pf_get_max_q_per_tc(pf));
1842         }
1843
1844         vsi->tc_config.numtc = numtc;
1845         vsi->tc_config.enabled_tc = enabled_tc ? enabled_tc : 1;
1846
1847         /* Do not allow use more TC queue pairs than MSI-X vectors exist */
1848         if (pf->flags & I40E_FLAG_MSIX_ENABLED)
1849                 num_tc_qps = min_t(int, num_tc_qps, pf->num_lan_msix);
1850
1851         /* Setup queue offset/count for all TCs for given VSI */
1852         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
1853                 /* See if the given TC is enabled for the given VSI */
1854                 if (vsi->tc_config.enabled_tc & BIT(i)) {
1855                         /* TC is enabled */
1856                         int pow, num_qps;
1857
1858                         switch (vsi->type) {
1859                         case I40E_VSI_MAIN:
1860                                 if (!(pf->flags & (I40E_FLAG_FD_SB_ENABLED |
1861                                     I40E_FLAG_FD_ATR_ENABLED)) ||
1862                                     vsi->tc_config.enabled_tc != 1) {
1863                                         qcount = min_t(int, pf->alloc_rss_size,
1864                                                        num_tc_qps);
1865                                         break;
1866                                 }
1867                                 /* fall through */
1868                         case I40E_VSI_FDIR:
1869                         case I40E_VSI_SRIOV:
1870                         case I40E_VSI_VMDQ2:
1871                         default:
1872                                 qcount = num_tc_qps;
1873                                 WARN_ON(i != 0);
1874                                 break;
1875                         }
1876                         vsi->tc_config.tc_info[i].qoffset = offset;
1877                         vsi->tc_config.tc_info[i].qcount = qcount;
1878
1879                         /* find the next higher power-of-2 of num queue pairs */
1880                         num_qps = qcount;
1881                         pow = 0;
1882                         while (num_qps && (BIT_ULL(pow) < qcount)) {
1883                                 pow++;
1884                                 num_qps >>= 1;
1885                         }
1886
1887                         vsi->tc_config.tc_info[i].netdev_tc = netdev_tc++;
1888                         qmap =
1889                             (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
1890                             (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT);
1891
1892                         offset += qcount;
1893                 } else {
1894                         /* TC is not enabled so set the offset to
1895                          * default queue and allocate one queue
1896                          * for the given TC.
1897                          */
1898                         vsi->tc_config.tc_info[i].qoffset = 0;
1899                         vsi->tc_config.tc_info[i].qcount = 1;
1900                         vsi->tc_config.tc_info[i].netdev_tc = 0;
1901
1902                         qmap = 0;
1903                 }
1904                 ctxt->info.tc_mapping[i] = cpu_to_le16(qmap);
1905         }
1906         /* Do not change previously set num_queue_pairs for PFs */
1907         if ((vsi->type == I40E_VSI_MAIN && numtc != 1) ||
1908             vsi->type != I40E_VSI_MAIN)
1909                 vsi->num_queue_pairs = offset;
1910         /* Scheduler section valid can only be set for ADD VSI */
1911         if (is_add) {
1912                 sections |= I40E_AQ_VSI_PROP_SCHED_VALID;
1913
1914                 ctxt->info.up_enable_bits = enabled_tc;
1915         }
1916         if (vsi->type == I40E_VSI_SRIOV) {
1917                 ctxt->info.mapping_flags |=
1918                                      cpu_to_le16(I40E_AQ_VSI_QUE_MAP_NONCONTIG);
1919                 for (i = 0; i < vsi->num_queue_pairs; i++)
1920                         ctxt->info.queue_mapping[i] =
1921                                                cpu_to_le16(vsi->base_queue + i);
1922         } else {
1923                 ctxt->info.mapping_flags |=
1924                                         cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG);
1925                 ctxt->info.queue_mapping[0] = cpu_to_le16(vsi->base_queue);
1926         }
1927         ctxt->info.valid_sections |= cpu_to_le16(sections);
1928 }
1929
1930 /**
1931  * i40e_addr_sync - Callback for dev_(mc|uc)_sync to add address
1932  * @netdev: the netdevice
1933  * @addr: address to add
1934  *
1935  * Called by __dev_(mc|uc)_sync when an address needs to be added. We call
1936  * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock.
1937  */
1938 static int i40e_addr_sync(struct net_device *netdev, const u8 *addr)
1939 {
1940         struct i40e_netdev_priv *np = netdev_priv(netdev);
1941         struct i40e_vsi *vsi = np->vsi;
1942
1943         if (i40e_add_mac_filter(vsi, addr))
1944                 return 0;
1945         else
1946                 return -ENOMEM;
1947 }
1948
1949 /**
1950  * i40e_addr_unsync - Callback for dev_(mc|uc)_sync to remove address
1951  * @netdev: the netdevice
1952  * @addr: address to add
1953  *
1954  * Called by __dev_(mc|uc)_sync when an address needs to be removed. We call
1955  * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock.
1956  */
1957 static int i40e_addr_unsync(struct net_device *netdev, const u8 *addr)
1958 {
1959         struct i40e_netdev_priv *np = netdev_priv(netdev);
1960         struct i40e_vsi *vsi = np->vsi;
1961
1962         /* Under some circumstances, we might receive a request to delete
1963          * our own device address from our uc list. Because we store the
1964          * device address in the VSI's MAC/VLAN filter list, we need to ignore
1965          * such requests and not delete our device address from this list.
1966          */
1967         if (ether_addr_equal(addr, netdev->dev_addr))
1968                 return 0;
1969
1970         i40e_del_mac_filter(vsi, addr);
1971
1972         return 0;
1973 }
1974
1975 /**
1976  * i40e_set_rx_mode - NDO callback to set the netdev filters
1977  * @netdev: network interface device structure
1978  **/
1979 static void i40e_set_rx_mode(struct net_device *netdev)
1980 {
1981         struct i40e_netdev_priv *np = netdev_priv(netdev);
1982         struct i40e_vsi *vsi = np->vsi;
1983
1984         spin_lock_bh(&vsi->mac_filter_hash_lock);
1985
1986         __dev_uc_sync(netdev, i40e_addr_sync, i40e_addr_unsync);
1987         __dev_mc_sync(netdev, i40e_addr_sync, i40e_addr_unsync);
1988
1989         spin_unlock_bh(&vsi->mac_filter_hash_lock);
1990
1991         /* check for other flag changes */
1992         if (vsi->current_netdev_flags != vsi->netdev->flags) {
1993                 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
1994                 set_bit(__I40E_MACVLAN_SYNC_PENDING, vsi->back->state);
1995         }
1996 }
1997
1998 /**
1999  * i40e_undo_del_filter_entries - Undo the changes made to MAC filter entries
2000  * @vsi: Pointer to VSI struct
2001  * @from: Pointer to list which contains MAC filter entries - changes to
2002  *        those entries needs to be undone.
2003  *
2004  * MAC filter entries from this list were slated for deletion.
2005  **/
2006 static void i40e_undo_del_filter_entries(struct i40e_vsi *vsi,
2007                                          struct hlist_head *from)
2008 {
2009         struct i40e_mac_filter *f;
2010         struct hlist_node *h;
2011
2012         hlist_for_each_entry_safe(f, h, from, hlist) {
2013                 u64 key = i40e_addr_to_hkey(f->macaddr);
2014
2015                 /* Move the element back into MAC filter list*/
2016                 hlist_del(&f->hlist);
2017                 hash_add(vsi->mac_filter_hash, &f->hlist, key);
2018         }
2019 }
2020
2021 /**
2022  * i40e_undo_add_filter_entries - Undo the changes made to MAC filter entries
2023  * @vsi: Pointer to vsi struct
2024  * @from: Pointer to list which contains MAC filter entries - changes to
2025  *        those entries needs to be undone.
2026  *
2027  * MAC filter entries from this list were slated for addition.
2028  **/
2029 static void i40e_undo_add_filter_entries(struct i40e_vsi *vsi,
2030                                          struct hlist_head *from)
2031 {
2032         struct i40e_new_mac_filter *new;
2033         struct hlist_node *h;
2034
2035         hlist_for_each_entry_safe(new, h, from, hlist) {
2036                 /* We can simply free the wrapper structure */
2037                 hlist_del(&new->hlist);
2038                 netdev_hw_addr_refcnt(new->f, vsi->netdev, -1);
2039                 kfree(new);
2040         }
2041 }
2042
2043 /**
2044  * i40e_next_entry - Get the next non-broadcast filter from a list
2045  * @next: pointer to filter in list
2046  *
2047  * Returns the next non-broadcast filter in the list. Required so that we
2048  * ignore broadcast filters within the list, since these are not handled via
2049  * the normal firmware update path.
2050  */
2051 static
2052 struct i40e_new_mac_filter *i40e_next_filter(struct i40e_new_mac_filter *next)
2053 {
2054         hlist_for_each_entry_continue(next, hlist) {
2055                 if (!is_broadcast_ether_addr(next->f->macaddr))
2056                         return next;
2057         }
2058
2059         return NULL;
2060 }
2061
2062 /**
2063  * i40e_update_filter_state - Update filter state based on return data
2064  * from firmware
2065  * @count: Number of filters added
2066  * @add_list: return data from fw
2067  * @add_head: pointer to first filter in current batch
2068  *
2069  * MAC filter entries from list were slated to be added to device. Returns
2070  * number of successful filters. Note that 0 does NOT mean success!
2071  **/
2072 static int
2073 i40e_update_filter_state(int count,
2074                          struct i40e_aqc_add_macvlan_element_data *add_list,
2075                          struct i40e_new_mac_filter *add_head)
2076 {
2077         int retval = 0;
2078         int i;
2079
2080         for (i = 0; i < count; i++) {
2081                 /* Always check status of each filter. We don't need to check
2082                  * the firmware return status because we pre-set the filter
2083                  * status to I40E_AQC_MM_ERR_NO_RES when sending the filter
2084                  * request to the adminq. Thus, if it no longer matches then
2085                  * we know the filter is active.
2086                  */
2087                 if (add_list[i].match_method == I40E_AQC_MM_ERR_NO_RES) {
2088                         add_head->state = I40E_FILTER_FAILED;
2089                 } else {
2090                         add_head->state = I40E_FILTER_ACTIVE;
2091                         retval++;
2092                 }
2093
2094                 add_head = i40e_next_filter(add_head);
2095                 if (!add_head)
2096                         break;
2097         }
2098
2099         return retval;
2100 }
2101
2102 /**
2103  * i40e_aqc_del_filters - Request firmware to delete a set of filters
2104  * @vsi: ptr to the VSI
2105  * @vsi_name: name to display in messages
2106  * @list: the list of filters to send to firmware
2107  * @num_del: the number of filters to delete
2108  * @retval: Set to -EIO on failure to delete
2109  *
2110  * Send a request to firmware via AdminQ to delete a set of filters. Uses
2111  * *retval instead of a return value so that success does not force ret_val to
2112  * be set to 0. This ensures that a sequence of calls to this function
2113  * preserve the previous value of *retval on successful delete.
2114  */
2115 static
2116 void i40e_aqc_del_filters(struct i40e_vsi *vsi, const char *vsi_name,
2117                           struct i40e_aqc_remove_macvlan_element_data *list,
2118                           int num_del, int *retval)
2119 {
2120         struct i40e_hw *hw = &vsi->back->hw;
2121         i40e_status aq_ret;
2122         int aq_err;
2123
2124         aq_ret = i40e_aq_remove_macvlan(hw, vsi->seid, list, num_del, NULL);
2125         aq_err = hw->aq.asq_last_status;
2126
2127         /* Explicitly ignore and do not report when firmware returns ENOENT */
2128         if (aq_ret && !(aq_err == I40E_AQ_RC_ENOENT)) {
2129                 *retval = -EIO;
2130                 dev_info(&vsi->back->pdev->dev,
2131                          "ignoring delete macvlan error on %s, err %s, aq_err %s\n",
2132                          vsi_name, i40e_stat_str(hw, aq_ret),
2133                          i40e_aq_str(hw, aq_err));
2134         }
2135 }
2136
2137 /**
2138  * i40e_aqc_add_filters - Request firmware to add a set of filters
2139  * @vsi: ptr to the VSI
2140  * @vsi_name: name to display in messages
2141  * @list: the list of filters to send to firmware
2142  * @add_head: Position in the add hlist
2143  * @num_add: the number of filters to add
2144  *
2145  * Send a request to firmware via AdminQ to add a chunk of filters. Will set
2146  * __I40E_VSI_OVERFLOW_PROMISC bit in vsi->state if the firmware has run out of
2147  * space for more filters.
2148  */
2149 static
2150 void i40e_aqc_add_filters(struct i40e_vsi *vsi, const char *vsi_name,
2151                           struct i40e_aqc_add_macvlan_element_data *list,
2152                           struct i40e_new_mac_filter *add_head,
2153                           int num_add)
2154 {
2155         struct i40e_hw *hw = &vsi->back->hw;
2156         int aq_err, fcnt;
2157
2158         i40e_aq_add_macvlan(hw, vsi->seid, list, num_add, NULL);
2159         aq_err = hw->aq.asq_last_status;
2160         fcnt = i40e_update_filter_state(num_add, list, add_head);
2161
2162         if (fcnt != num_add) {
2163                 set_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
2164                 dev_warn(&vsi->back->pdev->dev,
2165                          "Error %s adding RX filters on %s, promiscuous mode forced on\n",
2166                          i40e_aq_str(hw, aq_err),
2167                          vsi_name);
2168         }
2169 }
2170
2171 /**
2172  * i40e_aqc_broadcast_filter - Set promiscuous broadcast flags
2173  * @vsi: pointer to the VSI
2174  * @vsi_name: the VSI name
2175  * @f: filter data
2176  *
2177  * This function sets or clears the promiscuous broadcast flags for VLAN
2178  * filters in order to properly receive broadcast frames. Assumes that only
2179  * broadcast filters are passed.
2180  *
2181  * Returns status indicating success or failure;
2182  **/
2183 static i40e_status
2184 i40e_aqc_broadcast_filter(struct i40e_vsi *vsi, const char *vsi_name,
2185                           struct i40e_mac_filter *f)
2186 {
2187         bool enable = f->state == I40E_FILTER_NEW;
2188         struct i40e_hw *hw = &vsi->back->hw;
2189         i40e_status aq_ret;
2190
2191         if (f->vlan == I40E_VLAN_ANY) {
2192                 aq_ret = i40e_aq_set_vsi_broadcast(hw,
2193                                                    vsi->seid,
2194                                                    enable,
2195                                                    NULL);
2196         } else {
2197                 aq_ret = i40e_aq_set_vsi_bc_promisc_on_vlan(hw,
2198                                                             vsi->seid,
2199                                                             enable,
2200                                                             f->vlan,
2201                                                             NULL);
2202         }
2203
2204         if (aq_ret) {
2205                 set_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
2206                 dev_warn(&vsi->back->pdev->dev,
2207                          "Error %s, forcing overflow promiscuous on %s\n",
2208                          i40e_aq_str(hw, hw->aq.asq_last_status),
2209                          vsi_name);
2210         }
2211
2212         return aq_ret;
2213 }
2214
2215 /**
2216  * i40e_set_promiscuous - set promiscuous mode
2217  * @pf: board private structure
2218  * @promisc: promisc on or off
2219  *
2220  * There are different ways of setting promiscuous mode on a PF depending on
2221  * what state/environment we're in.  This identifies and sets it appropriately.
2222  * Returns 0 on success.
2223  **/
2224 static int i40e_set_promiscuous(struct i40e_pf *pf, bool promisc)
2225 {
2226         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
2227         struct i40e_hw *hw = &pf->hw;
2228         i40e_status aq_ret;
2229
2230         if (vsi->type == I40E_VSI_MAIN &&
2231             pf->lan_veb != I40E_NO_VEB &&
2232             !(pf->flags & I40E_FLAG_MFP_ENABLED)) {
2233                 /* set defport ON for Main VSI instead of true promisc
2234                  * this way we will get all unicast/multicast and VLAN
2235                  * promisc behavior but will not get VF or VMDq traffic
2236                  * replicated on the Main VSI.
2237                  */
2238                 if (promisc)
2239                         aq_ret = i40e_aq_set_default_vsi(hw,
2240                                                          vsi->seid,
2241                                                          NULL);
2242                 else
2243                         aq_ret = i40e_aq_clear_default_vsi(hw,
2244                                                            vsi->seid,
2245                                                            NULL);
2246                 if (aq_ret) {
2247                         dev_info(&pf->pdev->dev,
2248                                  "Set default VSI failed, err %s, aq_err %s\n",
2249                                  i40e_stat_str(hw, aq_ret),
2250                                  i40e_aq_str(hw, hw->aq.asq_last_status));
2251                 }
2252         } else {
2253                 aq_ret = i40e_aq_set_vsi_unicast_promiscuous(
2254                                                   hw,
2255                                                   vsi->seid,
2256                                                   promisc, NULL,
2257                                                   true);
2258                 if (aq_ret) {
2259                         dev_info(&pf->pdev->dev,
2260                                  "set unicast promisc failed, err %s, aq_err %s\n",
2261                                  i40e_stat_str(hw, aq_ret),
2262                                  i40e_aq_str(hw, hw->aq.asq_last_status));
2263                 }
2264                 aq_ret = i40e_aq_set_vsi_multicast_promiscuous(
2265                                                   hw,
2266                                                   vsi->seid,
2267                                                   promisc, NULL);
2268                 if (aq_ret) {
2269                         dev_info(&pf->pdev->dev,
2270                                  "set multicast promisc failed, err %s, aq_err %s\n",
2271                                  i40e_stat_str(hw, aq_ret),
2272                                  i40e_aq_str(hw, hw->aq.asq_last_status));
2273                 }
2274         }
2275
2276         if (!aq_ret)
2277                 pf->cur_promisc = promisc;
2278
2279         return aq_ret;
2280 }
2281
2282 /**
2283  * i40e_sync_vsi_filters - Update the VSI filter list to the HW
2284  * @vsi: ptr to the VSI
2285  *
2286  * Push any outstanding VSI filter changes through the AdminQ.
2287  *
2288  * Returns 0 or error value
2289  **/
2290 int i40e_sync_vsi_filters(struct i40e_vsi *vsi)
2291 {
2292         struct hlist_head tmp_add_list, tmp_del_list;
2293         struct i40e_mac_filter *f;
2294         struct i40e_new_mac_filter *new, *add_head = NULL;
2295         struct i40e_hw *hw = &vsi->back->hw;
2296         bool old_overflow, new_overflow;
2297         unsigned int failed_filters = 0;
2298         unsigned int vlan_filters = 0;
2299         char vsi_name[16] = "PF";
2300         int filter_list_len = 0;
2301         i40e_status aq_ret = 0;
2302         u32 changed_flags = 0;
2303         struct hlist_node *h;
2304         struct i40e_pf *pf;
2305         int num_add = 0;
2306         int num_del = 0;
2307         int retval = 0;
2308         u16 cmd_flags;
2309         int list_size;
2310         int bkt;
2311
2312         /* empty array typed pointers, kcalloc later */
2313         struct i40e_aqc_add_macvlan_element_data *add_list;
2314         struct i40e_aqc_remove_macvlan_element_data *del_list;
2315
2316         while (test_and_set_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state))
2317                 usleep_range(1000, 2000);
2318         pf = vsi->back;
2319
2320         old_overflow = test_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
2321
2322         if (vsi->netdev) {
2323                 changed_flags = vsi->current_netdev_flags ^ vsi->netdev->flags;
2324                 vsi->current_netdev_flags = vsi->netdev->flags;
2325         }
2326
2327         INIT_HLIST_HEAD(&tmp_add_list);
2328         INIT_HLIST_HEAD(&tmp_del_list);
2329
2330         if (vsi->type == I40E_VSI_SRIOV)
2331                 snprintf(vsi_name, sizeof(vsi_name) - 1, "VF %d", vsi->vf_id);
2332         else if (vsi->type != I40E_VSI_MAIN)
2333                 snprintf(vsi_name, sizeof(vsi_name) - 1, "vsi %d", vsi->seid);
2334
2335         if (vsi->flags & I40E_VSI_FLAG_FILTER_CHANGED) {
2336                 vsi->flags &= ~I40E_VSI_FLAG_FILTER_CHANGED;
2337
2338                 spin_lock_bh(&vsi->mac_filter_hash_lock);
2339                 /* Create a list of filters to delete. */
2340                 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
2341                         if (f->state == I40E_FILTER_REMOVE) {
2342                                 /* Move the element into temporary del_list */
2343                                 hash_del(&f->hlist);
2344                                 hlist_add_head(&f->hlist, &tmp_del_list);
2345
2346                                 /* Avoid counting removed filters */
2347                                 continue;
2348                         }
2349                         if (f->state == I40E_FILTER_NEW) {
2350                                 /* Create a temporary i40e_new_mac_filter */
2351                                 new = kzalloc(sizeof(*new), GFP_ATOMIC);
2352                                 if (!new)
2353                                         goto err_no_memory_locked;
2354
2355                                 /* Store pointer to the real filter */
2356                                 new->f = f;
2357                                 new->state = f->state;
2358
2359                                 /* Add it to the hash list */
2360                                 hlist_add_head(&new->hlist, &tmp_add_list);
2361                         }
2362
2363                         /* Count the number of active (current and new) VLAN
2364                          * filters we have now. Does not count filters which
2365                          * are marked for deletion.
2366                          */
2367                         if (f->vlan > 0)
2368                                 vlan_filters++;
2369                 }
2370
2371                 retval = i40e_correct_mac_vlan_filters(vsi,
2372                                                        &tmp_add_list,
2373                                                        &tmp_del_list,
2374                                                        vlan_filters);
2375
2376                 hlist_for_each_entry(new, &tmp_add_list, hlist)
2377                         netdev_hw_addr_refcnt(new->f, vsi->netdev, 1);
2378
2379                 if (retval)
2380                         goto err_no_memory_locked;
2381
2382                 spin_unlock_bh(&vsi->mac_filter_hash_lock);
2383         }
2384
2385         /* Now process 'del_list' outside the lock */
2386         if (!hlist_empty(&tmp_del_list)) {
2387                 filter_list_len = hw->aq.asq_buf_size /
2388                             sizeof(struct i40e_aqc_remove_macvlan_element_data);
2389                 list_size = filter_list_len *
2390                             sizeof(struct i40e_aqc_remove_macvlan_element_data);
2391                 del_list = kzalloc(list_size, GFP_ATOMIC);
2392                 if (!del_list)
2393                         goto err_no_memory;
2394
2395                 hlist_for_each_entry_safe(f, h, &tmp_del_list, hlist) {
2396                         cmd_flags = 0;
2397
2398                         /* handle broadcast filters by updating the broadcast
2399                          * promiscuous flag and release filter list.
2400                          */
2401                         if (is_broadcast_ether_addr(f->macaddr)) {
2402                                 i40e_aqc_broadcast_filter(vsi, vsi_name, f);
2403
2404                                 hlist_del(&f->hlist);
2405                                 kfree(f);
2406                                 continue;
2407                         }
2408
2409                         /* add to delete list */
2410                         ether_addr_copy(del_list[num_del].mac_addr, f->macaddr);
2411                         if (f->vlan == I40E_VLAN_ANY) {
2412                                 del_list[num_del].vlan_tag = 0;
2413                                 cmd_flags |= I40E_AQC_MACVLAN_DEL_IGNORE_VLAN;
2414                         } else {
2415                                 del_list[num_del].vlan_tag =
2416                                         cpu_to_le16((u16)(f->vlan));
2417                         }
2418
2419                         cmd_flags |= I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;
2420                         del_list[num_del].flags = cmd_flags;
2421                         num_del++;
2422
2423                         /* flush a full buffer */
2424                         if (num_del == filter_list_len) {
2425                                 i40e_aqc_del_filters(vsi, vsi_name, del_list,
2426                                                      num_del, &retval);
2427                                 memset(del_list, 0, list_size);
2428                                 num_del = 0;
2429                         }
2430                         /* Release memory for MAC filter entries which were
2431                          * synced up with HW.
2432                          */
2433                         hlist_del(&f->hlist);
2434                         kfree(f);
2435                 }
2436
2437                 if (num_del) {
2438                         i40e_aqc_del_filters(vsi, vsi_name, del_list,
2439                                              num_del, &retval);
2440                 }
2441
2442                 kfree(del_list);
2443                 del_list = NULL;
2444         }
2445
2446         if (!hlist_empty(&tmp_add_list)) {
2447                 /* Do all the adds now. */
2448                 filter_list_len = hw->aq.asq_buf_size /
2449                                sizeof(struct i40e_aqc_add_macvlan_element_data);
2450                 list_size = filter_list_len *
2451                                sizeof(struct i40e_aqc_add_macvlan_element_data);
2452                 add_list = kzalloc(list_size, GFP_ATOMIC);
2453                 if (!add_list)
2454                         goto err_no_memory;
2455
2456                 num_add = 0;
2457                 hlist_for_each_entry_safe(new, h, &tmp_add_list, hlist) {
2458                         /* handle broadcast filters by updating the broadcast
2459                          * promiscuous flag instead of adding a MAC filter.
2460                          */
2461                         if (is_broadcast_ether_addr(new->f->macaddr)) {
2462                                 if (i40e_aqc_broadcast_filter(vsi, vsi_name,
2463                                                               new->f))
2464                                         new->state = I40E_FILTER_FAILED;
2465                                 else
2466                                         new->state = I40E_FILTER_ACTIVE;
2467                                 continue;
2468                         }
2469
2470                         /* add to add array */
2471                         if (num_add == 0)
2472                                 add_head = new;
2473                         cmd_flags = 0;
2474                         ether_addr_copy(add_list[num_add].mac_addr,
2475                                         new->f->macaddr);
2476                         if (new->f->vlan == I40E_VLAN_ANY) {
2477                                 add_list[num_add].vlan_tag = 0;
2478                                 cmd_flags |= I40E_AQC_MACVLAN_ADD_IGNORE_VLAN;
2479                         } else {
2480                                 add_list[num_add].vlan_tag =
2481                                         cpu_to_le16((u16)(new->f->vlan));
2482                         }
2483                         add_list[num_add].queue_number = 0;
2484                         /* set invalid match method for later detection */
2485                         add_list[num_add].match_method = I40E_AQC_MM_ERR_NO_RES;
2486                         cmd_flags |= I40E_AQC_MACVLAN_ADD_PERFECT_MATCH;
2487                         add_list[num_add].flags = cpu_to_le16(cmd_flags);
2488                         num_add++;
2489
2490                         /* flush a full buffer */
2491                         if (num_add == filter_list_len) {
2492                                 i40e_aqc_add_filters(vsi, vsi_name, add_list,
2493                                                      add_head, num_add);
2494                                 memset(add_list, 0, list_size);
2495                                 num_add = 0;
2496                         }
2497                 }
2498                 if (num_add) {
2499                         i40e_aqc_add_filters(vsi, vsi_name, add_list, add_head,
2500                                              num_add);
2501                 }
2502                 /* Now move all of the filters from the temp add list back to
2503                  * the VSI's list.
2504                  */
2505                 spin_lock_bh(&vsi->mac_filter_hash_lock);
2506                 hlist_for_each_entry_safe(new, h, &tmp_add_list, hlist) {
2507                         /* Only update the state if we're still NEW */
2508                         if (new->f->state == I40E_FILTER_NEW)
2509                                 new->f->state = new->state;
2510                         hlist_del(&new->hlist);
2511                         netdev_hw_addr_refcnt(new->f, vsi->netdev, -1);
2512                         kfree(new);
2513                 }
2514                 spin_unlock_bh(&vsi->mac_filter_hash_lock);
2515                 kfree(add_list);
2516                 add_list = NULL;
2517         }
2518
2519         /* Determine the number of active and failed filters. */
2520         spin_lock_bh(&vsi->mac_filter_hash_lock);
2521         vsi->active_filters = 0;
2522         hash_for_each(vsi->mac_filter_hash, bkt, f, hlist) {
2523                 if (f->state == I40E_FILTER_ACTIVE)
2524                         vsi->active_filters++;
2525                 else if (f->state == I40E_FILTER_FAILED)
2526                         failed_filters++;
2527         }
2528         spin_unlock_bh(&vsi->mac_filter_hash_lock);
2529
2530         /* Check if we are able to exit overflow promiscuous mode. We can
2531          * safely exit if we didn't just enter, we no longer have any failed
2532          * filters, and we have reduced filters below the threshold value.
2533          */
2534         if (old_overflow && !failed_filters &&
2535             vsi->active_filters < vsi->promisc_threshold) {
2536                 dev_info(&pf->pdev->dev,
2537                          "filter logjam cleared on %s, leaving overflow promiscuous mode\n",
2538                          vsi_name);
2539                 clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
2540                 vsi->promisc_threshold = 0;
2541         }
2542
2543         /* if the VF is not trusted do not do promisc */
2544         if ((vsi->type == I40E_VSI_SRIOV) && !pf->vf[vsi->vf_id].trusted) {
2545                 clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
2546                 goto out;
2547         }
2548
2549         new_overflow = test_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
2550
2551         /* If we are entering overflow promiscuous, we need to calculate a new
2552          * threshold for when we are safe to exit
2553          */
2554         if (!old_overflow && new_overflow)
2555                 vsi->promisc_threshold = (vsi->active_filters * 3) / 4;
2556
2557         /* check for changes in promiscuous modes */
2558         if (changed_flags & IFF_ALLMULTI) {
2559                 bool cur_multipromisc;
2560
2561                 cur_multipromisc = !!(vsi->current_netdev_flags & IFF_ALLMULTI);
2562                 aq_ret = i40e_aq_set_vsi_multicast_promiscuous(&vsi->back->hw,
2563                                                                vsi->seid,
2564                                                                cur_multipromisc,
2565                                                                NULL);
2566                 if (aq_ret) {
2567                         retval = i40e_aq_rc_to_posix(aq_ret,
2568                                                      hw->aq.asq_last_status);
2569                         dev_info(&pf->pdev->dev,
2570                                  "set multi promisc failed on %s, err %s aq_err %s\n",
2571                                  vsi_name,
2572                                  i40e_stat_str(hw, aq_ret),
2573                                  i40e_aq_str(hw, hw->aq.asq_last_status));
2574                 }
2575         }
2576
2577         if ((changed_flags & IFF_PROMISC) || old_overflow != new_overflow) {
2578                 bool cur_promisc;
2579
2580                 cur_promisc = (!!(vsi->current_netdev_flags & IFF_PROMISC) ||
2581                                new_overflow);
2582                 aq_ret = i40e_set_promiscuous(pf, cur_promisc);
2583                 if (aq_ret) {
2584                         retval = i40e_aq_rc_to_posix(aq_ret,
2585                                                      hw->aq.asq_last_status);
2586                         dev_info(&pf->pdev->dev,
2587                                  "Setting promiscuous %s failed on %s, err %s aq_err %s\n",
2588                                  cur_promisc ? "on" : "off",
2589                                  vsi_name,
2590                                  i40e_stat_str(hw, aq_ret),
2591                                  i40e_aq_str(hw, hw->aq.asq_last_status));
2592                 }
2593         }
2594 out:
2595         /* if something went wrong then set the changed flag so we try again */
2596         if (retval)
2597                 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
2598
2599         clear_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state);
2600         return retval;
2601
2602 err_no_memory:
2603         /* Restore elements on the temporary add and delete lists */
2604         spin_lock_bh(&vsi->mac_filter_hash_lock);
2605 err_no_memory_locked:
2606         i40e_undo_del_filter_entries(vsi, &tmp_del_list);
2607         i40e_undo_add_filter_entries(vsi, &tmp_add_list);
2608         spin_unlock_bh(&vsi->mac_filter_hash_lock);
2609
2610         vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
2611         clear_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state);
2612         return -ENOMEM;
2613 }
2614
2615 /**
2616  * i40e_sync_filters_subtask - Sync the VSI filter list with HW
2617  * @pf: board private structure
2618  **/
2619 static void i40e_sync_filters_subtask(struct i40e_pf *pf)
2620 {
2621         int v;
2622
2623         if (!pf)
2624                 return;
2625         if (!test_and_clear_bit(__I40E_MACVLAN_SYNC_PENDING, pf->state))
2626                 return;
2627         if (test_bit(__I40E_VF_DISABLE, pf->state)) {
2628                 set_bit(__I40E_MACVLAN_SYNC_PENDING, pf->state);
2629                 return;
2630         }
2631
2632         for (v = 0; v < pf->num_alloc_vsi; v++) {
2633                 if (pf->vsi[v] &&
2634                     (pf->vsi[v]->flags & I40E_VSI_FLAG_FILTER_CHANGED) &&
2635                     !test_bit(__I40E_VSI_RELEASING, pf->vsi[v]->state)) {
2636                         int ret = i40e_sync_vsi_filters(pf->vsi[v]);
2637
2638                         if (ret) {
2639                                 /* come back and try again later */
2640                                 set_bit(__I40E_MACVLAN_SYNC_PENDING,
2641                                         pf->state);
2642                                 break;
2643                         }
2644                 }
2645         }
2646 }
2647
2648 /**
2649  * i40e_max_xdp_frame_size - returns the maximum allowed frame size for XDP
2650  * @vsi: the vsi
2651  **/
2652 static int i40e_max_xdp_frame_size(struct i40e_vsi *vsi)
2653 {
2654         if (PAGE_SIZE >= 8192 || (vsi->back->flags & I40E_FLAG_LEGACY_RX))
2655                 return I40E_RXBUFFER_2048;
2656         else
2657                 return I40E_RXBUFFER_3072;
2658 }
2659
2660 /**
2661  * i40e_change_mtu - NDO callback to change the Maximum Transfer Unit
2662  * @netdev: network interface device structure
2663  * @new_mtu: new value for maximum frame size
2664  *
2665  * Returns 0 on success, negative on failure
2666  **/
2667 static int i40e_change_mtu(struct net_device *netdev, int new_mtu)
2668 {
2669         struct i40e_netdev_priv *np = netdev_priv(netdev);
2670         struct i40e_vsi *vsi = np->vsi;
2671         struct i40e_pf *pf = vsi->back;
2672
2673         if (i40e_enabled_xdp_vsi(vsi)) {
2674                 int frame_size = new_mtu + I40E_PACKET_HDR_PAD;
2675
2676                 if (frame_size > i40e_max_xdp_frame_size(vsi))
2677                         return -EINVAL;
2678         }
2679
2680         netdev_info(netdev, "changing MTU from %d to %d\n",
2681                     netdev->mtu, new_mtu);
2682         netdev->mtu = new_mtu;
2683         if (netif_running(netdev))
2684                 i40e_vsi_reinit_locked(vsi);
2685         set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state);
2686         set_bit(__I40E_CLIENT_L2_CHANGE, pf->state);
2687         return 0;
2688 }
2689
2690 /**
2691  * i40e_ioctl - Access the hwtstamp interface
2692  * @netdev: network interface device structure
2693  * @ifr: interface request data
2694  * @cmd: ioctl command
2695  **/
2696 int i40e_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
2697 {
2698         struct i40e_netdev_priv *np = netdev_priv(netdev);
2699         struct i40e_pf *pf = np->vsi->back;
2700
2701         switch (cmd) {
2702         case SIOCGHWTSTAMP:
2703                 return i40e_ptp_get_ts_config(pf, ifr);
2704         case SIOCSHWTSTAMP:
2705                 return i40e_ptp_set_ts_config(pf, ifr);
2706         default:
2707                 return -EOPNOTSUPP;
2708         }
2709 }
2710
2711 /**
2712  * i40e_vlan_stripping_enable - Turn on vlan stripping for the VSI
2713  * @vsi: the vsi being adjusted
2714  **/
2715 void i40e_vlan_stripping_enable(struct i40e_vsi *vsi)
2716 {
2717         struct i40e_vsi_context ctxt;
2718         i40e_status ret;
2719
2720         /* Don't modify stripping options if a port VLAN is active */
2721         if (vsi->info.pvid)
2722                 return;
2723
2724         if ((vsi->info.valid_sections &
2725              cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) &&
2726             ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_MODE_MASK) == 0))
2727                 return;  /* already enabled */
2728
2729         vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
2730         vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |
2731                                     I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH;
2732
2733         ctxt.seid = vsi->seid;
2734         ctxt.info = vsi->info;
2735         ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
2736         if (ret) {
2737                 dev_info(&vsi->back->pdev->dev,
2738                          "update vlan stripping failed, err %s aq_err %s\n",
2739                          i40e_stat_str(&vsi->back->hw, ret),
2740                          i40e_aq_str(&vsi->back->hw,
2741                                      vsi->back->hw.aq.asq_last_status));
2742         }
2743 }
2744
2745 /**
2746  * i40e_vlan_stripping_disable - Turn off vlan stripping for the VSI
2747  * @vsi: the vsi being adjusted
2748  **/
2749 void i40e_vlan_stripping_disable(struct i40e_vsi *vsi)
2750 {
2751         struct i40e_vsi_context ctxt;
2752         i40e_status ret;
2753
2754         /* Don't modify stripping options if a port VLAN is active */
2755         if (vsi->info.pvid)
2756                 return;
2757
2758         if ((vsi->info.valid_sections &
2759              cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) &&
2760             ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_EMOD_MASK) ==
2761              I40E_AQ_VSI_PVLAN_EMOD_MASK))
2762                 return;  /* already disabled */
2763
2764         vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
2765         vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |
2766                                     I40E_AQ_VSI_PVLAN_EMOD_NOTHING;
2767
2768         ctxt.seid = vsi->seid;
2769         ctxt.info = vsi->info;
2770         ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
2771         if (ret) {
2772                 dev_info(&vsi->back->pdev->dev,
2773                          "update vlan stripping failed, err %s aq_err %s\n",
2774                          i40e_stat_str(&vsi->back->hw, ret),
2775                          i40e_aq_str(&vsi->back->hw,
2776                                      vsi->back->hw.aq.asq_last_status));
2777         }
2778 }
2779
2780 /**
2781  * i40e_add_vlan_all_mac - Add a MAC/VLAN filter for each existing MAC address
2782  * @vsi: the vsi being configured
2783  * @vid: vlan id to be added (0 = untagged only , -1 = any)
2784  *
2785  * This is a helper function for adding a new MAC/VLAN filter with the
2786  * specified VLAN for each existing MAC address already in the hash table.
2787  * This function does *not* perform any accounting to update filters based on
2788  * VLAN mode.
2789  *
2790  * NOTE: this function expects to be called while under the
2791  * mac_filter_hash_lock
2792  **/
2793 int i40e_add_vlan_all_mac(struct i40e_vsi *vsi, s16 vid)
2794 {
2795         struct i40e_mac_filter *f, *add_f;
2796         struct hlist_node *h;
2797         int bkt;
2798
2799         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
2800                 if (f->state == I40E_FILTER_REMOVE)
2801                         continue;
2802                 add_f = i40e_add_filter(vsi, f->macaddr, vid);
2803                 if (!add_f) {
2804                         dev_info(&vsi->back->pdev->dev,
2805                                  "Could not add vlan filter %d for %pM\n",
2806                                  vid, f->macaddr);
2807                         return -ENOMEM;
2808                 }
2809         }
2810
2811         return 0;
2812 }
2813
2814 /**
2815  * i40e_vsi_add_vlan - Add VSI membership for given VLAN
2816  * @vsi: the VSI being configured
2817  * @vid: VLAN id to be added
2818  **/
2819 int i40e_vsi_add_vlan(struct i40e_vsi *vsi, u16 vid)
2820 {
2821         int err;
2822
2823         if (vsi->info.pvid)
2824                 return -EINVAL;
2825
2826         /* The network stack will attempt to add VID=0, with the intention to
2827          * receive priority tagged packets with a VLAN of 0. Our HW receives
2828          * these packets by default when configured to receive untagged
2829          * packets, so we don't need to add a filter for this case.
2830          * Additionally, HW interprets adding a VID=0 filter as meaning to
2831          * receive *only* tagged traffic and stops receiving untagged traffic.
2832          * Thus, we do not want to actually add a filter for VID=0
2833          */
2834         if (!vid)
2835                 return 0;
2836
2837         /* Locked once because all functions invoked below iterates list*/
2838         spin_lock_bh(&vsi->mac_filter_hash_lock);
2839         err = i40e_add_vlan_all_mac(vsi, vid);
2840         spin_unlock_bh(&vsi->mac_filter_hash_lock);
2841         if (err)
2842                 return err;
2843
2844         /* schedule our worker thread which will take care of
2845          * applying the new filter changes
2846          */
2847         i40e_service_event_schedule(vsi->back);
2848         return 0;
2849 }
2850
2851 /**
2852  * i40e_rm_vlan_all_mac - Remove MAC/VLAN pair for all MAC with the given VLAN
2853  * @vsi: the vsi being configured
2854  * @vid: vlan id to be removed (0 = untagged only , -1 = any)
2855  *
2856  * This function should be used to remove all VLAN filters which match the
2857  * given VID. It does not schedule the service event and does not take the
2858  * mac_filter_hash_lock so it may be combined with other operations under
2859  * a single invocation of the mac_filter_hash_lock.
2860  *
2861  * NOTE: this function expects to be called while under the
2862  * mac_filter_hash_lock
2863  */
2864 void i40e_rm_vlan_all_mac(struct i40e_vsi *vsi, s16 vid)
2865 {
2866         struct i40e_mac_filter *f;
2867         struct hlist_node *h;
2868         int bkt;
2869
2870         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
2871                 if (f->vlan == vid)
2872                         __i40e_del_filter(vsi, f);
2873         }
2874 }
2875
2876 /**
2877  * i40e_vsi_kill_vlan - Remove VSI membership for given VLAN
2878  * @vsi: the VSI being configured
2879  * @vid: VLAN id to be removed
2880  **/
2881 void i40e_vsi_kill_vlan(struct i40e_vsi *vsi, u16 vid)
2882 {
2883         if (!vid || vsi->info.pvid)
2884                 return;
2885
2886         spin_lock_bh(&vsi->mac_filter_hash_lock);
2887         i40e_rm_vlan_all_mac(vsi, vid);
2888         spin_unlock_bh(&vsi->mac_filter_hash_lock);
2889
2890         /* schedule our worker thread which will take care of
2891          * applying the new filter changes
2892          */
2893         i40e_service_event_schedule(vsi->back);
2894 }
2895
2896 /**
2897  * i40e_vlan_rx_add_vid - Add a vlan id filter to HW offload
2898  * @netdev: network interface to be adjusted
2899  * @proto: unused protocol value
2900  * @vid: vlan id to be added
2901  *
2902  * net_device_ops implementation for adding vlan ids
2903  **/
2904 static int i40e_vlan_rx_add_vid(struct net_device *netdev,
2905                                 __always_unused __be16 proto, u16 vid)
2906 {
2907         struct i40e_netdev_priv *np = netdev_priv(netdev);
2908         struct i40e_vsi *vsi = np->vsi;
2909         int ret = 0;
2910
2911         if (vid >= VLAN_N_VID)
2912                 return -EINVAL;
2913
2914         ret = i40e_vsi_add_vlan(vsi, vid);
2915         if (!ret)
2916                 set_bit(vid, vsi->active_vlans);
2917
2918         return ret;
2919 }
2920
2921 /**
2922  * i40e_vlan_rx_add_vid_up - Add a vlan id filter to HW offload in UP path
2923  * @netdev: network interface to be adjusted
2924  * @proto: unused protocol value
2925  * @vid: vlan id to be added
2926  **/
2927 static void i40e_vlan_rx_add_vid_up(struct net_device *netdev,
2928                                     __always_unused __be16 proto, u16 vid)
2929 {
2930         struct i40e_netdev_priv *np = netdev_priv(netdev);
2931         struct i40e_vsi *vsi = np->vsi;
2932
2933         if (vid >= VLAN_N_VID)
2934                 return;
2935         set_bit(vid, vsi->active_vlans);
2936 }
2937
2938 /**
2939  * i40e_vlan_rx_kill_vid - Remove a vlan id filter from HW offload
2940  * @netdev: network interface to be adjusted
2941  * @proto: unused protocol value
2942  * @vid: vlan id to be removed
2943  *
2944  * net_device_ops implementation for removing vlan ids
2945  **/
2946 static int i40e_vlan_rx_kill_vid(struct net_device *netdev,
2947                                  __always_unused __be16 proto, u16 vid)
2948 {
2949         struct i40e_netdev_priv *np = netdev_priv(netdev);
2950         struct i40e_vsi *vsi = np->vsi;
2951
2952         /* return code is ignored as there is nothing a user
2953          * can do about failure to remove and a log message was
2954          * already printed from the other function
2955          */
2956         i40e_vsi_kill_vlan(vsi, vid);
2957
2958         clear_bit(vid, vsi->active_vlans);
2959
2960         return 0;
2961 }
2962
2963 /**
2964  * i40e_restore_vlan - Reinstate vlans when vsi/netdev comes back up
2965  * @vsi: the vsi being brought back up
2966  **/
2967 static void i40e_restore_vlan(struct i40e_vsi *vsi)
2968 {
2969         u16 vid;
2970
2971         if (!vsi->netdev)
2972                 return;
2973
2974         if (vsi->netdev->features & NETIF_F_HW_VLAN_CTAG_RX)
2975                 i40e_vlan_stripping_enable(vsi);
2976         else
2977                 i40e_vlan_stripping_disable(vsi);
2978
2979         for_each_set_bit(vid, vsi->active_vlans, VLAN_N_VID)
2980                 i40e_vlan_rx_add_vid_up(vsi->netdev, htons(ETH_P_8021Q),
2981                                         vid);
2982 }
2983
2984 /**
2985  * i40e_vsi_add_pvid - Add pvid for the VSI
2986  * @vsi: the vsi being adjusted
2987  * @vid: the vlan id to set as a PVID
2988  **/
2989 int i40e_vsi_add_pvid(struct i40e_vsi *vsi, u16 vid)
2990 {
2991         struct i40e_vsi_context ctxt;
2992         i40e_status ret;
2993
2994         vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
2995         vsi->info.pvid = cpu_to_le16(vid);
2996         vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_TAGGED |
2997                                     I40E_AQ_VSI_PVLAN_INSERT_PVID |
2998                                     I40E_AQ_VSI_PVLAN_EMOD_STR;
2999
3000         ctxt.seid = vsi->seid;
3001         ctxt.info = vsi->info;
3002         ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
3003         if (ret) {
3004                 dev_info(&vsi->back->pdev->dev,
3005                          "add pvid failed, err %s aq_err %s\n",
3006                          i40e_stat_str(&vsi->back->hw, ret),
3007                          i40e_aq_str(&vsi->back->hw,
3008                                      vsi->back->hw.aq.asq_last_status));
3009                 return -ENOENT;
3010         }
3011
3012         return 0;
3013 }
3014
3015 /**
3016  * i40e_vsi_remove_pvid - Remove the pvid from the VSI
3017  * @vsi: the vsi being adjusted
3018  *
3019  * Just use the vlan_rx_register() service to put it back to normal
3020  **/
3021 void i40e_vsi_remove_pvid(struct i40e_vsi *vsi)
3022 {
3023         i40e_vlan_stripping_disable(vsi);
3024
3025         vsi->info.pvid = 0;
3026 }
3027
3028 /**
3029  * i40e_vsi_setup_tx_resources - Allocate VSI Tx queue resources
3030  * @vsi: ptr to the VSI
3031  *
3032  * If this function returns with an error, then it's possible one or
3033  * more of the rings is populated (while the rest are not).  It is the
3034  * callers duty to clean those orphaned rings.
3035  *
3036  * Return 0 on success, negative on failure
3037  **/
3038 static int i40e_vsi_setup_tx_resources(struct i40e_vsi *vsi)
3039 {
3040         int i, err = 0;
3041
3042         for (i = 0; i < vsi->num_queue_pairs && !err; i++)
3043                 err = i40e_setup_tx_descriptors(vsi->tx_rings[i]);
3044
3045         if (!i40e_enabled_xdp_vsi(vsi))
3046                 return err;
3047
3048         for (i = 0; i < vsi->num_queue_pairs && !err; i++)
3049                 err = i40e_setup_tx_descriptors(vsi->xdp_rings[i]);
3050
3051         return err;
3052 }
3053
3054 /**
3055  * i40e_vsi_free_tx_resources - Free Tx resources for VSI queues
3056  * @vsi: ptr to the VSI
3057  *
3058  * Free VSI's transmit software resources
3059  **/
3060 static void i40e_vsi_free_tx_resources(struct i40e_vsi *vsi)
3061 {
3062         int i;
3063
3064         if (vsi->tx_rings) {
3065                 for (i = 0; i < vsi->num_queue_pairs; i++)
3066                         if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc)
3067                                 i40e_free_tx_resources(vsi->tx_rings[i]);
3068         }
3069
3070         if (vsi->xdp_rings) {
3071                 for (i = 0; i < vsi->num_queue_pairs; i++)
3072                         if (vsi->xdp_rings[i] && vsi->xdp_rings[i]->desc)
3073                                 i40e_free_tx_resources(vsi->xdp_rings[i]);
3074         }
3075 }
3076
3077 /**
3078  * i40e_vsi_setup_rx_resources - Allocate VSI queues Rx resources
3079  * @vsi: ptr to the VSI
3080  *
3081  * If this function returns with an error, then it's possible one or
3082  * more of the rings is populated (while the rest are not).  It is the
3083  * callers duty to clean those orphaned rings.
3084  *
3085  * Return 0 on success, negative on failure
3086  **/
3087 static int i40e_vsi_setup_rx_resources(struct i40e_vsi *vsi)
3088 {
3089         int i, err = 0;
3090
3091         for (i = 0; i < vsi->num_queue_pairs && !err; i++)
3092                 err = i40e_setup_rx_descriptors(vsi->rx_rings[i]);
3093         return err;
3094 }
3095
3096 /**
3097  * i40e_vsi_free_rx_resources - Free Rx Resources for VSI queues
3098  * @vsi: ptr to the VSI
3099  *
3100  * Free all receive software resources
3101  **/
3102 static void i40e_vsi_free_rx_resources(struct i40e_vsi *vsi)
3103 {
3104         int i;
3105
3106         if (!vsi->rx_rings)
3107                 return;
3108
3109         for (i = 0; i < vsi->num_queue_pairs; i++)
3110                 if (vsi->rx_rings[i] && vsi->rx_rings[i]->desc)
3111                         i40e_free_rx_resources(vsi->rx_rings[i]);
3112 }
3113
3114 /**
3115  * i40e_config_xps_tx_ring - Configure XPS for a Tx ring
3116  * @ring: The Tx ring to configure
3117  *
3118  * This enables/disables XPS for a given Tx descriptor ring
3119  * based on the TCs enabled for the VSI that ring belongs to.
3120  **/
3121 static void i40e_config_xps_tx_ring(struct i40e_ring *ring)
3122 {
3123         int cpu;
3124
3125         if (!ring->q_vector || !ring->netdev || ring->ch)
3126                 return;
3127
3128         /* We only initialize XPS once, so as not to overwrite user settings */
3129         if (test_and_set_bit(__I40E_TX_XPS_INIT_DONE, ring->state))
3130                 return;
3131
3132         cpu = cpumask_local_spread(ring->q_vector->v_idx, -1);
3133         netif_set_xps_queue(ring->netdev, get_cpu_mask(cpu),
3134                             ring->queue_index);
3135 }
3136
3137 /**
3138  * i40e_configure_tx_ring - Configure a transmit ring context and rest
3139  * @ring: The Tx ring to configure
3140  *
3141  * Configure the Tx descriptor ring in the HMC context.
3142  **/
3143 static int i40e_configure_tx_ring(struct i40e_ring *ring)
3144 {
3145         struct i40e_vsi *vsi = ring->vsi;
3146         u16 pf_q = vsi->base_queue + ring->queue_index;
3147         struct i40e_hw *hw = &vsi->back->hw;
3148         struct i40e_hmc_obj_txq tx_ctx;
3149         i40e_status err = 0;
3150         u32 qtx_ctl = 0;
3151
3152         /* some ATR related tx ring init */
3153         if (vsi->back->flags & I40E_FLAG_FD_ATR_ENABLED) {
3154                 ring->atr_sample_rate = vsi->back->atr_sample_rate;
3155                 ring->atr_count = 0;
3156         } else {
3157                 ring->atr_sample_rate = 0;
3158         }
3159
3160         /* configure XPS */
3161         i40e_config_xps_tx_ring(ring);
3162
3163         /* clear the context structure first */
3164         memset(&tx_ctx, 0, sizeof(tx_ctx));
3165
3166         tx_ctx.new_context = 1;
3167         tx_ctx.base = (ring->dma / 128);
3168         tx_ctx.qlen = ring->count;
3169         tx_ctx.fd_ena = !!(vsi->back->flags & (I40E_FLAG_FD_SB_ENABLED |
3170                                                I40E_FLAG_FD_ATR_ENABLED));
3171         tx_ctx.timesync_ena = !!(vsi->back->flags & I40E_FLAG_PTP);
3172         /* FDIR VSI tx ring can still use RS bit and writebacks */
3173         if (vsi->type != I40E_VSI_FDIR)
3174                 tx_ctx.head_wb_ena = 1;
3175         tx_ctx.head_wb_addr = ring->dma +
3176                               (ring->count * sizeof(struct i40e_tx_desc));
3177
3178         /* As part of VSI creation/update, FW allocates certain
3179          * Tx arbitration queue sets for each TC enabled for
3180          * the VSI. The FW returns the handles to these queue
3181          * sets as part of the response buffer to Add VSI,
3182          * Update VSI, etc. AQ commands. It is expected that
3183          * these queue set handles be associated with the Tx
3184          * queues by the driver as part of the TX queue context
3185          * initialization. This has to be done regardless of
3186          * DCB as by default everything is mapped to TC0.
3187          */
3188
3189         if (ring->ch)
3190                 tx_ctx.rdylist =
3191                         le16_to_cpu(ring->ch->info.qs_handle[ring->dcb_tc]);
3192
3193         else
3194                 tx_ctx.rdylist = le16_to_cpu(vsi->info.qs_handle[ring->dcb_tc]);
3195
3196         tx_ctx.rdylist_act = 0;
3197
3198         /* clear the context in the HMC */
3199         err = i40e_clear_lan_tx_queue_context(hw, pf_q);
3200         if (err) {
3201                 dev_info(&vsi->back->pdev->dev,
3202                          "Failed to clear LAN Tx queue context on Tx ring %d (pf_q %d), error: %d\n",
3203                          ring->queue_index, pf_q, err);
3204                 return -ENOMEM;
3205         }
3206
3207         /* set the context in the HMC */
3208         err = i40e_set_lan_tx_queue_context(hw, pf_q, &tx_ctx);
3209         if (err) {
3210                 dev_info(&vsi->back->pdev->dev,
3211                          "Failed to set LAN Tx queue context on Tx ring %d (pf_q %d, error: %d\n",
3212                          ring->queue_index, pf_q, err);
3213                 return -ENOMEM;
3214         }
3215
3216         /* Now associate this queue with this PCI function */
3217         if (ring->ch) {
3218                 if (ring->ch->type == I40E_VSI_VMDQ2)
3219                         qtx_ctl = I40E_QTX_CTL_VM_QUEUE;
3220                 else
3221                         return -EINVAL;
3222
3223                 qtx_ctl |= (ring->ch->vsi_number <<
3224                             I40E_QTX_CTL_VFVM_INDX_SHIFT) &
3225                             I40E_QTX_CTL_VFVM_INDX_MASK;
3226         } else {
3227                 if (vsi->type == I40E_VSI_VMDQ2) {
3228                         qtx_ctl = I40E_QTX_CTL_VM_QUEUE;
3229                         qtx_ctl |= ((vsi->id) << I40E_QTX_CTL_VFVM_INDX_SHIFT) &
3230                                     I40E_QTX_CTL_VFVM_INDX_MASK;
3231                 } else {
3232                         qtx_ctl = I40E_QTX_CTL_PF_QUEUE;
3233                 }
3234         }
3235
3236         qtx_ctl |= ((hw->pf_id << I40E_QTX_CTL_PF_INDX_SHIFT) &
3237                     I40E_QTX_CTL_PF_INDX_MASK);
3238         wr32(hw, I40E_QTX_CTL(pf_q), qtx_ctl);
3239         i40e_flush(hw);
3240
3241         /* cache tail off for easier writes later */
3242         ring->tail = hw->hw_addr + I40E_QTX_TAIL(pf_q);
3243
3244         return 0;
3245 }
3246
3247 /**
3248  * i40e_configure_rx_ring - Configure a receive ring context
3249  * @ring: The Rx ring to configure
3250  *
3251  * Configure the Rx descriptor ring in the HMC context.
3252  **/
3253 static int i40e_configure_rx_ring(struct i40e_ring *ring)
3254 {
3255         struct i40e_vsi *vsi = ring->vsi;
3256         u32 chain_len = vsi->back->hw.func_caps.rx_buf_chain_len;
3257         u16 pf_q = vsi->base_queue + ring->queue_index;
3258         struct i40e_hw *hw = &vsi->back->hw;
3259         struct i40e_hmc_obj_rxq rx_ctx;
3260         i40e_status err = 0;
3261
3262         bitmap_zero(ring->state, __I40E_RING_STATE_NBITS);
3263
3264         /* clear the context structure first */
3265         memset(&rx_ctx, 0, sizeof(rx_ctx));
3266
3267         ring->rx_buf_len = vsi->rx_buf_len;
3268
3269         rx_ctx.dbuff = DIV_ROUND_UP(ring->rx_buf_len,
3270                                     BIT_ULL(I40E_RXQ_CTX_DBUFF_SHIFT));
3271
3272         rx_ctx.base = (ring->dma / 128);
3273         rx_ctx.qlen = ring->count;
3274
3275         /* use 32 byte descriptors */
3276         rx_ctx.dsize = 1;
3277
3278         /* descriptor type is always zero
3279          * rx_ctx.dtype = 0;
3280          */
3281         rx_ctx.hsplit_0 = 0;
3282
3283         rx_ctx.rxmax = min_t(u16, vsi->max_frame, chain_len * ring->rx_buf_len);
3284         if (hw->revision_id == 0)
3285                 rx_ctx.lrxqthresh = 0;
3286         else
3287                 rx_ctx.lrxqthresh = 1;
3288         rx_ctx.crcstrip = 1;
3289         rx_ctx.l2tsel = 1;
3290         /* this controls whether VLAN is stripped from inner headers */
3291         rx_ctx.showiv = 0;
3292         /* set the prefena field to 1 because the manual says to */
3293         rx_ctx.prefena = 1;
3294
3295         /* clear the context in the HMC */
3296         err = i40e_clear_lan_rx_queue_context(hw, pf_q);
3297         if (err) {
3298                 dev_info(&vsi->back->pdev->dev,
3299                          "Failed to clear LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n",
3300                          ring->queue_index, pf_q, err);
3301                 return -ENOMEM;
3302         }
3303
3304         /* set the context in the HMC */
3305         err = i40e_set_lan_rx_queue_context(hw, pf_q, &rx_ctx);
3306         if (err) {
3307                 dev_info(&vsi->back->pdev->dev,
3308                          "Failed to set LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n",
3309                          ring->queue_index, pf_q, err);
3310                 return -ENOMEM;
3311         }
3312
3313         /* configure Rx buffer alignment */
3314         if (!vsi->netdev || (vsi->back->flags & I40E_FLAG_LEGACY_RX))
3315                 clear_ring_build_skb_enabled(ring);
3316         else
3317                 set_ring_build_skb_enabled(ring);
3318
3319         /* cache tail for quicker writes, and clear the reg before use */
3320         ring->tail = hw->hw_addr + I40E_QRX_TAIL(pf_q);
3321         writel(0, ring->tail);
3322
3323         i40e_alloc_rx_buffers(ring, I40E_DESC_UNUSED(ring));
3324
3325         return 0;
3326 }
3327
3328 /**
3329  * i40e_vsi_configure_tx - Configure the VSI for Tx
3330  * @vsi: VSI structure describing this set of rings and resources
3331  *
3332  * Configure the Tx VSI for operation.
3333  **/
3334 static int i40e_vsi_configure_tx(struct i40e_vsi *vsi)
3335 {
3336         int err = 0;
3337         u16 i;
3338
3339         for (i = 0; (i < vsi->num_queue_pairs) && !err; i++)
3340                 err = i40e_configure_tx_ring(vsi->tx_rings[i]);
3341
3342         if (!i40e_enabled_xdp_vsi(vsi))
3343                 return err;
3344
3345         for (i = 0; (i < vsi->num_queue_pairs) && !err; i++)
3346                 err = i40e_configure_tx_ring(vsi->xdp_rings[i]);
3347
3348         return err;
3349 }
3350
3351 /**
3352  * i40e_vsi_configure_rx - Configure the VSI for Rx
3353  * @vsi: the VSI being configured
3354  *
3355  * Configure the Rx VSI for operation.
3356  **/
3357 static int i40e_vsi_configure_rx(struct i40e_vsi *vsi)
3358 {
3359         int err = 0;
3360         u16 i;
3361
3362         if (!vsi->netdev || (vsi->back->flags & I40E_FLAG_LEGACY_RX)) {
3363                 vsi->max_frame = I40E_MAX_RXBUFFER;
3364                 vsi->rx_buf_len = I40E_RXBUFFER_2048;
3365 #if (PAGE_SIZE < 8192)
3366         } else if (!I40E_2K_TOO_SMALL_WITH_PADDING &&
3367                    (vsi->netdev->mtu <= ETH_DATA_LEN)) {
3368                 vsi->max_frame = I40E_RXBUFFER_1536 - NET_IP_ALIGN;
3369                 vsi->rx_buf_len = I40E_RXBUFFER_1536 - NET_IP_ALIGN;
3370 #endif
3371         } else {
3372                 vsi->max_frame = I40E_MAX_RXBUFFER;
3373                 vsi->rx_buf_len = (PAGE_SIZE < 8192) ? I40E_RXBUFFER_3072 :
3374                                                        I40E_RXBUFFER_2048;
3375         }
3376
3377         /* set up individual rings */
3378         for (i = 0; i < vsi->num_queue_pairs && !err; i++)
3379                 err = i40e_configure_rx_ring(vsi->rx_rings[i]);
3380
3381         return err;
3382 }
3383
3384 /**
3385  * i40e_vsi_config_dcb_rings - Update rings to reflect DCB TC
3386  * @vsi: ptr to the VSI
3387  **/
3388 static void i40e_vsi_config_dcb_rings(struct i40e_vsi *vsi)
3389 {
3390         struct i40e_ring *tx_ring, *rx_ring;
3391         u16 qoffset, qcount;
3392         int i, n;
3393
3394         if (!(vsi->back->flags & I40E_FLAG_DCB_ENABLED)) {
3395                 /* Reset the TC information */
3396                 for (i = 0; i < vsi->num_queue_pairs; i++) {
3397                         rx_ring = vsi->rx_rings[i];
3398                         tx_ring = vsi->tx_rings[i];
3399                         rx_ring->dcb_tc = 0;
3400                         tx_ring->dcb_tc = 0;
3401                 }
3402                 return;
3403         }
3404
3405         for (n = 0; n < I40E_MAX_TRAFFIC_CLASS; n++) {
3406                 if (!(vsi->tc_config.enabled_tc & BIT_ULL(n)))
3407                         continue;
3408
3409                 qoffset = vsi->tc_config.tc_info[n].qoffset;
3410                 qcount = vsi->tc_config.tc_info[n].qcount;
3411                 for (i = qoffset; i < (qoffset + qcount); i++) {
3412                         rx_ring = vsi->rx_rings[i];
3413                         tx_ring = vsi->tx_rings[i];
3414                         rx_ring->dcb_tc = n;
3415                         tx_ring->dcb_tc = n;
3416                 }
3417         }
3418 }
3419
3420 /**
3421  * i40e_set_vsi_rx_mode - Call set_rx_mode on a VSI
3422  * @vsi: ptr to the VSI
3423  **/
3424 static void i40e_set_vsi_rx_mode(struct i40e_vsi *vsi)
3425 {
3426         if (vsi->netdev)
3427                 i40e_set_rx_mode(vsi->netdev);
3428 }
3429
3430 /**
3431  * i40e_fdir_filter_restore - Restore the Sideband Flow Director filters
3432  * @vsi: Pointer to the targeted VSI
3433  *
3434  * This function replays the hlist on the hw where all the SB Flow Director
3435  * filters were saved.
3436  **/
3437 static void i40e_fdir_filter_restore(struct i40e_vsi *vsi)
3438 {
3439         struct i40e_fdir_filter *filter;
3440         struct i40e_pf *pf = vsi->back;
3441         struct hlist_node *node;
3442
3443         if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED))
3444                 return;
3445
3446         /* Reset FDir counters as we're replaying all existing filters */
3447         pf->fd_tcp4_filter_cnt = 0;
3448         pf->fd_udp4_filter_cnt = 0;
3449         pf->fd_sctp4_filter_cnt = 0;
3450         pf->fd_ip4_filter_cnt = 0;
3451
3452         hlist_for_each_entry_safe(filter, node,
3453                                   &pf->fdir_filter_list, fdir_node) {
3454                 i40e_add_del_fdir(vsi, filter, true);
3455         }
3456 }
3457
3458 /**
3459  * i40e_vsi_configure - Set up the VSI for action
3460  * @vsi: the VSI being configured
3461  **/
3462 static int i40e_vsi_configure(struct i40e_vsi *vsi)
3463 {
3464         int err;
3465
3466         i40e_set_vsi_rx_mode(vsi);
3467         i40e_restore_vlan(vsi);
3468         i40e_vsi_config_dcb_rings(vsi);
3469         err = i40e_vsi_configure_tx(vsi);
3470         if (!err)
3471                 err = i40e_vsi_configure_rx(vsi);
3472
3473         return err;
3474 }
3475
3476 /**
3477  * i40e_vsi_configure_msix - MSIX mode Interrupt Config in the HW
3478  * @vsi: the VSI being configured
3479  **/
3480 static void i40e_vsi_configure_msix(struct i40e_vsi *vsi)
3481 {
3482         bool has_xdp = i40e_enabled_xdp_vsi(vsi);
3483         struct i40e_pf *pf = vsi->back;
3484         struct i40e_hw *hw = &pf->hw;
3485         u16 vector;
3486         int i, q;
3487         u32 qp;
3488
3489         /* The interrupt indexing is offset by 1 in the PFINT_ITRn
3490          * and PFINT_LNKLSTn registers, e.g.:
3491          *   PFINT_ITRn[0..n-1] gets msix-1..msix-n  (qpair interrupts)
3492          */
3493         qp = vsi->base_queue;
3494         vector = vsi->base_vector;
3495         for (i = 0; i < vsi->num_q_vectors; i++, vector++) {
3496                 struct i40e_q_vector *q_vector = vsi->q_vectors[i];
3497
3498                 q_vector->rx.next_update = jiffies + 1;
3499                 q_vector->rx.target_itr =
3500                         ITR_TO_REG(vsi->rx_rings[i]->itr_setting);
3501                 wr32(hw, I40E_PFINT_ITRN(I40E_RX_ITR, vector - 1),
3502                      q_vector->rx.target_itr >> 1);
3503                 q_vector->rx.current_itr = q_vector->rx.target_itr;
3504
3505                 q_vector->tx.next_update = jiffies + 1;
3506                 q_vector->tx.target_itr =
3507                         ITR_TO_REG(vsi->tx_rings[i]->itr_setting);
3508                 wr32(hw, I40E_PFINT_ITRN(I40E_TX_ITR, vector - 1),
3509                      q_vector->tx.target_itr >> 1);
3510                 q_vector->tx.current_itr = q_vector->tx.target_itr;
3511
3512                 wr32(hw, I40E_PFINT_RATEN(vector - 1),
3513                      i40e_intrl_usec_to_reg(vsi->int_rate_limit));
3514
3515                 /* Linked list for the queuepairs assigned to this vector */
3516                 wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), qp);
3517                 for (q = 0; q < q_vector->num_ringpairs; q++) {
3518                         u32 nextqp = has_xdp ? qp + vsi->alloc_queue_pairs : qp;
3519                         u32 val;
3520
3521                         val = I40E_QINT_RQCTL_CAUSE_ENA_MASK |
3522                               (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) |
3523                               (vector << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) |
3524                               (nextqp << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) |
3525                               (I40E_QUEUE_TYPE_TX <<
3526                                I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT);
3527
3528                         wr32(hw, I40E_QINT_RQCTL(qp), val);
3529
3530                         if (has_xdp) {
3531                                 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK |
3532                                       (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) |
3533                                       (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) |
3534                                       (qp << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) |
3535                                       (I40E_QUEUE_TYPE_TX <<
3536                                        I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
3537
3538                                 wr32(hw, I40E_QINT_TQCTL(nextqp), val);
3539                         }
3540
3541                         val = I40E_QINT_TQCTL_CAUSE_ENA_MASK |
3542                               (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) |
3543                               (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) |
3544                               ((qp + 1) << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) |
3545                               (I40E_QUEUE_TYPE_RX <<
3546                                I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
3547
3548                         /* Terminate the linked list */
3549                         if (q == (q_vector->num_ringpairs - 1))
3550                                 val |= (I40E_QUEUE_END_OF_LIST <<
3551                                         I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT);
3552
3553                         wr32(hw, I40E_QINT_TQCTL(qp), val);
3554                         qp++;
3555                 }
3556         }
3557
3558         i40e_flush(hw);
3559 }
3560
3561 /**
3562  * i40e_enable_misc_int_causes - enable the non-queue interrupts
3563  * @pf: pointer to private device data structure
3564  **/
3565 static void i40e_enable_misc_int_causes(struct i40e_pf *pf)
3566 {
3567         struct i40e_hw *hw = &pf->hw;
3568         u32 val;
3569
3570         /* clear things first */
3571         wr32(hw, I40E_PFINT_ICR0_ENA, 0);  /* disable all */
3572         rd32(hw, I40E_PFINT_ICR0);         /* read to clear */
3573
3574         val = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK       |
3575               I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK    |
3576               I40E_PFINT_ICR0_ENA_GRST_MASK          |
3577               I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK |
3578               I40E_PFINT_ICR0_ENA_GPIO_MASK          |
3579               I40E_PFINT_ICR0_ENA_HMC_ERR_MASK       |
3580               I40E_PFINT_ICR0_ENA_VFLR_MASK          |
3581               I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
3582
3583         if (pf->flags & I40E_FLAG_IWARP_ENABLED)
3584                 val |= I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK;
3585
3586         if (pf->flags & I40E_FLAG_PTP)
3587                 val |= I40E_PFINT_ICR0_ENA_TIMESYNC_MASK;
3588
3589         wr32(hw, I40E_PFINT_ICR0_ENA, val);
3590
3591         /* SW_ITR_IDX = 0, but don't change INTENA */
3592         wr32(hw, I40E_PFINT_DYN_CTL0, I40E_PFINT_DYN_CTL0_SW_ITR_INDX_MASK |
3593                                         I40E_PFINT_DYN_CTL0_INTENA_MSK_MASK);
3594
3595         /* OTHER_ITR_IDX = 0 */
3596         wr32(hw, I40E_PFINT_STAT_CTL0, 0);
3597 }
3598
3599 /**
3600  * i40e_configure_msi_and_legacy - Legacy mode interrupt config in the HW
3601  * @vsi: the VSI being configured
3602  **/
3603 static void i40e_configure_msi_and_legacy(struct i40e_vsi *vsi)
3604 {
3605         u32 nextqp = i40e_enabled_xdp_vsi(vsi) ? vsi->alloc_queue_pairs : 0;
3606         struct i40e_q_vector *q_vector = vsi->q_vectors[0];
3607         struct i40e_pf *pf = vsi->back;
3608         struct i40e_hw *hw = &pf->hw;
3609         u32 val;
3610
3611         /* set the ITR configuration */
3612         q_vector->rx.next_update = jiffies + 1;
3613         q_vector->rx.target_itr = ITR_TO_REG(vsi->rx_rings[0]->itr_setting);
3614         wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), q_vector->rx.target_itr >> 1);
3615         q_vector->rx.current_itr = q_vector->rx.target_itr;
3616         q_vector->tx.next_update = jiffies + 1;
3617         q_vector->tx.target_itr = ITR_TO_REG(vsi->tx_rings[0]->itr_setting);
3618         wr32(hw, I40E_PFINT_ITR0(I40E_TX_ITR), q_vector->tx.target_itr >> 1);
3619         q_vector->tx.current_itr = q_vector->tx.target_itr;
3620
3621         i40e_enable_misc_int_causes(pf);
3622
3623         /* FIRSTQ_INDX = 0, FIRSTQ_TYPE = 0 (rx) */
3624         wr32(hw, I40E_PFINT_LNKLST0, 0);
3625
3626         /* Associate the queue pair to the vector and enable the queue int */
3627         val = I40E_QINT_RQCTL_CAUSE_ENA_MASK                   |
3628               (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT)  |
3629               (nextqp      << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT)|
3630               (I40E_QUEUE_TYPE_TX << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
3631
3632         wr32(hw, I40E_QINT_RQCTL(0), val);
3633
3634         if (i40e_enabled_xdp_vsi(vsi)) {
3635                 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK                 |
3636                       (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT)|
3637                       (I40E_QUEUE_TYPE_TX
3638                        << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
3639
3640                wr32(hw, I40E_QINT_TQCTL(nextqp), val);
3641         }
3642
3643         val = I40E_QINT_TQCTL_CAUSE_ENA_MASK                  |
3644               (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) |
3645               (I40E_QUEUE_END_OF_LIST << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT);
3646
3647         wr32(hw, I40E_QINT_TQCTL(0), val);
3648         i40e_flush(hw);
3649 }
3650
3651 /**
3652  * i40e_irq_dynamic_disable_icr0 - Disable default interrupt generation for icr0
3653  * @pf: board private structure
3654  **/
3655 void i40e_irq_dynamic_disable_icr0(struct i40e_pf *pf)
3656 {
3657         struct i40e_hw *hw = &pf->hw;
3658
3659         wr32(hw, I40E_PFINT_DYN_CTL0,
3660              I40E_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT);
3661         i40e_flush(hw);
3662 }
3663
3664 /**
3665  * i40e_irq_dynamic_enable_icr0 - Enable default interrupt generation for icr0
3666  * @pf: board private structure
3667  **/
3668 void i40e_irq_dynamic_enable_icr0(struct i40e_pf *pf)
3669 {
3670         struct i40e_hw *hw = &pf->hw;
3671         u32 val;
3672
3673         val = I40E_PFINT_DYN_CTL0_INTENA_MASK   |
3674               I40E_PFINT_DYN_CTL0_CLEARPBA_MASK |
3675               (I40E_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT);
3676
3677         wr32(hw, I40E_PFINT_DYN_CTL0, val);
3678         i40e_flush(hw);
3679 }
3680
3681 /**
3682  * i40e_msix_clean_rings - MSIX mode Interrupt Handler
3683  * @irq: interrupt number
3684  * @data: pointer to a q_vector
3685  **/
3686 static irqreturn_t i40e_msix_clean_rings(int irq, void *data)
3687 {
3688         struct i40e_q_vector *q_vector = data;
3689
3690         if (!q_vector->tx.ring && !q_vector->rx.ring)
3691                 return IRQ_HANDLED;
3692
3693         napi_schedule_irqoff(&q_vector->napi);
3694
3695         return IRQ_HANDLED;
3696 }
3697
3698 /**
3699  * i40e_irq_affinity_notify - Callback for affinity changes
3700  * @notify: context as to what irq was changed
3701  * @mask: the new affinity mask
3702  *
3703  * This is a callback function used by the irq_set_affinity_notifier function
3704  * so that we may register to receive changes to the irq affinity masks.
3705  **/
3706 static void i40e_irq_affinity_notify(struct irq_affinity_notify *notify,
3707                                      const cpumask_t *mask)
3708 {
3709         struct i40e_q_vector *q_vector =
3710                 container_of(notify, struct i40e_q_vector, affinity_notify);
3711
3712         cpumask_copy(&q_vector->affinity_mask, mask);
3713 }
3714
3715 /**
3716  * i40e_irq_affinity_release - Callback for affinity notifier release
3717  * @ref: internal core kernel usage
3718  *
3719  * This is a callback function used by the irq_set_affinity_notifier function
3720  * to inform the current notification subscriber that they will no longer
3721  * receive notifications.
3722  **/
3723 static void i40e_irq_affinity_release(struct kref *ref) {}
3724
3725 /**
3726  * i40e_vsi_request_irq_msix - Initialize MSI-X interrupts
3727  * @vsi: the VSI being configured
3728  * @basename: name for the vector
3729  *
3730  * Allocates MSI-X vectors and requests interrupts from the kernel.
3731  **/
3732 static int i40e_vsi_request_irq_msix(struct i40e_vsi *vsi, char *basename)
3733 {
3734         int q_vectors = vsi->num_q_vectors;
3735         struct i40e_pf *pf = vsi->back;
3736         int base = vsi->base_vector;
3737         int rx_int_idx = 0;
3738         int tx_int_idx = 0;
3739         int vector, err;
3740         int irq_num;
3741         int cpu;
3742
3743         for (vector = 0; vector < q_vectors; vector++) {
3744                 struct i40e_q_vector *q_vector = vsi->q_vectors[vector];
3745
3746                 irq_num = pf->msix_entries[base + vector].vector;
3747
3748                 if (q_vector->tx.ring && q_vector->rx.ring) {
3749                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
3750                                  "%s-%s-%d", basename, "TxRx", rx_int_idx++);
3751                         tx_int_idx++;
3752                 } else if (q_vector->rx.ring) {
3753                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
3754                                  "%s-%s-%d", basename, "rx", rx_int_idx++);
3755                 } else if (q_vector->tx.ring) {
3756                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
3757                                  "%s-%s-%d", basename, "tx", tx_int_idx++);
3758                 } else {
3759                         /* skip this unused q_vector */
3760                         continue;
3761                 }
3762                 err = request_irq(irq_num,
3763                                   vsi->irq_handler,
3764                                   0,
3765                                   q_vector->name,
3766                                   q_vector);
3767                 if (err) {
3768                         dev_info(&pf->pdev->dev,
3769                                  "MSIX request_irq failed, error: %d\n", err);
3770                         goto free_queue_irqs;
3771                 }
3772
3773                 /* register for affinity change notifications */
3774                 q_vector->affinity_notify.notify = i40e_irq_affinity_notify;
3775                 q_vector->affinity_notify.release = i40e_irq_affinity_release;
3776                 irq_set_affinity_notifier(irq_num, &q_vector->affinity_notify);
3777                 /* Spread affinity hints out across online CPUs.
3778                  *
3779                  * get_cpu_mask returns a static constant mask with
3780                  * a permanent lifetime so it's ok to pass to
3781                  * irq_set_affinity_hint without making a copy.
3782                  */
3783                 cpu = cpumask_local_spread(q_vector->v_idx, -1);
3784                 irq_set_affinity_hint(irq_num, get_cpu_mask(cpu));
3785         }
3786
3787         vsi->irqs_ready = true;
3788         return 0;
3789
3790 free_queue_irqs:
3791         while (vector) {
3792                 vector--;
3793                 irq_num = pf->msix_entries[base + vector].vector;
3794                 irq_set_affinity_notifier(irq_num, NULL);
3795                 irq_set_affinity_hint(irq_num, NULL);
3796                 free_irq(irq_num, &vsi->q_vectors[vector]);
3797         }
3798         return err;
3799 }
3800
3801 /**
3802  * i40e_vsi_disable_irq - Mask off queue interrupt generation on the VSI
3803  * @vsi: the VSI being un-configured
3804  **/
3805 static void i40e_vsi_disable_irq(struct i40e_vsi *vsi)
3806 {
3807         struct i40e_pf *pf = vsi->back;
3808         struct i40e_hw *hw = &pf->hw;
3809         int base = vsi->base_vector;
3810         int i;
3811
3812         /* disable interrupt causation from each queue */
3813         for (i = 0; i < vsi->num_queue_pairs; i++) {
3814                 u32 val;
3815
3816                 val = rd32(hw, I40E_QINT_TQCTL(vsi->tx_rings[i]->reg_idx));
3817                 val &= ~I40E_QINT_TQCTL_CAUSE_ENA_MASK;
3818                 wr32(hw, I40E_QINT_TQCTL(vsi->tx_rings[i]->reg_idx), val);
3819
3820                 val = rd32(hw, I40E_QINT_RQCTL(vsi->rx_rings[i]->reg_idx));
3821                 val &= ~I40E_QINT_RQCTL_CAUSE_ENA_MASK;
3822                 wr32(hw, I40E_QINT_RQCTL(vsi->rx_rings[i]->reg_idx), val);
3823
3824                 if (!i40e_enabled_xdp_vsi(vsi))
3825                         continue;
3826                 wr32(hw, I40E_QINT_TQCTL(vsi->xdp_rings[i]->reg_idx), 0);
3827         }
3828
3829         /* disable each interrupt */
3830         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
3831                 for (i = vsi->base_vector;
3832                      i < (vsi->num_q_vectors + vsi->base_vector); i++)
3833                         wr32(hw, I40E_PFINT_DYN_CTLN(i - 1), 0);
3834
3835                 i40e_flush(hw);
3836                 for (i = 0; i < vsi->num_q_vectors; i++)
3837                         synchronize_irq(pf->msix_entries[i + base].vector);
3838         } else {
3839                 /* Legacy and MSI mode - this stops all interrupt handling */
3840                 wr32(hw, I40E_PFINT_ICR0_ENA, 0);
3841                 wr32(hw, I40E_PFINT_DYN_CTL0, 0);
3842                 i40e_flush(hw);
3843                 synchronize_irq(pf->pdev->irq);
3844         }
3845 }
3846
3847 /**
3848  * i40e_vsi_enable_irq - Enable IRQ for the given VSI
3849  * @vsi: the VSI being configured
3850  **/
3851 static int i40e_vsi_enable_irq(struct i40e_vsi *vsi)
3852 {
3853         struct i40e_pf *pf = vsi->back;
3854         int i;
3855
3856         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
3857                 for (i = 0; i < vsi->num_q_vectors; i++)
3858                         i40e_irq_dynamic_enable(vsi, i);
3859         } else {
3860                 i40e_irq_dynamic_enable_icr0(pf);
3861         }
3862
3863         i40e_flush(&pf->hw);
3864         return 0;
3865 }
3866
3867 /**
3868  * i40e_free_misc_vector - Free the vector that handles non-queue events
3869  * @pf: board private structure
3870  **/
3871 static void i40e_free_misc_vector(struct i40e_pf *pf)
3872 {
3873         /* Disable ICR 0 */
3874         wr32(&pf->hw, I40E_PFINT_ICR0_ENA, 0);
3875         i40e_flush(&pf->hw);
3876
3877         if (pf->flags & I40E_FLAG_MSIX_ENABLED && pf->msix_entries) {
3878                 synchronize_irq(pf->msix_entries[0].vector);
3879                 free_irq(pf->msix_entries[0].vector, pf);
3880                 clear_bit(__I40E_MISC_IRQ_REQUESTED, pf->state);
3881         }
3882 }
3883
3884 /**
3885  * i40e_intr - MSI/Legacy and non-queue interrupt handler
3886  * @irq: interrupt number
3887  * @data: pointer to a q_vector
3888  *
3889  * This is the handler used for all MSI/Legacy interrupts, and deals
3890  * with both queue and non-queue interrupts.  This is also used in
3891  * MSIX mode to handle the non-queue interrupts.
3892  **/
3893 static irqreturn_t i40e_intr(int irq, void *data)
3894 {
3895         struct i40e_pf *pf = (struct i40e_pf *)data;
3896         struct i40e_hw *hw = &pf->hw;
3897         irqreturn_t ret = IRQ_NONE;
3898         u32 icr0, icr0_remaining;
3899         u32 val, ena_mask;
3900
3901         icr0 = rd32(hw, I40E_PFINT_ICR0);
3902         ena_mask = rd32(hw, I40E_PFINT_ICR0_ENA);
3903
3904         /* if sharing a legacy IRQ, we might get called w/o an intr pending */
3905         if ((icr0 & I40E_PFINT_ICR0_INTEVENT_MASK) == 0)
3906                 goto enable_intr;
3907
3908         /* if interrupt but no bits showing, must be SWINT */
3909         if (((icr0 & ~I40E_PFINT_ICR0_INTEVENT_MASK) == 0) ||
3910             (icr0 & I40E_PFINT_ICR0_SWINT_MASK))
3911                 pf->sw_int_count++;
3912
3913         if ((pf->flags & I40E_FLAG_IWARP_ENABLED) &&
3914             (icr0 & I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK)) {
3915                 ena_mask &= ~I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK;
3916                 dev_dbg(&pf->pdev->dev, "cleared PE_CRITERR\n");
3917                 set_bit(__I40E_CORE_RESET_REQUESTED, pf->state);
3918         }
3919
3920         /* only q0 is used in MSI/Legacy mode, and none are used in MSIX */
3921         if (icr0 & I40E_PFINT_ICR0_QUEUE_0_MASK) {
3922                 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
3923                 struct i40e_q_vector *q_vector = vsi->q_vectors[0];
3924
3925                 /* We do not have a way to disarm Queue causes while leaving
3926                  * interrupt enabled for all other causes, ideally
3927                  * interrupt should be disabled while we are in NAPI but
3928                  * this is not a performance path and napi_schedule()
3929                  * can deal with rescheduling.
3930                  */
3931                 if (!test_bit(__I40E_DOWN, pf->state))
3932                         napi_schedule_irqoff(&q_vector->napi);
3933         }
3934
3935         if (icr0 & I40E_PFINT_ICR0_ADMINQ_MASK) {
3936                 ena_mask &= ~I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
3937                 set_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state);
3938                 i40e_debug(&pf->hw, I40E_DEBUG_NVM, "AdminQ event\n");
3939         }
3940
3941         if (icr0 & I40E_PFINT_ICR0_MAL_DETECT_MASK) {
3942                 ena_mask &= ~I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK;
3943                 set_bit(__I40E_MDD_EVENT_PENDING, pf->state);
3944         }
3945
3946         if (icr0 & I40E_PFINT_ICR0_VFLR_MASK) {
3947                 /* disable any further VFLR event notifications */
3948                 if (test_bit(__I40E_VF_RESETS_DISABLED, pf->state)) {
3949                         u32 reg = rd32(hw, I40E_PFINT_ICR0_ENA);
3950
3951                         reg &= ~I40E_PFINT_ICR0_VFLR_MASK;
3952                         wr32(hw, I40E_PFINT_ICR0_ENA, reg);
3953                 } else {
3954                         ena_mask &= ~I40E_PFINT_ICR0_ENA_VFLR_MASK;
3955                         set_bit(__I40E_VFLR_EVENT_PENDING, pf->state);
3956                 }
3957         }
3958
3959         if (icr0 & I40E_PFINT_ICR0_GRST_MASK) {
3960                 if (!test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
3961                         set_bit(__I40E_RESET_INTR_RECEIVED, pf->state);
3962                 ena_mask &= ~I40E_PFINT_ICR0_ENA_GRST_MASK;
3963                 val = rd32(hw, I40E_GLGEN_RSTAT);
3964                 val = (val & I40E_GLGEN_RSTAT_RESET_TYPE_MASK)
3965                        >> I40E_GLGEN_RSTAT_RESET_TYPE_SHIFT;
3966                 if (val == I40E_RESET_CORER) {
3967                         pf->corer_count++;
3968                 } else if (val == I40E_RESET_GLOBR) {
3969                         pf->globr_count++;
3970                 } else if (val == I40E_RESET_EMPR) {
3971                         pf->empr_count++;
3972                         set_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state);
3973                 }
3974         }
3975
3976         if (icr0 & I40E_PFINT_ICR0_HMC_ERR_MASK) {
3977                 icr0 &= ~I40E_PFINT_ICR0_HMC_ERR_MASK;
3978                 dev_info(&pf->pdev->dev, "HMC error interrupt\n");
3979                 dev_info(&pf->pdev->dev, "HMC error info 0x%x, HMC error data 0x%x\n",
3980                          rd32(hw, I40E_PFHMC_ERRORINFO),
3981                          rd32(hw, I40E_PFHMC_ERRORDATA));
3982         }
3983
3984         if (icr0 & I40E_PFINT_ICR0_TIMESYNC_MASK) {
3985                 u32 prttsyn_stat = rd32(hw, I40E_PRTTSYN_STAT_0);
3986
3987                 if (prttsyn_stat & I40E_PRTTSYN_STAT_0_TXTIME_MASK) {
3988                         icr0 &= ~I40E_PFINT_ICR0_ENA_TIMESYNC_MASK;
3989                         i40e_ptp_tx_hwtstamp(pf);
3990                 }
3991         }
3992
3993         /* If a critical error is pending we have no choice but to reset the
3994          * device.
3995          * Report and mask out any remaining unexpected interrupts.
3996          */
3997         icr0_remaining = icr0 & ena_mask;
3998         if (icr0_remaining) {
3999                 dev_info(&pf->pdev->dev, "unhandled interrupt icr0=0x%08x\n",
4000                          icr0_remaining);
4001                 if ((icr0_remaining & I40E_PFINT_ICR0_PE_CRITERR_MASK) ||
4002                     (icr0_remaining & I40E_PFINT_ICR0_PCI_EXCEPTION_MASK) ||
4003                     (icr0_remaining & I40E_PFINT_ICR0_ECC_ERR_MASK)) {
4004                         dev_info(&pf->pdev->dev, "device will be reset\n");
4005                         set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
4006                         i40e_service_event_schedule(pf);
4007                 }
4008                 ena_mask &= ~icr0_remaining;
4009         }
4010         ret = IRQ_HANDLED;
4011
4012 enable_intr:
4013         /* re-enable interrupt causes */
4014         wr32(hw, I40E_PFINT_ICR0_ENA, ena_mask);
4015         if (!test_bit(__I40E_DOWN, pf->state)) {
4016                 i40e_service_event_schedule(pf);
4017                 i40e_irq_dynamic_enable_icr0(pf);
4018         }
4019
4020         return ret;
4021 }
4022
4023 /**
4024  * i40e_clean_fdir_tx_irq - Reclaim resources after transmit completes
4025  * @tx_ring:  tx ring to clean
4026  * @budget:   how many cleans we're allowed
4027  *
4028  * Returns true if there's any budget left (e.g. the clean is finished)
4029  **/
4030 static bool i40e_clean_fdir_tx_irq(struct i40e_ring *tx_ring, int budget)
4031 {
4032         struct i40e_vsi *vsi = tx_ring->vsi;
4033         u16 i = tx_ring->next_to_clean;
4034         struct i40e_tx_buffer *tx_buf;
4035         struct i40e_tx_desc *tx_desc;
4036
4037         tx_buf = &tx_ring->tx_bi[i];
4038         tx_desc = I40E_TX_DESC(tx_ring, i);
4039         i -= tx_ring->count;
4040
4041         do {
4042                 struct i40e_tx_desc *eop_desc = tx_buf->next_to_watch;
4043
4044                 /* if next_to_watch is not set then there is no work pending */
4045                 if (!eop_desc)
4046                         break;
4047
4048                 /* prevent any other reads prior to eop_desc */
4049                 smp_rmb();
4050
4051                 /* if the descriptor isn't done, no work yet to do */
4052                 if (!(eop_desc->cmd_type_offset_bsz &
4053                       cpu_to_le64(I40E_TX_DESC_DTYPE_DESC_DONE)))
4054                         break;
4055
4056                 /* clear next_to_watch to prevent false hangs */
4057                 tx_buf->next_to_watch = NULL;
4058
4059                 tx_desc->buffer_addr = 0;
4060                 tx_desc->cmd_type_offset_bsz = 0;
4061                 /* move past filter desc */
4062                 tx_buf++;
4063                 tx_desc++;
4064                 i++;
4065                 if (unlikely(!i)) {
4066                         i -= tx_ring->count;
4067                         tx_buf = tx_ring->tx_bi;
4068                         tx_desc = I40E_TX_DESC(tx_ring, 0);
4069                 }
4070                 /* unmap skb header data */
4071                 dma_unmap_single(tx_ring->dev,
4072                                  dma_unmap_addr(tx_buf, dma),
4073                                  dma_unmap_len(tx_buf, len),
4074                                  DMA_TO_DEVICE);
4075                 if (tx_buf->tx_flags & I40E_TX_FLAGS_FD_SB)
4076                         kfree(tx_buf->raw_buf);
4077
4078                 tx_buf->raw_buf = NULL;
4079                 tx_buf->tx_flags = 0;
4080                 tx_buf->next_to_watch = NULL;
4081                 dma_unmap_len_set(tx_buf, len, 0);
4082                 tx_desc->buffer_addr = 0;
4083                 tx_desc->cmd_type_offset_bsz = 0;
4084
4085                 /* move us past the eop_desc for start of next FD desc */
4086                 tx_buf++;
4087                 tx_desc++;
4088                 i++;
4089                 if (unlikely(!i)) {
4090                         i -= tx_ring->count;
4091                         tx_buf = tx_ring->tx_bi;
4092                         tx_desc = I40E_TX_DESC(tx_ring, 0);
4093                 }
4094
4095                 /* update budget accounting */
4096                 budget--;
4097         } while (likely(budget));
4098
4099         i += tx_ring->count;
4100         tx_ring->next_to_clean = i;
4101
4102         if (vsi->back->flags & I40E_FLAG_MSIX_ENABLED)
4103                 i40e_irq_dynamic_enable(vsi, tx_ring->q_vector->v_idx);
4104
4105         return budget > 0;
4106 }
4107
4108 /**
4109  * i40e_fdir_clean_ring - Interrupt Handler for FDIR SB ring
4110  * @irq: interrupt number
4111  * @data: pointer to a q_vector
4112  **/
4113 static irqreturn_t i40e_fdir_clean_ring(int irq, void *data)
4114 {
4115         struct i40e_q_vector *q_vector = data;
4116         struct i40e_vsi *vsi;
4117
4118         if (!q_vector->tx.ring)
4119                 return IRQ_HANDLED;
4120
4121         vsi = q_vector->tx.ring->vsi;
4122         i40e_clean_fdir_tx_irq(q_vector->tx.ring, vsi->work_limit);
4123
4124         return IRQ_HANDLED;
4125 }
4126
4127 /**
4128  * i40e_map_vector_to_qp - Assigns the queue pair to the vector
4129  * @vsi: the VSI being configured
4130  * @v_idx: vector index
4131  * @qp_idx: queue pair index
4132  **/
4133 static void i40e_map_vector_to_qp(struct i40e_vsi *vsi, int v_idx, int qp_idx)
4134 {
4135         struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx];
4136         struct i40e_ring *tx_ring = vsi->tx_rings[qp_idx];
4137         struct i40e_ring *rx_ring = vsi->rx_rings[qp_idx];
4138
4139         tx_ring->q_vector = q_vector;
4140         tx_ring->next = q_vector->tx.ring;
4141         q_vector->tx.ring = tx_ring;
4142         q_vector->tx.count++;
4143
4144         /* Place XDP Tx ring in the same q_vector ring list as regular Tx */
4145         if (i40e_enabled_xdp_vsi(vsi)) {
4146                 struct i40e_ring *xdp_ring = vsi->xdp_rings[qp_idx];
4147
4148                 xdp_ring->q_vector = q_vector;
4149                 xdp_ring->next = q_vector->tx.ring;
4150                 q_vector->tx.ring = xdp_ring;
4151                 q_vector->tx.count++;
4152         }
4153
4154         rx_ring->q_vector = q_vector;
4155         rx_ring->next = q_vector->rx.ring;
4156         q_vector->rx.ring = rx_ring;
4157         q_vector->rx.count++;
4158 }
4159
4160 /**
4161  * i40e_vsi_map_rings_to_vectors - Maps descriptor rings to vectors
4162  * @vsi: the VSI being configured
4163  *
4164  * This function maps descriptor rings to the queue-specific vectors
4165  * we were allotted through the MSI-X enabling code.  Ideally, we'd have
4166  * one vector per queue pair, but on a constrained vector budget, we
4167  * group the queue pairs as "efficiently" as possible.
4168  **/
4169 static void i40e_vsi_map_rings_to_vectors(struct i40e_vsi *vsi)
4170 {
4171         int qp_remaining = vsi->num_queue_pairs;
4172         int q_vectors = vsi->num_q_vectors;
4173         int num_ringpairs;
4174         int v_start = 0;
4175         int qp_idx = 0;
4176
4177         /* If we don't have enough vectors for a 1-to-1 mapping, we'll have to
4178          * group them so there are multiple queues per vector.
4179          * It is also important to go through all the vectors available to be
4180          * sure that if we don't use all the vectors, that the remaining vectors
4181          * are cleared. This is especially important when decreasing the
4182          * number of queues in use.
4183          */
4184         for (; v_start < q_vectors; v_start++) {
4185                 struct i40e_q_vector *q_vector = vsi->q_vectors[v_start];
4186
4187                 num_ringpairs = DIV_ROUND_UP(qp_remaining, q_vectors - v_start);
4188
4189                 q_vector->num_ringpairs = num_ringpairs;
4190                 q_vector->reg_idx = q_vector->v_idx + vsi->base_vector - 1;
4191
4192                 q_vector->rx.count = 0;
4193                 q_vector->tx.count = 0;
4194                 q_vector->rx.ring = NULL;
4195                 q_vector->tx.ring = NULL;
4196
4197                 while (num_ringpairs--) {
4198                         i40e_map_vector_to_qp(vsi, v_start, qp_idx);
4199                         qp_idx++;
4200                         qp_remaining--;
4201                 }
4202         }
4203 }
4204
4205 /**
4206  * i40e_vsi_request_irq - Request IRQ from the OS
4207  * @vsi: the VSI being configured
4208  * @basename: name for the vector
4209  **/
4210 static int i40e_vsi_request_irq(struct i40e_vsi *vsi, char *basename)
4211 {
4212         struct i40e_pf *pf = vsi->back;
4213         int err;
4214
4215         if (pf->flags & I40E_FLAG_MSIX_ENABLED)
4216                 err = i40e_vsi_request_irq_msix(vsi, basename);
4217         else if (pf->flags & I40E_FLAG_MSI_ENABLED)
4218                 err = request_irq(pf->pdev->irq, i40e_intr, 0,
4219                                   pf->int_name, pf);
4220         else
4221                 err = request_irq(pf->pdev->irq, i40e_intr, IRQF_SHARED,
4222                                   pf->int_name, pf);
4223
4224         if (err)
4225                 dev_info(&pf->pdev->dev, "request_irq failed, Error %d\n", err);
4226
4227         return err;
4228 }
4229
4230 #ifdef CONFIG_NET_POLL_CONTROLLER
4231 /**
4232  * i40e_netpoll - A Polling 'interrupt' handler
4233  * @netdev: network interface device structure
4234  *
4235  * This is used by netconsole to send skbs without having to re-enable
4236  * interrupts.  It's not called while the normal interrupt routine is executing.
4237  **/
4238 static void i40e_netpoll(struct net_device *netdev)
4239 {
4240         struct i40e_netdev_priv *np = netdev_priv(netdev);
4241         struct i40e_vsi *vsi = np->vsi;
4242         struct i40e_pf *pf = vsi->back;
4243         int i;
4244
4245         /* if interface is down do nothing */
4246         if (test_bit(__I40E_VSI_DOWN, vsi->state))
4247                 return;
4248
4249         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
4250                 for (i = 0; i < vsi->num_q_vectors; i++)
4251                         i40e_msix_clean_rings(0, vsi->q_vectors[i]);
4252         } else {
4253                 i40e_intr(pf->pdev->irq, netdev);
4254         }
4255 }
4256 #endif
4257
4258 #define I40E_QTX_ENA_WAIT_COUNT 50
4259
4260 /**
4261  * i40e_pf_txq_wait - Wait for a PF's Tx queue to be enabled or disabled
4262  * @pf: the PF being configured
4263  * @pf_q: the PF queue
4264  * @enable: enable or disable state of the queue
4265  *
4266  * This routine will wait for the given Tx queue of the PF to reach the
4267  * enabled or disabled state.
4268  * Returns -ETIMEDOUT in case of failing to reach the requested state after
4269  * multiple retries; else will return 0 in case of success.
4270  **/
4271 static int i40e_pf_txq_wait(struct i40e_pf *pf, int pf_q, bool enable)
4272 {
4273         int i;
4274         u32 tx_reg;
4275
4276         for (i = 0; i < I40E_QUEUE_WAIT_RETRY_LIMIT; i++) {
4277                 tx_reg = rd32(&pf->hw, I40E_QTX_ENA(pf_q));
4278                 if (enable == !!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK))
4279                         break;
4280
4281                 usleep_range(10, 20);
4282         }
4283         if (i >= I40E_QUEUE_WAIT_RETRY_LIMIT)
4284                 return -ETIMEDOUT;
4285
4286         return 0;
4287 }
4288
4289 /**
4290  * i40e_control_tx_q - Start or stop a particular Tx queue
4291  * @pf: the PF structure
4292  * @pf_q: the PF queue to configure
4293  * @enable: start or stop the queue
4294  *
4295  * This function enables or disables a single queue. Note that any delay
4296  * required after the operation is expected to be handled by the caller of
4297  * this function.
4298  **/
4299 static void i40e_control_tx_q(struct i40e_pf *pf, int pf_q, bool enable)
4300 {
4301         struct i40e_hw *hw = &pf->hw;
4302         u32 tx_reg;
4303         int i;
4304
4305         /* warn the TX unit of coming changes */
4306         i40e_pre_tx_queue_cfg(&pf->hw, pf_q, enable);
4307         if (!enable)
4308                 usleep_range(10, 20);
4309
4310         for (i = 0; i < I40E_QTX_ENA_WAIT_COUNT; i++) {
4311                 tx_reg = rd32(hw, I40E_QTX_ENA(pf_q));
4312                 if (((tx_reg >> I40E_QTX_ENA_QENA_REQ_SHIFT) & 1) ==
4313                     ((tx_reg >> I40E_QTX_ENA_QENA_STAT_SHIFT) & 1))
4314                         break;
4315                 usleep_range(1000, 2000);
4316         }
4317
4318         /* Skip if the queue is already in the requested state */
4319         if (enable == !!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK))
4320                 return;
4321
4322         /* turn on/off the queue */
4323         if (enable) {
4324                 wr32(hw, I40E_QTX_HEAD(pf_q), 0);
4325                 tx_reg |= I40E_QTX_ENA_QENA_REQ_MASK;
4326         } else {
4327                 tx_reg &= ~I40E_QTX_ENA_QENA_REQ_MASK;
4328         }
4329
4330         wr32(hw, I40E_QTX_ENA(pf_q), tx_reg);
4331 }
4332
4333 /**
4334  * i40e_control_wait_tx_q - Start/stop Tx queue and wait for completion
4335  * @seid: VSI SEID
4336  * @pf: the PF structure
4337  * @pf_q: the PF queue to configure
4338  * @is_xdp: true if the queue is used for XDP
4339  * @enable: start or stop the queue
4340  **/
4341 int i40e_control_wait_tx_q(int seid, struct i40e_pf *pf, int pf_q,
4342                            bool is_xdp, bool enable)
4343 {
4344         int ret;
4345
4346         i40e_control_tx_q(pf, pf_q, enable);
4347
4348         /* wait for the change to finish */
4349         ret = i40e_pf_txq_wait(pf, pf_q, enable);
4350         if (ret) {
4351                 dev_info(&pf->pdev->dev,
4352                          "VSI seid %d %sTx ring %d %sable timeout\n",
4353                          seid, (is_xdp ? "XDP " : ""), pf_q,
4354                          (enable ? "en" : "dis"));
4355         }
4356
4357         return ret;
4358 }
4359
4360 /**
4361  * i40e_vsi_enable_tx - Start a VSI's rings
4362  * @vsi: the VSI being configured
4363  **/
4364 static int i40e_vsi_enable_tx(struct i40e_vsi *vsi)
4365 {
4366         struct i40e_pf *pf = vsi->back;
4367         int i, pf_q, ret = 0;
4368
4369         pf_q = vsi->base_queue;
4370         for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
4371                 ret = i40e_control_wait_tx_q(vsi->seid, pf,
4372                                              pf_q,
4373                                              false /*is xdp*/, true);
4374                 if (ret)
4375                         break;
4376
4377                 if (!i40e_enabled_xdp_vsi(vsi))
4378                         continue;
4379
4380                 ret = i40e_control_wait_tx_q(vsi->seid, pf,
4381                                              pf_q + vsi->alloc_queue_pairs,
4382                                              true /*is xdp*/, true);
4383                 if (ret)
4384                         break;
4385         }
4386         return ret;
4387 }
4388
4389 /**
4390  * i40e_pf_rxq_wait - Wait for a PF's Rx queue to be enabled or disabled
4391  * @pf: the PF being configured
4392  * @pf_q: the PF queue
4393  * @enable: enable or disable state of the queue
4394  *
4395  * This routine will wait for the given Rx queue of the PF to reach the
4396  * enabled or disabled state.
4397  * Returns -ETIMEDOUT in case of failing to reach the requested state after
4398  * multiple retries; else will return 0 in case of success.
4399  **/
4400 static int i40e_pf_rxq_wait(struct i40e_pf *pf, int pf_q, bool enable)
4401 {
4402         int i;
4403         u32 rx_reg;
4404
4405         for (i = 0; i < I40E_QUEUE_WAIT_RETRY_LIMIT; i++) {
4406                 rx_reg = rd32(&pf->hw, I40E_QRX_ENA(pf_q));
4407                 if (enable == !!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK))
4408                         break;
4409
4410                 usleep_range(10, 20);
4411         }
4412         if (i >= I40E_QUEUE_WAIT_RETRY_LIMIT)
4413                 return -ETIMEDOUT;
4414
4415         return 0;
4416 }
4417
4418 /**
4419  * i40e_control_rx_q - Start or stop a particular Rx queue
4420  * @pf: the PF structure
4421  * @pf_q: the PF queue to configure
4422  * @enable: start or stop the queue
4423  *
4424  * This function enables or disables a single queue. Note that
4425  * any delay required after the operation is expected to be
4426  * handled by the caller of this function.
4427  **/
4428 static void i40e_control_rx_q(struct i40e_pf *pf, int pf_q, bool enable)
4429 {
4430         struct i40e_hw *hw = &pf->hw;
4431         u32 rx_reg;
4432         int i;
4433
4434         for (i = 0; i < I40E_QTX_ENA_WAIT_COUNT; i++) {
4435                 rx_reg = rd32(hw, I40E_QRX_ENA(pf_q));
4436                 if (((rx_reg >> I40E_QRX_ENA_QENA_REQ_SHIFT) & 1) ==
4437                     ((rx_reg >> I40E_QRX_ENA_QENA_STAT_SHIFT) & 1))
4438                         break;
4439                 usleep_range(1000, 2000);
4440         }
4441
4442         /* Skip if the queue is already in the requested state */
4443         if (enable == !!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK))
4444                 return;
4445
4446         /* turn on/off the queue */
4447         if (enable)
4448                 rx_reg |= I40E_QRX_ENA_QENA_REQ_MASK;
4449         else
4450                 rx_reg &= ~I40E_QRX_ENA_QENA_REQ_MASK;
4451
4452         wr32(hw, I40E_QRX_ENA(pf_q), rx_reg);
4453 }
4454
4455 /**
4456  * i40e_control_wait_rx_q
4457  * @pf: the PF structure
4458  * @pf_q: queue being configured
4459  * @enable: start or stop the rings
4460  *
4461  * This function enables or disables a single queue along with waiting
4462  * for the change to finish. The caller of this function should handle
4463  * the delays needed in the case of disabling queues.
4464  **/
4465 int i40e_control_wait_rx_q(struct i40e_pf *pf, int pf_q, bool enable)
4466 {
4467         int ret = 0;
4468
4469         i40e_control_rx_q(pf, pf_q, enable);
4470
4471         /* wait for the change to finish */
4472         ret = i40e_pf_rxq_wait(pf, pf_q, enable);
4473         if (ret)
4474                 return ret;
4475
4476         return ret;
4477 }
4478
4479 /**
4480  * i40e_vsi_enable_rx - Start a VSI's rings
4481  * @vsi: the VSI being configured
4482  **/
4483 static int i40e_vsi_enable_rx(struct i40e_vsi *vsi)
4484 {
4485         struct i40e_pf *pf = vsi->back;
4486         int i, pf_q, ret = 0;
4487
4488         pf_q = vsi->base_queue;
4489         for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
4490                 ret = i40e_control_wait_rx_q(pf, pf_q, true);
4491                 if (ret) {
4492                         dev_info(&pf->pdev->dev,
4493                                  "VSI seid %d Rx ring %d enable timeout\n",
4494                                  vsi->seid, pf_q);
4495                         break;
4496                 }
4497         }
4498
4499         return ret;
4500 }
4501
4502 /**
4503  * i40e_vsi_start_rings - Start a VSI's rings
4504  * @vsi: the VSI being configured
4505  **/
4506 int i40e_vsi_start_rings(struct i40e_vsi *vsi)
4507 {
4508         int ret = 0;
4509
4510         /* do rx first for enable and last for disable */
4511         ret = i40e_vsi_enable_rx(vsi);
4512         if (ret)
4513                 return ret;
4514         ret = i40e_vsi_enable_tx(vsi);
4515
4516         return ret;
4517 }
4518
4519 #define I40E_DISABLE_TX_GAP_MSEC        50
4520
4521 /**
4522  * i40e_vsi_stop_rings - Stop a VSI's rings
4523  * @vsi: the VSI being configured
4524  **/
4525 void i40e_vsi_stop_rings(struct i40e_vsi *vsi)
4526 {
4527         struct i40e_pf *pf = vsi->back;
4528         int pf_q, err, q_end;
4529
4530         /* When port TX is suspended, don't wait */
4531         if (test_bit(__I40E_PORT_SUSPENDED, vsi->back->state))
4532                 return i40e_vsi_stop_rings_no_wait(vsi);
4533
4534         q_end = vsi->base_queue + vsi->num_queue_pairs;
4535         for (pf_q = vsi->base_queue; pf_q < q_end; pf_q++)
4536                 i40e_pre_tx_queue_cfg(&pf->hw, (u32)pf_q, false);
4537
4538         for (pf_q = vsi->base_queue; pf_q < q_end; pf_q++) {
4539                 err = i40e_control_wait_rx_q(pf, pf_q, false);
4540                 if (err)
4541                         dev_info(&pf->pdev->dev,
4542                                  "VSI seid %d Rx ring %d dissable timeout\n",
4543                                  vsi->seid, pf_q);
4544         }
4545
4546         msleep(I40E_DISABLE_TX_GAP_MSEC);
4547         pf_q = vsi->base_queue;
4548         for (pf_q = vsi->base_queue; pf_q < q_end; pf_q++)
4549                 wr32(&pf->hw, I40E_QTX_ENA(pf_q), 0);
4550
4551         i40e_vsi_wait_queues_disabled(vsi);
4552 }
4553
4554 /**
4555  * i40e_vsi_stop_rings_no_wait - Stop a VSI's rings and do not delay
4556  * @vsi: the VSI being shutdown
4557  *
4558  * This function stops all the rings for a VSI but does not delay to verify
4559  * that rings have been disabled. It is expected that the caller is shutting
4560  * down multiple VSIs at once and will delay together for all the VSIs after
4561  * initiating the shutdown. This is particularly useful for shutting down lots
4562  * of VFs together. Otherwise, a large delay can be incurred while configuring
4563  * each VSI in serial.
4564  **/
4565 void i40e_vsi_stop_rings_no_wait(struct i40e_vsi *vsi)
4566 {
4567         struct i40e_pf *pf = vsi->back;
4568         int i, pf_q;
4569
4570         pf_q = vsi->base_queue;
4571         for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
4572                 i40e_control_tx_q(pf, pf_q, false);
4573                 i40e_control_rx_q(pf, pf_q, false);
4574         }
4575 }
4576
4577 /**
4578  * i40e_vsi_free_irq - Free the irq association with the OS
4579  * @vsi: the VSI being configured
4580  **/
4581 static void i40e_vsi_free_irq(struct i40e_vsi *vsi)
4582 {
4583         struct i40e_pf *pf = vsi->back;
4584         struct i40e_hw *hw = &pf->hw;
4585         int base = vsi->base_vector;
4586         u32 val, qp;
4587         int i;
4588
4589         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
4590                 if (!vsi->q_vectors)
4591                         return;
4592
4593                 if (!vsi->irqs_ready)
4594                         return;
4595
4596                 vsi->irqs_ready = false;
4597                 for (i = 0; i < vsi->num_q_vectors; i++) {
4598                         int irq_num;
4599                         u16 vector;
4600
4601                         vector = i + base;
4602                         irq_num = pf->msix_entries[vector].vector;
4603
4604                         /* free only the irqs that were actually requested */
4605                         if (!vsi->q_vectors[i] ||
4606                             !vsi->q_vectors[i]->num_ringpairs)
4607                                 continue;
4608
4609                         /* clear the affinity notifier in the IRQ descriptor */
4610                         irq_set_affinity_notifier(irq_num, NULL);
4611                         /* remove our suggested affinity mask for this IRQ */
4612                         irq_set_affinity_hint(irq_num, NULL);
4613                         synchronize_irq(irq_num);
4614                         free_irq(irq_num, vsi->q_vectors[i]);
4615
4616                         /* Tear down the interrupt queue link list
4617                          *
4618                          * We know that they come in pairs and always
4619                          * the Rx first, then the Tx.  To clear the
4620                          * link list, stick the EOL value into the
4621                          * next_q field of the registers.
4622                          */
4623                         val = rd32(hw, I40E_PFINT_LNKLSTN(vector - 1));
4624                         qp = (val & I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK)
4625                                 >> I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT;
4626                         val |= I40E_QUEUE_END_OF_LIST
4627                                 << I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT;
4628                         wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), val);
4629
4630                         while (qp != I40E_QUEUE_END_OF_LIST) {
4631                                 u32 next;
4632
4633                                 val = rd32(hw, I40E_QINT_RQCTL(qp));
4634
4635                                 val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK  |
4636                                          I40E_QINT_RQCTL_MSIX0_INDX_MASK |
4637                                          I40E_QINT_RQCTL_CAUSE_ENA_MASK  |
4638                                          I40E_QINT_RQCTL_INTEVENT_MASK);
4639
4640                                 val |= (I40E_QINT_RQCTL_ITR_INDX_MASK |
4641                                          I40E_QINT_RQCTL_NEXTQ_INDX_MASK);
4642
4643                                 wr32(hw, I40E_QINT_RQCTL(qp), val);
4644
4645                                 val = rd32(hw, I40E_QINT_TQCTL(qp));
4646
4647                                 next = (val & I40E_QINT_TQCTL_NEXTQ_INDX_MASK)
4648                                         >> I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT;
4649
4650                                 val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK  |
4651                                          I40E_QINT_TQCTL_MSIX0_INDX_MASK |
4652                                          I40E_QINT_TQCTL_CAUSE_ENA_MASK  |
4653                                          I40E_QINT_TQCTL_INTEVENT_MASK);
4654
4655                                 val |= (I40E_QINT_TQCTL_ITR_INDX_MASK |
4656                                          I40E_QINT_TQCTL_NEXTQ_INDX_MASK);
4657
4658                                 wr32(hw, I40E_QINT_TQCTL(qp), val);
4659                                 qp = next;
4660                         }
4661                 }
4662         } else {
4663                 free_irq(pf->pdev->irq, pf);
4664
4665                 val = rd32(hw, I40E_PFINT_LNKLST0);
4666                 qp = (val & I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK)
4667                         >> I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT;
4668                 val |= I40E_QUEUE_END_OF_LIST
4669                         << I40E_PFINT_LNKLST0_FIRSTQ_INDX_SHIFT;
4670                 wr32(hw, I40E_PFINT_LNKLST0, val);
4671
4672                 val = rd32(hw, I40E_QINT_RQCTL(qp));
4673                 val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK  |
4674                          I40E_QINT_RQCTL_MSIX0_INDX_MASK |
4675                          I40E_QINT_RQCTL_CAUSE_ENA_MASK  |
4676                          I40E_QINT_RQCTL_INTEVENT_MASK);
4677
4678                 val |= (I40E_QINT_RQCTL_ITR_INDX_MASK |
4679                         I40E_QINT_RQCTL_NEXTQ_INDX_MASK);
4680
4681                 wr32(hw, I40E_QINT_RQCTL(qp), val);
4682
4683                 val = rd32(hw, I40E_QINT_TQCTL(qp));
4684
4685                 val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK  |
4686                          I40E_QINT_TQCTL_MSIX0_INDX_MASK |
4687                          I40E_QINT_TQCTL_CAUSE_ENA_MASK  |
4688                          I40E_QINT_TQCTL_INTEVENT_MASK);
4689
4690                 val |= (I40E_QINT_TQCTL_ITR_INDX_MASK |
4691                         I40E_QINT_TQCTL_NEXTQ_INDX_MASK);
4692
4693                 wr32(hw, I40E_QINT_TQCTL(qp), val);
4694         }
4695 }
4696
4697 /**
4698  * i40e_free_q_vector - Free memory allocated for specific interrupt vector
4699  * @vsi: the VSI being configured
4700  * @v_idx: Index of vector to be freed
4701  *
4702  * This function frees the memory allocated to the q_vector.  In addition if
4703  * NAPI is enabled it will delete any references to the NAPI struct prior
4704  * to freeing the q_vector.
4705  **/
4706 static void i40e_free_q_vector(struct i40e_vsi *vsi, int v_idx)
4707 {
4708         struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx];
4709         struct i40e_ring *ring;
4710
4711         if (!q_vector)
4712                 return;
4713
4714         /* disassociate q_vector from rings */
4715         i40e_for_each_ring(ring, q_vector->tx)
4716                 ring->q_vector = NULL;
4717
4718         i40e_for_each_ring(ring, q_vector->rx)
4719                 ring->q_vector = NULL;
4720
4721         /* only VSI w/ an associated netdev is set up w/ NAPI */
4722         if (vsi->netdev)
4723                 netif_napi_del(&q_vector->napi);
4724
4725         vsi->q_vectors[v_idx] = NULL;
4726
4727         kfree_rcu(q_vector, rcu);
4728 }
4729
4730 /**
4731  * i40e_vsi_free_q_vectors - Free memory allocated for interrupt vectors
4732  * @vsi: the VSI being un-configured
4733  *
4734  * This frees the memory allocated to the q_vectors and
4735  * deletes references to the NAPI struct.
4736  **/
4737 static void i40e_vsi_free_q_vectors(struct i40e_vsi *vsi)
4738 {
4739         int v_idx;
4740
4741         for (v_idx = 0; v_idx < vsi->num_q_vectors; v_idx++)
4742                 i40e_free_q_vector(vsi, v_idx);
4743 }
4744
4745 /**
4746  * i40e_reset_interrupt_capability - Disable interrupt setup in OS
4747  * @pf: board private structure
4748  **/
4749 static void i40e_reset_interrupt_capability(struct i40e_pf *pf)
4750 {
4751         /* If we're in Legacy mode, the interrupt was cleaned in vsi_close */
4752         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
4753                 pci_disable_msix(pf->pdev);
4754                 kfree(pf->msix_entries);
4755                 pf->msix_entries = NULL;
4756                 kfree(pf->irq_pile);
4757                 pf->irq_pile = NULL;
4758         } else if (pf->flags & I40E_FLAG_MSI_ENABLED) {
4759                 pci_disable_msi(pf->pdev);
4760         }
4761         pf->flags &= ~(I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED);
4762 }
4763
4764 /**
4765  * i40e_clear_interrupt_scheme - Clear the current interrupt scheme settings
4766  * @pf: board private structure
4767  *
4768  * We go through and clear interrupt specific resources and reset the structure
4769  * to pre-load conditions
4770  **/
4771 static void i40e_clear_interrupt_scheme(struct i40e_pf *pf)
4772 {
4773         int i;
4774
4775         if (test_bit(__I40E_MISC_IRQ_REQUESTED, pf->state))
4776                 i40e_free_misc_vector(pf);
4777
4778         i40e_put_lump(pf->irq_pile, pf->iwarp_base_vector,
4779                       I40E_IWARP_IRQ_PILE_ID);
4780
4781         i40e_put_lump(pf->irq_pile, 0, I40E_PILE_VALID_BIT-1);
4782         for (i = 0; i < pf->num_alloc_vsi; i++)
4783                 if (pf->vsi[i])
4784                         i40e_vsi_free_q_vectors(pf->vsi[i]);
4785         i40e_reset_interrupt_capability(pf);
4786 }
4787
4788 /**
4789  * i40e_napi_enable_all - Enable NAPI for all q_vectors in the VSI
4790  * @vsi: the VSI being configured
4791  **/
4792 static void i40e_napi_enable_all(struct i40e_vsi *vsi)
4793 {
4794         int q_idx;
4795
4796         if (!vsi->netdev)
4797                 return;
4798
4799         for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++) {
4800                 struct i40e_q_vector *q_vector = vsi->q_vectors[q_idx];
4801
4802                 if (q_vector->rx.ring || q_vector->tx.ring)
4803                         napi_enable(&q_vector->napi);
4804         }
4805 }
4806
4807 /**
4808  * i40e_napi_disable_all - Disable NAPI for all q_vectors in the VSI
4809  * @vsi: the VSI being configured
4810  **/
4811 static void i40e_napi_disable_all(struct i40e_vsi *vsi)
4812 {
4813         int q_idx;
4814
4815         if (!vsi->netdev)
4816                 return;
4817
4818         for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++) {
4819                 struct i40e_q_vector *q_vector = vsi->q_vectors[q_idx];
4820
4821                 if (q_vector->rx.ring || q_vector->tx.ring)
4822                         napi_disable(&q_vector->napi);
4823         }
4824 }
4825
4826 /**
4827  * i40e_vsi_close - Shut down a VSI
4828  * @vsi: the vsi to be quelled
4829  **/
4830 static void i40e_vsi_close(struct i40e_vsi *vsi)
4831 {
4832         struct i40e_pf *pf = vsi->back;
4833         if (!test_and_set_bit(__I40E_VSI_DOWN, vsi->state))
4834                 i40e_down(vsi);
4835         i40e_vsi_free_irq(vsi);
4836         i40e_vsi_free_tx_resources(vsi);
4837         i40e_vsi_free_rx_resources(vsi);
4838         vsi->current_netdev_flags = 0;
4839         set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state);
4840         if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
4841                 set_bit(__I40E_CLIENT_RESET, pf->state);
4842 }
4843
4844 /**
4845  * i40e_quiesce_vsi - Pause a given VSI
4846  * @vsi: the VSI being paused
4847  **/
4848 static void i40e_quiesce_vsi(struct i40e_vsi *vsi)
4849 {
4850         if (test_bit(__I40E_VSI_DOWN, vsi->state))
4851                 return;
4852
4853         set_bit(__I40E_VSI_NEEDS_RESTART, vsi->state);
4854         if (vsi->netdev && netif_running(vsi->netdev))
4855                 vsi->netdev->netdev_ops->ndo_stop(vsi->netdev);
4856         else
4857                 i40e_vsi_close(vsi);
4858 }
4859
4860 /**
4861  * i40e_unquiesce_vsi - Resume a given VSI
4862  * @vsi: the VSI being resumed
4863  **/
4864 static void i40e_unquiesce_vsi(struct i40e_vsi *vsi)
4865 {
4866         if (!test_and_clear_bit(__I40E_VSI_NEEDS_RESTART, vsi->state))
4867                 return;
4868
4869         if (vsi->netdev && netif_running(vsi->netdev))
4870                 vsi->netdev->netdev_ops->ndo_open(vsi->netdev);
4871         else
4872                 i40e_vsi_open(vsi);   /* this clears the DOWN bit */
4873 }
4874
4875 /**
4876  * i40e_pf_quiesce_all_vsi - Pause all VSIs on a PF
4877  * @pf: the PF
4878  **/
4879 static void i40e_pf_quiesce_all_vsi(struct i40e_pf *pf)
4880 {
4881         int v;
4882
4883         for (v = 0; v < pf->num_alloc_vsi; v++) {
4884                 if (pf->vsi[v])
4885                         i40e_quiesce_vsi(pf->vsi[v]);
4886         }
4887 }
4888
4889 /**
4890  * i40e_pf_unquiesce_all_vsi - Resume all VSIs on a PF
4891  * @pf: the PF
4892  **/
4893 static void i40e_pf_unquiesce_all_vsi(struct i40e_pf *pf)
4894 {
4895         int v;
4896
4897         for (v = 0; v < pf->num_alloc_vsi; v++) {
4898                 if (pf->vsi[v])
4899                         i40e_unquiesce_vsi(pf->vsi[v]);
4900         }
4901 }
4902
4903 /**
4904  * i40e_vsi_wait_queues_disabled - Wait for VSI's queues to be disabled
4905  * @vsi: the VSI being configured
4906  *
4907  * Wait until all queues on a given VSI have been disabled.
4908  **/
4909 int i40e_vsi_wait_queues_disabled(struct i40e_vsi *vsi)
4910 {
4911         struct i40e_pf *pf = vsi->back;
4912         int i, pf_q, ret;
4913
4914         pf_q = vsi->base_queue;
4915         for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
4916                 /* Check and wait for the Tx queue */
4917                 ret = i40e_pf_txq_wait(pf, pf_q, false);
4918                 if (ret) {
4919                         dev_info(&pf->pdev->dev,
4920                                  "VSI seid %d Tx ring %d disable timeout\n",
4921                                  vsi->seid, pf_q);
4922                         return ret;
4923                 }
4924
4925                 if (!i40e_enabled_xdp_vsi(vsi))
4926                         goto wait_rx;
4927
4928                 /* Check and wait for the XDP Tx queue */
4929                 ret = i40e_pf_txq_wait(pf, pf_q + vsi->alloc_queue_pairs,
4930                                        false);
4931                 if (ret) {
4932                         dev_info(&pf->pdev->dev,
4933                                  "VSI seid %d XDP Tx ring %d disable timeout\n",
4934                                  vsi->seid, pf_q);
4935                         return ret;
4936                 }
4937 wait_rx:
4938                 /* Check and wait for the Rx queue */
4939                 ret = i40e_pf_rxq_wait(pf, pf_q, false);
4940                 if (ret) {
4941                         dev_info(&pf->pdev->dev,
4942                                  "VSI seid %d Rx ring %d disable timeout\n",
4943                                  vsi->seid, pf_q);
4944                         return ret;
4945                 }
4946         }
4947
4948         return 0;
4949 }
4950
4951 #ifdef CONFIG_I40E_DCB
4952 /**
4953  * i40e_pf_wait_queues_disabled - Wait for all queues of PF VSIs to be disabled
4954  * @pf: the PF
4955  *
4956  * This function waits for the queues to be in disabled state for all the
4957  * VSIs that are managed by this PF.
4958  **/
4959 static int i40e_pf_wait_queues_disabled(struct i40e_pf *pf)
4960 {
4961         int v, ret = 0;
4962
4963         for (v = 0; v < pf->hw.func_caps.num_vsis; v++) {
4964                 if (pf->vsi[v]) {
4965                         ret = i40e_vsi_wait_queues_disabled(pf->vsi[v]);
4966                         if (ret)
4967                                 break;
4968                 }
4969         }
4970
4971         return ret;
4972 }
4973
4974 #endif
4975
4976 /**
4977  * i40e_get_iscsi_tc_map - Return TC map for iSCSI APP
4978  * @pf: pointer to PF
4979  *
4980  * Get TC map for ISCSI PF type that will include iSCSI TC
4981  * and LAN TC.
4982  **/
4983 static u8 i40e_get_iscsi_tc_map(struct i40e_pf *pf)
4984 {
4985         struct i40e_dcb_app_priority_table app;
4986         struct i40e_hw *hw = &pf->hw;
4987         u8 enabled_tc = 1; /* TC0 is always enabled */
4988         u8 tc, i;
4989         /* Get the iSCSI APP TLV */
4990         struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config;
4991
4992         for (i = 0; i < dcbcfg->numapps; i++) {
4993                 app = dcbcfg->app[i];
4994                 if (app.selector == I40E_APP_SEL_TCPIP &&
4995                     app.protocolid == I40E_APP_PROTOID_ISCSI) {
4996                         tc = dcbcfg->etscfg.prioritytable[app.priority];
4997                         enabled_tc |= BIT(tc);
4998                         break;
4999                 }
5000         }
5001
5002         return enabled_tc;
5003 }
5004
5005 /**
5006  * i40e_dcb_get_num_tc -  Get the number of TCs from DCBx config
5007  * @dcbcfg: the corresponding DCBx configuration structure
5008  *
5009  * Return the number of TCs from given DCBx configuration
5010  **/
5011 static u8 i40e_dcb_get_num_tc(struct i40e_dcbx_config *dcbcfg)
5012 {
5013         int i, tc_unused = 0;
5014         u8 num_tc = 0;
5015         u8 ret = 0;
5016
5017         /* Scan the ETS Config Priority Table to find
5018          * traffic class enabled for a given priority
5019          * and create a bitmask of enabled TCs
5020          */
5021         for (i = 0; i < I40E_MAX_USER_PRIORITY; i++)
5022                 num_tc |= BIT(dcbcfg->etscfg.prioritytable[i]);
5023
5024         /* Now scan the bitmask to check for
5025          * contiguous TCs starting with TC0
5026          */
5027         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5028                 if (num_tc & BIT(i)) {
5029                         if (!tc_unused) {
5030                                 ret++;
5031                         } else {
5032                                 pr_err("Non-contiguous TC - Disabling DCB\n");
5033                                 return 1;
5034                         }
5035                 } else {
5036                         tc_unused = 1;
5037                 }
5038         }
5039
5040         /* There is always at least TC0 */
5041         if (!ret)
5042                 ret = 1;
5043
5044         return ret;
5045 }
5046
5047 /**
5048  * i40e_dcb_get_enabled_tc - Get enabled traffic classes
5049  * @dcbcfg: the corresponding DCBx configuration structure
5050  *
5051  * Query the current DCB configuration and return the number of
5052  * traffic classes enabled from the given DCBX config
5053  **/
5054 static u8 i40e_dcb_get_enabled_tc(struct i40e_dcbx_config *dcbcfg)
5055 {
5056         u8 num_tc = i40e_dcb_get_num_tc(dcbcfg);
5057         u8 enabled_tc = 1;
5058         u8 i;
5059
5060         for (i = 0; i < num_tc; i++)
5061                 enabled_tc |= BIT(i);
5062
5063         return enabled_tc;
5064 }
5065
5066 /**
5067  * i40e_mqprio_get_enabled_tc - Get enabled traffic classes
5068  * @pf: PF being queried
5069  *
5070  * Query the current MQPRIO configuration and return the number of
5071  * traffic classes enabled.
5072  **/
5073 static u8 i40e_mqprio_get_enabled_tc(struct i40e_pf *pf)
5074 {
5075         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
5076         u8 num_tc = vsi->mqprio_qopt.qopt.num_tc;
5077         u8 enabled_tc = 1, i;
5078
5079         for (i = 1; i < num_tc; i++)
5080                 enabled_tc |= BIT(i);
5081         return enabled_tc;
5082 }
5083
5084 /**
5085  * i40e_pf_get_num_tc - Get enabled traffic classes for PF
5086  * @pf: PF being queried
5087  *
5088  * Return number of traffic classes enabled for the given PF
5089  **/
5090 static u8 i40e_pf_get_num_tc(struct i40e_pf *pf)
5091 {
5092         struct i40e_hw *hw = &pf->hw;
5093         u8 i, enabled_tc = 1;
5094         u8 num_tc = 0;
5095         struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config;
5096
5097         if (pf->flags & I40E_FLAG_TC_MQPRIO)
5098                 return pf->vsi[pf->lan_vsi]->mqprio_qopt.qopt.num_tc;
5099
5100         /* If neither MQPRIO nor DCB is enabled, then always use single TC */
5101         if (!(pf->flags & I40E_FLAG_DCB_ENABLED))
5102                 return 1;
5103
5104         /* SFP mode will be enabled for all TCs on port */
5105         if (!(pf->flags & I40E_FLAG_MFP_ENABLED))
5106                 return i40e_dcb_get_num_tc(dcbcfg);
5107
5108         /* MFP mode return count of enabled TCs for this PF */
5109         if (pf->hw.func_caps.iscsi)
5110                 enabled_tc =  i40e_get_iscsi_tc_map(pf);
5111         else
5112                 return 1; /* Only TC0 */
5113
5114         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5115                 if (enabled_tc & BIT(i))
5116                         num_tc++;
5117         }
5118         return num_tc;
5119 }
5120
5121 /**
5122  * i40e_pf_get_pf_tc_map - Get bitmap for enabled traffic classes
5123  * @pf: PF being queried
5124  *
5125  * Return a bitmap for enabled traffic classes for this PF.
5126  **/
5127 static u8 i40e_pf_get_tc_map(struct i40e_pf *pf)
5128 {
5129         if (pf->flags & I40E_FLAG_TC_MQPRIO)
5130                 return i40e_mqprio_get_enabled_tc(pf);
5131
5132         /* If neither MQPRIO nor DCB is enabled for this PF then just return
5133          * default TC
5134          */
5135         if (!(pf->flags & I40E_FLAG_DCB_ENABLED))
5136                 return I40E_DEFAULT_TRAFFIC_CLASS;
5137
5138         /* SFP mode we want PF to be enabled for all TCs */
5139         if (!(pf->flags & I40E_FLAG_MFP_ENABLED))
5140                 return i40e_dcb_get_enabled_tc(&pf->hw.local_dcbx_config);
5141
5142         /* MFP enabled and iSCSI PF type */
5143         if (pf->hw.func_caps.iscsi)
5144                 return i40e_get_iscsi_tc_map(pf);
5145         else
5146                 return I40E_DEFAULT_TRAFFIC_CLASS;
5147 }
5148
5149 /**
5150  * i40e_vsi_get_bw_info - Query VSI BW Information
5151  * @vsi: the VSI being queried
5152  *
5153  * Returns 0 on success, negative value on failure
5154  **/
5155 static int i40e_vsi_get_bw_info(struct i40e_vsi *vsi)
5156 {
5157         struct i40e_aqc_query_vsi_ets_sla_config_resp bw_ets_config = {0};
5158         struct i40e_aqc_query_vsi_bw_config_resp bw_config = {0};
5159         struct i40e_pf *pf = vsi->back;
5160         struct i40e_hw *hw = &pf->hw;
5161         i40e_status ret;
5162         u32 tc_bw_max;
5163         int i;
5164
5165         /* Get the VSI level BW configuration */
5166         ret = i40e_aq_query_vsi_bw_config(hw, vsi->seid, &bw_config, NULL);
5167         if (ret) {
5168                 dev_info(&pf->pdev->dev,
5169                          "couldn't get PF vsi bw config, err %s aq_err %s\n",
5170                          i40e_stat_str(&pf->hw, ret),
5171                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
5172                 return -EINVAL;
5173         }
5174
5175         /* Get the VSI level BW configuration per TC */
5176         ret = i40e_aq_query_vsi_ets_sla_config(hw, vsi->seid, &bw_ets_config,
5177                                                NULL);
5178         if (ret) {
5179                 dev_info(&pf->pdev->dev,
5180                          "couldn't get PF vsi ets bw config, err %s aq_err %s\n",
5181                          i40e_stat_str(&pf->hw, ret),
5182                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
5183                 return -EINVAL;
5184         }
5185
5186         if (bw_config.tc_valid_bits != bw_ets_config.tc_valid_bits) {
5187                 dev_info(&pf->pdev->dev,
5188                          "Enabled TCs mismatch from querying VSI BW info 0x%08x 0x%08x\n",
5189                          bw_config.tc_valid_bits,
5190                          bw_ets_config.tc_valid_bits);
5191                 /* Still continuing */
5192         }
5193
5194         vsi->bw_limit = le16_to_cpu(bw_config.port_bw_limit);
5195         vsi->bw_max_quanta = bw_config.max_bw;
5196         tc_bw_max = le16_to_cpu(bw_ets_config.tc_bw_max[0]) |
5197                     (le16_to_cpu(bw_ets_config.tc_bw_max[1]) << 16);
5198         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5199                 vsi->bw_ets_share_credits[i] = bw_ets_config.share_credits[i];
5200                 vsi->bw_ets_limit_credits[i] =
5201                                         le16_to_cpu(bw_ets_config.credits[i]);
5202                 /* 3 bits out of 4 for each TC */
5203                 vsi->bw_ets_max_quanta[i] = (u8)((tc_bw_max >> (i*4)) & 0x7);
5204         }
5205
5206         return 0;
5207 }
5208
5209 /**
5210  * i40e_vsi_configure_bw_alloc - Configure VSI BW allocation per TC
5211  * @vsi: the VSI being configured
5212  * @enabled_tc: TC bitmap
5213  * @bw_share: BW shared credits per TC
5214  *
5215  * Returns 0 on success, negative value on failure
5216  **/
5217 static int i40e_vsi_configure_bw_alloc(struct i40e_vsi *vsi, u8 enabled_tc,
5218                                        u8 *bw_share)
5219 {
5220         struct i40e_aqc_configure_vsi_tc_bw_data bw_data;
5221         struct i40e_pf *pf = vsi->back;
5222         i40e_status ret;
5223         int i;
5224
5225         /* There is no need to reset BW when mqprio mode is on.  */
5226         if (pf->flags & I40E_FLAG_TC_MQPRIO)
5227                 return 0;
5228         if (!vsi->mqprio_qopt.qopt.hw && !(pf->flags & I40E_FLAG_DCB_ENABLED)) {
5229                 ret = i40e_set_bw_limit(vsi, vsi->seid, 0);
5230                 if (ret)
5231                         dev_info(&pf->pdev->dev,
5232                                  "Failed to reset tx rate for vsi->seid %u\n",
5233                                  vsi->seid);
5234                 return ret;
5235         }
5236         bw_data.tc_valid_bits = enabled_tc;
5237         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
5238                 bw_data.tc_bw_credits[i] = bw_share[i];
5239
5240         ret = i40e_aq_config_vsi_tc_bw(&pf->hw, vsi->seid, &bw_data, NULL);
5241         if (ret) {
5242                 dev_info(&pf->pdev->dev,
5243                          "AQ command Config VSI BW allocation per TC failed = %d\n",
5244                          pf->hw.aq.asq_last_status);
5245                 return -EINVAL;
5246         }
5247
5248         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
5249                 vsi->info.qs_handle[i] = bw_data.qs_handles[i];
5250
5251         return 0;
5252 }
5253
5254 /**
5255  * i40e_vsi_config_netdev_tc - Setup the netdev TC configuration
5256  * @vsi: the VSI being configured
5257  * @enabled_tc: TC map to be enabled
5258  *
5259  **/
5260 static void i40e_vsi_config_netdev_tc(struct i40e_vsi *vsi, u8 enabled_tc)
5261 {
5262         struct net_device *netdev = vsi->netdev;
5263         struct i40e_pf *pf = vsi->back;
5264         struct i40e_hw *hw = &pf->hw;
5265         u8 netdev_tc = 0;
5266         int i;
5267         struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config;
5268
5269         if (!netdev)
5270                 return;
5271
5272         if (!enabled_tc) {
5273                 netdev_reset_tc(netdev);
5274                 return;
5275         }
5276
5277         /* Set up actual enabled TCs on the VSI */
5278         if (netdev_set_num_tc(netdev, vsi->tc_config.numtc))
5279                 return;
5280
5281         /* set per TC queues for the VSI */
5282         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5283                 /* Only set TC queues for enabled tcs
5284                  *
5285                  * e.g. For a VSI that has TC0 and TC3 enabled the
5286                  * enabled_tc bitmap would be 0x00001001; the driver
5287                  * will set the numtc for netdev as 2 that will be
5288                  * referenced by the netdev layer as TC 0 and 1.
5289                  */
5290                 if (vsi->tc_config.enabled_tc & BIT(i))
5291                         netdev_set_tc_queue(netdev,
5292                                         vsi->tc_config.tc_info[i].netdev_tc,
5293                                         vsi->tc_config.tc_info[i].qcount,
5294                                         vsi->tc_config.tc_info[i].qoffset);
5295         }
5296
5297         if (pf->flags & I40E_FLAG_TC_MQPRIO)
5298                 return;
5299
5300         /* Assign UP2TC map for the VSI */
5301         for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) {
5302                 /* Get the actual TC# for the UP */
5303                 u8 ets_tc = dcbcfg->etscfg.prioritytable[i];
5304                 /* Get the mapped netdev TC# for the UP */
5305                 netdev_tc =  vsi->tc_config.tc_info[ets_tc].netdev_tc;
5306                 netdev_set_prio_tc_map(netdev, i, netdev_tc);
5307         }
5308 }
5309
5310 /**
5311  * i40e_vsi_update_queue_map - Update our copy of VSi info with new queue map
5312  * @vsi: the VSI being configured
5313  * @ctxt: the ctxt buffer returned from AQ VSI update param command
5314  **/
5315 static void i40e_vsi_update_queue_map(struct i40e_vsi *vsi,
5316                                       struct i40e_vsi_context *ctxt)
5317 {
5318         /* copy just the sections touched not the entire info
5319          * since not all sections are valid as returned by
5320          * update vsi params
5321          */
5322         vsi->info.mapping_flags = ctxt->info.mapping_flags;
5323         memcpy(&vsi->info.queue_mapping,
5324                &ctxt->info.queue_mapping, sizeof(vsi->info.queue_mapping));
5325         memcpy(&vsi->info.tc_mapping, ctxt->info.tc_mapping,
5326                sizeof(vsi->info.tc_mapping));
5327 }
5328
5329 /**
5330  * i40e_vsi_config_tc - Configure VSI Tx Scheduler for given TC map
5331  * @vsi: VSI to be configured
5332  * @enabled_tc: TC bitmap
5333  *
5334  * This configures a particular VSI for TCs that are mapped to the
5335  * given TC bitmap. It uses default bandwidth share for TCs across
5336  * VSIs to configure TC for a particular VSI.
5337  *
5338  * NOTE:
5339  * It is expected that the VSI queues have been quisced before calling
5340  * this function.
5341  **/
5342 static int i40e_vsi_config_tc(struct i40e_vsi *vsi, u8 enabled_tc)
5343 {
5344         u8 bw_share[I40E_MAX_TRAFFIC_CLASS] = {0};
5345         struct i40e_pf *pf = vsi->back;
5346         struct i40e_hw *hw = &pf->hw;
5347         struct i40e_vsi_context ctxt;
5348         int ret = 0;
5349         int i;
5350
5351         /* Check if enabled_tc is same as existing or new TCs */
5352         if (vsi->tc_config.enabled_tc == enabled_tc &&
5353             vsi->mqprio_qopt.mode != TC_MQPRIO_MODE_CHANNEL)
5354                 return ret;
5355
5356         /* Enable ETS TCs with equal BW Share for now across all VSIs */
5357         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5358                 if (enabled_tc & BIT(i))
5359                         bw_share[i] = 1;
5360         }
5361
5362         ret = i40e_vsi_configure_bw_alloc(vsi, enabled_tc, bw_share);
5363         if (ret) {
5364                 struct i40e_aqc_query_vsi_bw_config_resp bw_config = {0};
5365
5366                 dev_info(&pf->pdev->dev,
5367                          "Failed configuring TC map %d for VSI %d\n",
5368                          enabled_tc, vsi->seid);
5369                 ret = i40e_aq_query_vsi_bw_config(hw, vsi->seid,
5370                                                   &bw_config, NULL);
5371                 if (ret) {
5372                         dev_info(&pf->pdev->dev,
5373                                  "Failed querying vsi bw info, err %s aq_err %s\n",
5374                                  i40e_stat_str(hw, ret),
5375                                  i40e_aq_str(hw, hw->aq.asq_last_status));
5376                         goto out;
5377                 }
5378                 if ((bw_config.tc_valid_bits & enabled_tc) != enabled_tc) {
5379                         u8 valid_tc = bw_config.tc_valid_bits & enabled_tc;
5380
5381                         if (!valid_tc)
5382                                 valid_tc = bw_config.tc_valid_bits;
5383                         /* Always enable TC0, no matter what */
5384                         valid_tc |= 1;
5385                         dev_info(&pf->pdev->dev,
5386                                  "Requested tc 0x%x, but FW reports 0x%x as valid. Attempting to use 0x%x.\n",
5387                                  enabled_tc, bw_config.tc_valid_bits, valid_tc);
5388                         enabled_tc = valid_tc;
5389                 }
5390
5391                 ret = i40e_vsi_configure_bw_alloc(vsi, enabled_tc, bw_share);
5392                 if (ret) {
5393                         dev_err(&pf->pdev->dev,
5394                                 "Unable to  configure TC map %d for VSI %d\n",
5395                                 enabled_tc, vsi->seid);
5396                         goto out;
5397                 }
5398         }
5399
5400         /* Update Queue Pairs Mapping for currently enabled UPs */
5401         ctxt.seid = vsi->seid;
5402         ctxt.pf_num = vsi->back->hw.pf_id;
5403         ctxt.vf_num = 0;
5404         ctxt.uplink_seid = vsi->uplink_seid;
5405         ctxt.info = vsi->info;
5406         if (vsi->back->flags & I40E_FLAG_TC_MQPRIO) {
5407                 ret = i40e_vsi_setup_queue_map_mqprio(vsi, &ctxt, enabled_tc);
5408                 if (ret)
5409                         goto out;
5410         } else {
5411                 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false);
5412         }
5413
5414         /* On destroying the qdisc, reset vsi->rss_size, as number of enabled
5415          * queues changed.
5416          */
5417         if (!vsi->mqprio_qopt.qopt.hw && vsi->reconfig_rss) {
5418                 vsi->rss_size = min_t(int, vsi->back->alloc_rss_size,
5419                                       vsi->num_queue_pairs);
5420                 ret = i40e_vsi_config_rss(vsi);
5421                 if (ret) {
5422                         dev_info(&vsi->back->pdev->dev,
5423                                  "Failed to reconfig rss for num_queues\n");
5424                         return ret;
5425                 }
5426                 vsi->reconfig_rss = false;
5427         }
5428         if (vsi->back->flags & I40E_FLAG_IWARP_ENABLED) {
5429                 ctxt.info.valid_sections |=
5430                                 cpu_to_le16(I40E_AQ_VSI_PROP_QUEUE_OPT_VALID);
5431                 ctxt.info.queueing_opt_flags |= I40E_AQ_VSI_QUE_OPT_TCP_ENA;
5432         }
5433
5434         /* Update the VSI after updating the VSI queue-mapping
5435          * information
5436          */
5437         ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
5438         if (ret) {
5439                 dev_info(&pf->pdev->dev,
5440                          "Update vsi tc config failed, err %s aq_err %s\n",
5441                          i40e_stat_str(hw, ret),
5442                          i40e_aq_str(hw, hw->aq.asq_last_status));
5443                 goto out;
5444         }
5445         /* update the local VSI info with updated queue map */
5446         i40e_vsi_update_queue_map(vsi, &ctxt);
5447         vsi->info.valid_sections = 0;
5448
5449         /* Update current VSI BW information */
5450         ret = i40e_vsi_get_bw_info(vsi);
5451         if (ret) {
5452                 dev_info(&pf->pdev->dev,
5453                          "Failed updating vsi bw info, err %s aq_err %s\n",
5454                          i40e_stat_str(hw, ret),
5455                          i40e_aq_str(hw, hw->aq.asq_last_status));
5456                 goto out;
5457         }
5458
5459         /* Update the netdev TC setup */
5460         i40e_vsi_config_netdev_tc(vsi, enabled_tc);
5461 out:
5462         return ret;
5463 }
5464
5465 /**
5466  * i40e_get_link_speed - Returns link speed for the interface
5467  * @vsi: VSI to be configured
5468  *
5469  **/
5470 static int i40e_get_link_speed(struct i40e_vsi *vsi)
5471 {
5472         struct i40e_pf *pf = vsi->back;
5473
5474         switch (pf->hw.phy.link_info.link_speed) {
5475         case I40E_LINK_SPEED_40GB:
5476                 return 40000;
5477         case I40E_LINK_SPEED_25GB:
5478                 return 25000;
5479         case I40E_LINK_SPEED_20GB:
5480                 return 20000;
5481         case I40E_LINK_SPEED_10GB:
5482                 return 10000;
5483         case I40E_LINK_SPEED_1GB:
5484                 return 1000;
5485         default:
5486                 return -EINVAL;
5487         }
5488 }
5489
5490 /**
5491  * i40e_bw_bytes_to_mbits - Convert max_tx_rate from bytes to mbits
5492  * @vsi: Pointer to vsi structure
5493  * @max_tx_rate: max TX rate in bytes to be converted into Mbits
5494  *
5495  * Helper function to convert units before send to set BW limit
5496  **/
5497 static u64 i40e_bw_bytes_to_mbits(struct i40e_vsi *vsi, u64 max_tx_rate)
5498 {
5499         if (max_tx_rate < I40E_BW_MBPS_DIVISOR) {
5500                 dev_warn(&vsi->back->pdev->dev,
5501                          "Setting max tx rate to minimum usable value of 50Mbps.\n");
5502                 max_tx_rate = I40E_BW_CREDIT_DIVISOR;
5503         } else {
5504                 do_div(max_tx_rate, I40E_BW_MBPS_DIVISOR);
5505         }
5506
5507         return max_tx_rate;
5508 }
5509
5510 /**
5511  * i40e_set_bw_limit - setup BW limit for Tx traffic based on max_tx_rate
5512  * @vsi: VSI to be configured
5513  * @seid: seid of the channel/VSI
5514  * @max_tx_rate: max TX rate to be configured as BW limit
5515  *
5516  * Helper function to set BW limit for a given VSI
5517  **/
5518 int i40e_set_bw_limit(struct i40e_vsi *vsi, u16 seid, u64 max_tx_rate)
5519 {
5520         struct i40e_pf *pf = vsi->back;
5521         u64 credits = 0;
5522         int speed = 0;
5523         int ret = 0;
5524
5525         speed = i40e_get_link_speed(vsi);
5526         if (max_tx_rate > speed) {
5527                 dev_err(&pf->pdev->dev,
5528                         "Invalid max tx rate %llu specified for VSI seid %d.",
5529                         max_tx_rate, seid);
5530                 return -EINVAL;
5531         }
5532         if (max_tx_rate && max_tx_rate < I40E_BW_CREDIT_DIVISOR) {
5533                 dev_warn(&pf->pdev->dev,
5534                          "Setting max tx rate to minimum usable value of 50Mbps.\n");
5535                 max_tx_rate = I40E_BW_CREDIT_DIVISOR;
5536         }
5537
5538         /* Tx rate credits are in values of 50Mbps, 0 is disabled */
5539         credits = max_tx_rate;
5540         do_div(credits, I40E_BW_CREDIT_DIVISOR);
5541         ret = i40e_aq_config_vsi_bw_limit(&pf->hw, seid, credits,
5542                                           I40E_MAX_BW_INACTIVE_ACCUM, NULL);
5543         if (ret)
5544                 dev_err(&pf->pdev->dev,
5545                         "Failed set tx rate (%llu Mbps) for vsi->seid %u, err %s aq_err %s\n",
5546                         max_tx_rate, seid, i40e_stat_str(&pf->hw, ret),
5547                         i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
5548         return ret;
5549 }
5550
5551 /**
5552  * i40e_remove_queue_channels - Remove queue channels for the TCs
5553  * @vsi: VSI to be configured
5554  *
5555  * Remove queue channels for the TCs
5556  **/
5557 static void i40e_remove_queue_channels(struct i40e_vsi *vsi)
5558 {
5559         enum i40e_admin_queue_err last_aq_status;
5560         struct i40e_cloud_filter *cfilter;
5561         struct i40e_channel *ch, *ch_tmp;
5562         struct i40e_pf *pf = vsi->back;
5563         struct hlist_node *node;
5564         int ret, i;
5565
5566         /* Reset rss size that was stored when reconfiguring rss for
5567          * channel VSIs with non-power-of-2 queue count.
5568          */
5569         vsi->current_rss_size = 0;
5570
5571         /* perform cleanup for channels if they exist */
5572         if (list_empty(&vsi->ch_list))
5573                 return;
5574
5575         list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) {
5576                 struct i40e_vsi *p_vsi;
5577
5578                 list_del(&ch->list);
5579                 p_vsi = ch->parent_vsi;
5580                 if (!p_vsi || !ch->initialized) {
5581                         kfree(ch);
5582                         continue;
5583                 }
5584                 /* Reset queue contexts */
5585                 for (i = 0; i < ch->num_queue_pairs; i++) {
5586                         struct i40e_ring *tx_ring, *rx_ring;
5587                         u16 pf_q;
5588
5589                         pf_q = ch->base_queue + i;
5590                         tx_ring = vsi->tx_rings[pf_q];
5591                         tx_ring->ch = NULL;
5592
5593                         rx_ring = vsi->rx_rings[pf_q];
5594                         rx_ring->ch = NULL;
5595                 }
5596
5597                 /* Reset BW configured for this VSI via mqprio */
5598                 ret = i40e_set_bw_limit(vsi, ch->seid, 0);
5599                 if (ret)
5600                         dev_info(&vsi->back->pdev->dev,
5601                                  "Failed to reset tx rate for ch->seid %u\n",
5602                                  ch->seid);
5603
5604                 /* delete cloud filters associated with this channel */
5605                 hlist_for_each_entry_safe(cfilter, node,
5606                                           &pf->cloud_filter_list, cloud_node) {
5607                         if (cfilter->seid != ch->seid)
5608                                 continue;
5609
5610                         hash_del(&cfilter->cloud_node);
5611                         if (cfilter->dst_port)
5612                                 ret = i40e_add_del_cloud_filter_big_buf(vsi,
5613                                                                         cfilter,
5614                                                                         false);
5615                         else
5616                                 ret = i40e_add_del_cloud_filter(vsi, cfilter,
5617                                                                 false);
5618                         last_aq_status = pf->hw.aq.asq_last_status;
5619                         if (ret)
5620                                 dev_info(&pf->pdev->dev,
5621                                          "Failed to delete cloud filter, err %s aq_err %s\n",
5622                                          i40e_stat_str(&pf->hw, ret),
5623                                          i40e_aq_str(&pf->hw, last_aq_status));
5624                         kfree(cfilter);
5625                 }
5626
5627                 /* delete VSI from FW */
5628                 ret = i40e_aq_delete_element(&vsi->back->hw, ch->seid,
5629                                              NULL);
5630                 if (ret)
5631                         dev_err(&vsi->back->pdev->dev,
5632                                 "unable to remove channel (%d) for parent VSI(%d)\n",
5633                                 ch->seid, p_vsi->seid);
5634                 kfree(ch);
5635         }
5636         INIT_LIST_HEAD(&vsi->ch_list);
5637 }
5638
5639 /**
5640  * i40e_is_any_channel - channel exist or not
5641  * @vsi: ptr to VSI to which channels are associated with
5642  *
5643  * Returns true or false if channel(s) exist for associated VSI or not
5644  **/
5645 static bool i40e_is_any_channel(struct i40e_vsi *vsi)
5646 {
5647         struct i40e_channel *ch, *ch_tmp;
5648
5649         list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) {
5650                 if (ch->initialized)
5651                         return true;
5652         }
5653
5654         return false;
5655 }
5656
5657 /**
5658  * i40e_get_max_queues_for_channel
5659  * @vsi: ptr to VSI to which channels are associated with
5660  *
5661  * Helper function which returns max value among the queue counts set on the
5662  * channels/TCs created.
5663  **/
5664 static int i40e_get_max_queues_for_channel(struct i40e_vsi *vsi)
5665 {
5666         struct i40e_channel *ch, *ch_tmp;
5667         int max = 0;
5668
5669         list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) {
5670                 if (!ch->initialized)
5671                         continue;
5672                 if (ch->num_queue_pairs > max)
5673                         max = ch->num_queue_pairs;
5674         }
5675
5676         return max;
5677 }
5678
5679 /**
5680  * i40e_validate_num_queues - validate num_queues w.r.t channel
5681  * @pf: ptr to PF device
5682  * @num_queues: number of queues
5683  * @vsi: the parent VSI
5684  * @reconfig_rss: indicates should the RSS be reconfigured or not
5685  *
5686  * This function validates number of queues in the context of new channel
5687  * which is being established and determines if RSS should be reconfigured
5688  * or not for parent VSI.
5689  **/
5690 static int i40e_validate_num_queues(struct i40e_pf *pf, int num_queues,
5691                                     struct i40e_vsi *vsi, bool *reconfig_rss)
5692 {
5693         int max_ch_queues;
5694
5695         if (!reconfig_rss)
5696                 return -EINVAL;
5697
5698         *reconfig_rss = false;
5699         if (vsi->current_rss_size) {
5700                 if (num_queues > vsi->current_rss_size) {
5701                         dev_dbg(&pf->pdev->dev,
5702                                 "Error: num_queues (%d) > vsi's current_size(%d)\n",
5703                                 num_queues, vsi->current_rss_size);
5704                         return -EINVAL;
5705                 } else if ((num_queues < vsi->current_rss_size) &&
5706                            (!is_power_of_2(num_queues))) {
5707                         dev_dbg(&pf->pdev->dev,
5708                                 "Error: num_queues (%d) < vsi's current_size(%d), but not power of 2\n",
5709                                 num_queues, vsi->current_rss_size);
5710                         return -EINVAL;
5711                 }
5712         }
5713
5714         if (!is_power_of_2(num_queues)) {
5715                 /* Find the max num_queues configured for channel if channel
5716                  * exist.
5717                  * if channel exist, then enforce 'num_queues' to be more than
5718                  * max ever queues configured for channel.
5719                  */
5720                 max_ch_queues = i40e_get_max_queues_for_channel(vsi);
5721                 if (num_queues < max_ch_queues) {
5722                         dev_dbg(&pf->pdev->dev,
5723                                 "Error: num_queues (%d) < max queues configured for channel(%d)\n",
5724                                 num_queues, max_ch_queues);
5725                         return -EINVAL;
5726                 }
5727                 *reconfig_rss = true;
5728         }
5729
5730         return 0;
5731 }
5732
5733 /**
5734  * i40e_vsi_reconfig_rss - reconfig RSS based on specified rss_size
5735  * @vsi: the VSI being setup
5736  * @rss_size: size of RSS, accordingly LUT gets reprogrammed
5737  *
5738  * This function reconfigures RSS by reprogramming LUTs using 'rss_size'
5739  **/
5740 static int i40e_vsi_reconfig_rss(struct i40e_vsi *vsi, u16 rss_size)
5741 {
5742         struct i40e_pf *pf = vsi->back;
5743         u8 seed[I40E_HKEY_ARRAY_SIZE];
5744         struct i40e_hw *hw = &pf->hw;
5745         int local_rss_size;
5746         u8 *lut;
5747         int ret;
5748
5749         if (!vsi->rss_size)
5750                 return -EINVAL;
5751
5752         if (rss_size > vsi->rss_size)
5753                 return -EINVAL;
5754
5755         local_rss_size = min_t(int, vsi->rss_size, rss_size);
5756         lut = kzalloc(vsi->rss_table_size, GFP_KERNEL);
5757         if (!lut)
5758                 return -ENOMEM;
5759
5760         /* Ignoring user configured lut if there is one */
5761         i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, local_rss_size);
5762
5763         /* Use user configured hash key if there is one, otherwise
5764          * use default.
5765          */
5766         if (vsi->rss_hkey_user)
5767                 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE);
5768         else
5769                 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE);
5770
5771         ret = i40e_config_rss(vsi, seed, lut, vsi->rss_table_size);
5772         if (ret) {
5773                 dev_info(&pf->pdev->dev,
5774                          "Cannot set RSS lut, err %s aq_err %s\n",
5775                          i40e_stat_str(hw, ret),
5776                          i40e_aq_str(hw, hw->aq.asq_last_status));
5777                 kfree(lut);
5778                 return ret;
5779         }
5780         kfree(lut);
5781
5782         /* Do the update w.r.t. storing rss_size */
5783         if (!vsi->orig_rss_size)
5784                 vsi->orig_rss_size = vsi->rss_size;
5785         vsi->current_rss_size = local_rss_size;
5786
5787         return ret;
5788 }
5789
5790 /**
5791  * i40e_channel_setup_queue_map - Setup a channel queue map
5792  * @pf: ptr to PF device
5793  * @vsi: the VSI being setup
5794  * @ctxt: VSI context structure
5795  * @ch: ptr to channel structure
5796  *
5797  * Setup queue map for a specific channel
5798  **/
5799 static void i40e_channel_setup_queue_map(struct i40e_pf *pf,
5800                                          struct i40e_vsi_context *ctxt,
5801                                          struct i40e_channel *ch)
5802 {
5803         u16 qcount, qmap, sections = 0;
5804         u8 offset = 0;
5805         int pow;
5806
5807         sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;
5808         sections |= I40E_AQ_VSI_PROP_SCHED_VALID;
5809
5810         qcount = min_t(int, ch->num_queue_pairs, pf->num_lan_msix);
5811         ch->num_queue_pairs = qcount;
5812
5813         /* find the next higher power-of-2 of num queue pairs */
5814         pow = ilog2(qcount);
5815         if (!is_power_of_2(qcount))
5816                 pow++;
5817
5818         qmap = (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
5819                 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT);
5820
5821         /* Setup queue TC[0].qmap for given VSI context */
5822         ctxt->info.tc_mapping[0] = cpu_to_le16(qmap);
5823
5824         ctxt->info.up_enable_bits = 0x1; /* TC0 enabled */
5825         ctxt->info.mapping_flags |= cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG);
5826         ctxt->info.queue_mapping[0] = cpu_to_le16(ch->base_queue);
5827         ctxt->info.valid_sections |= cpu_to_le16(sections);
5828 }
5829
5830 /**
5831  * i40e_add_channel - add a channel by adding VSI
5832  * @pf: ptr to PF device
5833  * @uplink_seid: underlying HW switching element (VEB) ID
5834  * @ch: ptr to channel structure
5835  *
5836  * Add a channel (VSI) using add_vsi and queue_map
5837  **/
5838 static int i40e_add_channel(struct i40e_pf *pf, u16 uplink_seid,
5839                             struct i40e_channel *ch)
5840 {
5841         struct i40e_hw *hw = &pf->hw;
5842         struct i40e_vsi_context ctxt;
5843         u8 enabled_tc = 0x1; /* TC0 enabled */
5844         int ret;
5845
5846         if (ch->type != I40E_VSI_VMDQ2) {
5847                 dev_info(&pf->pdev->dev,
5848                          "add new vsi failed, ch->type %d\n", ch->type);
5849                 return -EINVAL;
5850         }
5851
5852         memset(&ctxt, 0, sizeof(ctxt));
5853         ctxt.pf_num = hw->pf_id;
5854         ctxt.vf_num = 0;
5855         ctxt.uplink_seid = uplink_seid;
5856         ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
5857         if (ch->type == I40E_VSI_VMDQ2)
5858                 ctxt.flags = I40E_AQ_VSI_TYPE_VMDQ2;
5859
5860         if (pf->flags & I40E_FLAG_VEB_MODE_ENABLED) {
5861                 ctxt.info.valid_sections |=
5862                      cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
5863                 ctxt.info.switch_id =
5864                    cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
5865         }
5866
5867         /* Set queue map for a given VSI context */
5868         i40e_channel_setup_queue_map(pf, &ctxt, ch);
5869
5870         /* Now time to create VSI */
5871         ret = i40e_aq_add_vsi(hw, &ctxt, NULL);
5872         if (ret) {
5873                 dev_info(&pf->pdev->dev,
5874                          "add new vsi failed, err %s aq_err %s\n",
5875                          i40e_stat_str(&pf->hw, ret),
5876                          i40e_aq_str(&pf->hw,
5877                                      pf->hw.aq.asq_last_status));
5878                 return -ENOENT;
5879         }
5880
5881         /* Success, update channel */
5882         ch->enabled_tc = enabled_tc;
5883         ch->seid = ctxt.seid;
5884         ch->vsi_number = ctxt.vsi_number;
5885         ch->stat_counter_idx = cpu_to_le16(ctxt.info.stat_counter_idx);
5886
5887         /* copy just the sections touched not the entire info
5888          * since not all sections are valid as returned by
5889          * update vsi params
5890          */
5891         ch->info.mapping_flags = ctxt.info.mapping_flags;
5892         memcpy(&ch->info.queue_mapping,
5893                &ctxt.info.queue_mapping, sizeof(ctxt.info.queue_mapping));
5894         memcpy(&ch->info.tc_mapping, ctxt.info.tc_mapping,
5895                sizeof(ctxt.info.tc_mapping));
5896
5897         return 0;
5898 }
5899
5900 static int i40e_channel_config_bw(struct i40e_vsi *vsi, struct i40e_channel *ch,
5901                                   u8 *bw_share)
5902 {
5903         struct i40e_aqc_configure_vsi_tc_bw_data bw_data;
5904         i40e_status ret;
5905         int i;
5906
5907         bw_data.tc_valid_bits = ch->enabled_tc;
5908         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
5909                 bw_data.tc_bw_credits[i] = bw_share[i];
5910
5911         ret = i40e_aq_config_vsi_tc_bw(&vsi->back->hw, ch->seid,
5912                                        &bw_data, NULL);
5913         if (ret) {
5914                 dev_info(&vsi->back->pdev->dev,
5915                          "Config VSI BW allocation per TC failed, aq_err: %d for new_vsi->seid %u\n",
5916                          vsi->back->hw.aq.asq_last_status, ch->seid);
5917                 return -EINVAL;
5918         }
5919
5920         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
5921                 ch->info.qs_handle[i] = bw_data.qs_handles[i];
5922
5923         return 0;
5924 }
5925
5926 /**
5927  * i40e_channel_config_tx_ring - config TX ring associated with new channel
5928  * @pf: ptr to PF device
5929  * @vsi: the VSI being setup
5930  * @ch: ptr to channel structure
5931  *
5932  * Configure TX rings associated with channel (VSI) since queues are being
5933  * from parent VSI.
5934  **/
5935 static int i40e_channel_config_tx_ring(struct i40e_pf *pf,
5936                                        struct i40e_vsi *vsi,
5937                                        struct i40e_channel *ch)
5938 {
5939         i40e_status ret;
5940         int i;
5941         u8 bw_share[I40E_MAX_TRAFFIC_CLASS] = {0};
5942
5943         /* Enable ETS TCs with equal BW Share for now across all VSIs */
5944         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5945                 if (ch->enabled_tc & BIT(i))
5946                         bw_share[i] = 1;
5947         }
5948
5949         /* configure BW for new VSI */
5950         ret = i40e_channel_config_bw(vsi, ch, bw_share);
5951         if (ret) {
5952                 dev_info(&vsi->back->pdev->dev,
5953                          "Failed configuring TC map %d for channel (seid %u)\n",
5954                          ch->enabled_tc, ch->seid);
5955                 return ret;
5956         }
5957
5958         for (i = 0; i < ch->num_queue_pairs; i++) {
5959                 struct i40e_ring *tx_ring, *rx_ring;
5960                 u16 pf_q;
5961
5962                 pf_q = ch->base_queue + i;
5963
5964                 /* Get to TX ring ptr of main VSI, for re-setup TX queue
5965                  * context
5966                  */
5967                 tx_ring = vsi->tx_rings[pf_q];
5968                 tx_ring->ch = ch;
5969
5970                 /* Get the RX ring ptr */
5971                 rx_ring = vsi->rx_rings[pf_q];
5972                 rx_ring->ch = ch;
5973         }
5974
5975         return 0;
5976 }
5977
5978 /**
5979  * i40e_setup_hw_channel - setup new channel
5980  * @pf: ptr to PF device
5981  * @vsi: the VSI being setup
5982  * @ch: ptr to channel structure
5983  * @uplink_seid: underlying HW switching element (VEB) ID
5984  * @type: type of channel to be created (VMDq2/VF)
5985  *
5986  * Setup new channel (VSI) based on specified type (VMDq2/VF)
5987  * and configures TX rings accordingly
5988  **/
5989 static inline int i40e_setup_hw_channel(struct i40e_pf *pf,
5990                                         struct i40e_vsi *vsi,
5991                                         struct i40e_channel *ch,
5992                                         u16 uplink_seid, u8 type)
5993 {
5994         int ret;
5995
5996         ch->initialized = false;
5997         ch->base_queue = vsi->next_base_queue;
5998         ch->type = type;
5999
6000         /* Proceed with creation of channel (VMDq2) VSI */
6001         ret = i40e_add_channel(pf, uplink_seid, ch);
6002         if (ret) {
6003                 dev_info(&pf->pdev->dev,
6004                          "failed to add_channel using uplink_seid %u\n",
6005                          uplink_seid);
6006                 return ret;
6007         }
6008
6009         /* Mark the successful creation of channel */
6010         ch->initialized = true;
6011
6012         /* Reconfigure TX queues using QTX_CTL register */
6013         ret = i40e_channel_config_tx_ring(pf, vsi, ch);
6014         if (ret) {
6015                 dev_info(&pf->pdev->dev,
6016                          "failed to configure TX rings for channel %u\n",
6017                          ch->seid);
6018                 return ret;
6019         }
6020
6021         /* update 'next_base_queue' */
6022         vsi->next_base_queue = vsi->next_base_queue + ch->num_queue_pairs;
6023         dev_dbg(&pf->pdev->dev,
6024                 "Added channel: vsi_seid %u, vsi_number %u, stat_counter_idx %u, num_queue_pairs %u, pf->next_base_queue %d\n",
6025                 ch->seid, ch->vsi_number, ch->stat_counter_idx,
6026                 ch->num_queue_pairs,
6027                 vsi->next_base_queue);
6028         return ret;
6029 }
6030
6031 /**
6032  * i40e_setup_channel - setup new channel using uplink element
6033  * @pf: ptr to PF device
6034  * @type: type of channel to be created (VMDq2/VF)
6035  * @uplink_seid: underlying HW switching element (VEB) ID
6036  * @ch: ptr to channel structure
6037  *
6038  * Setup new channel (VSI) based on specified type (VMDq2/VF)
6039  * and uplink switching element (uplink_seid)
6040  **/
6041 static bool i40e_setup_channel(struct i40e_pf *pf, struct i40e_vsi *vsi,
6042                                struct i40e_channel *ch)
6043 {
6044         u8 vsi_type;
6045         u16 seid;
6046         int ret;
6047
6048         if (vsi->type == I40E_VSI_MAIN) {
6049                 vsi_type = I40E_VSI_VMDQ2;
6050         } else {
6051                 dev_err(&pf->pdev->dev, "unsupported parent vsi type(%d)\n",
6052                         vsi->type);
6053                 return false;
6054         }
6055
6056         /* underlying switching element */
6057         seid = pf->vsi[pf->lan_vsi]->uplink_seid;
6058
6059         /* create channel (VSI), configure TX rings */
6060         ret = i40e_setup_hw_channel(pf, vsi, ch, seid, vsi_type);
6061         if (ret) {
6062                 dev_err(&pf->pdev->dev, "failed to setup hw_channel\n");
6063                 return false;
6064         }
6065
6066         return ch->initialized ? true : false;
6067 }
6068
6069 /**
6070  * i40e_validate_and_set_switch_mode - sets up switch mode correctly
6071  * @vsi: ptr to VSI which has PF backing
6072  *
6073  * Sets up switch mode correctly if it needs to be changed and perform
6074  * what are allowed modes.
6075  **/
6076 static int i40e_validate_and_set_switch_mode(struct i40e_vsi *vsi)
6077 {
6078         u8 mode;
6079         struct i40e_pf *pf = vsi->back;
6080         struct i40e_hw *hw = &pf->hw;
6081         int ret;
6082
6083         ret = i40e_get_capabilities(pf, i40e_aqc_opc_list_dev_capabilities);
6084         if (ret)
6085                 return -EINVAL;
6086
6087         if (hw->dev_caps.switch_mode) {
6088                 /* if switch mode is set, support mode2 (non-tunneled for
6089                  * cloud filter) for now
6090                  */
6091                 u32 switch_mode = hw->dev_caps.switch_mode &
6092                                   I40E_SWITCH_MODE_MASK;
6093                 if (switch_mode >= I40E_CLOUD_FILTER_MODE1) {
6094                         if (switch_mode == I40E_CLOUD_FILTER_MODE2)
6095                                 return 0;
6096                         dev_err(&pf->pdev->dev,
6097                                 "Invalid switch_mode (%d), only non-tunneled mode for cloud filter is supported\n",
6098                                 hw->dev_caps.switch_mode);
6099                         return -EINVAL;
6100                 }
6101         }
6102
6103         /* Set Bit 7 to be valid */
6104         mode = I40E_AQ_SET_SWITCH_BIT7_VALID;
6105
6106         /* Set L4type for TCP support */
6107         mode |= I40E_AQ_SET_SWITCH_L4_TYPE_TCP;
6108
6109         /* Set cloud filter mode */
6110         mode |= I40E_AQ_SET_SWITCH_MODE_NON_TUNNEL;
6111
6112         /* Prep mode field for set_switch_config */
6113         ret = i40e_aq_set_switch_config(hw, pf->last_sw_conf_flags,
6114                                         pf->last_sw_conf_valid_flags,
6115                                         mode, NULL);
6116         if (ret && hw->aq.asq_last_status != I40E_AQ_RC_ESRCH)
6117                 dev_err(&pf->pdev->dev,
6118                         "couldn't set switch config bits, err %s aq_err %s\n",
6119                         i40e_stat_str(hw, ret),
6120                         i40e_aq_str(hw,
6121                                     hw->aq.asq_last_status));
6122
6123         return ret;
6124 }
6125
6126 /**
6127  * i40e_create_queue_channel - function to create channel
6128  * @vsi: VSI to be configured
6129  * @ch: ptr to channel (it contains channel specific params)
6130  *
6131  * This function creates channel (VSI) using num_queues specified by user,
6132  * reconfigs RSS if needed.
6133  **/
6134 int i40e_create_queue_channel(struct i40e_vsi *vsi,
6135                               struct i40e_channel *ch)
6136 {
6137         struct i40e_pf *pf = vsi->back;
6138         bool reconfig_rss;
6139         int err;
6140
6141         if (!ch)
6142                 return -EINVAL;
6143
6144         if (!ch->num_queue_pairs) {
6145                 dev_err(&pf->pdev->dev, "Invalid num_queues requested: %d\n",
6146                         ch->num_queue_pairs);
6147                 return -EINVAL;
6148         }
6149
6150         /* validate user requested num_queues for channel */
6151         err = i40e_validate_num_queues(pf, ch->num_queue_pairs, vsi,
6152                                        &reconfig_rss);
6153         if (err) {
6154                 dev_info(&pf->pdev->dev, "Failed to validate num_queues (%d)\n",
6155                          ch->num_queue_pairs);
6156                 return -EINVAL;
6157         }
6158
6159         /* By default we are in VEPA mode, if this is the first VF/VMDq
6160          * VSI to be added switch to VEB mode.
6161          */
6162         if ((!(pf->flags & I40E_FLAG_VEB_MODE_ENABLED)) ||
6163             (!i40e_is_any_channel(vsi))) {
6164                 if (!is_power_of_2(vsi->tc_config.tc_info[0].qcount)) {
6165                         dev_dbg(&pf->pdev->dev,
6166                                 "Failed to create channel. Override queues (%u) not power of 2\n",
6167                                 vsi->tc_config.tc_info[0].qcount);
6168                         return -EINVAL;
6169                 }
6170
6171                 if (!(pf->flags & I40E_FLAG_VEB_MODE_ENABLED)) {
6172                         pf->flags |= I40E_FLAG_VEB_MODE_ENABLED;
6173
6174                         if (vsi->type == I40E_VSI_MAIN) {
6175                                 if (pf->flags & I40E_FLAG_TC_MQPRIO)
6176                                         i40e_do_reset(pf, I40E_PF_RESET_FLAG,
6177                                                       true);
6178                                 else
6179                                         i40e_do_reset_safe(pf,
6180                                                            I40E_PF_RESET_FLAG);
6181                         }
6182                 }
6183                 /* now onwards for main VSI, number of queues will be value
6184                  * of TC0's queue count
6185                  */
6186         }
6187
6188         /* By this time, vsi->cnt_q_avail shall be set to non-zero and
6189          * it should be more than num_queues
6190          */
6191         if (!vsi->cnt_q_avail || vsi->cnt_q_avail < ch->num_queue_pairs) {
6192                 dev_dbg(&pf->pdev->dev,
6193                         "Error: cnt_q_avail (%u) less than num_queues %d\n",
6194                         vsi->cnt_q_avail, ch->num_queue_pairs);
6195                 return -EINVAL;
6196         }
6197
6198         /* reconfig_rss only if vsi type is MAIN_VSI */
6199         if (reconfig_rss && (vsi->type == I40E_VSI_MAIN)) {
6200                 err = i40e_vsi_reconfig_rss(vsi, ch->num_queue_pairs);
6201                 if (err) {
6202                         dev_info(&pf->pdev->dev,
6203                                  "Error: unable to reconfig rss for num_queues (%u)\n",
6204                                  ch->num_queue_pairs);
6205                         return -EINVAL;
6206                 }
6207         }
6208
6209         if (!i40e_setup_channel(pf, vsi, ch)) {
6210                 dev_info(&pf->pdev->dev, "Failed to setup channel\n");
6211                 return -EINVAL;
6212         }
6213
6214         dev_info(&pf->pdev->dev,
6215                  "Setup channel (id:%u) utilizing num_queues %d\n",
6216                  ch->seid, ch->num_queue_pairs);
6217
6218         /* configure VSI for BW limit */
6219         if (ch->max_tx_rate) {
6220                 u64 credits = ch->max_tx_rate;
6221
6222                 if (i40e_set_bw_limit(vsi, ch->seid, ch->max_tx_rate))
6223                         return -EINVAL;
6224
6225                 do_div(credits, I40E_BW_CREDIT_DIVISOR);
6226                 dev_dbg(&pf->pdev->dev,
6227                         "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n",
6228                         ch->max_tx_rate,
6229                         credits,
6230                         ch->seid);
6231         }
6232
6233         /* in case of VF, this will be main SRIOV VSI */
6234         ch->parent_vsi = vsi;
6235
6236         /* and update main_vsi's count for queue_available to use */
6237         vsi->cnt_q_avail -= ch->num_queue_pairs;
6238
6239         return 0;
6240 }
6241
6242 /**
6243  * i40e_configure_queue_channels - Add queue channel for the given TCs
6244  * @vsi: VSI to be configured
6245  *
6246  * Configures queue channel mapping to the given TCs
6247  **/
6248 static int i40e_configure_queue_channels(struct i40e_vsi *vsi)
6249 {
6250         struct i40e_channel *ch;
6251         u64 max_rate = 0;
6252         int ret = 0, i;
6253
6254         /* Create app vsi with the TCs. Main VSI with TC0 is already set up */
6255         vsi->tc_seid_map[0] = vsi->seid;
6256         for (i = 1; i < I40E_MAX_TRAFFIC_CLASS; i++) {
6257                 if (vsi->tc_config.enabled_tc & BIT(i)) {
6258                         ch = kzalloc(sizeof(*ch), GFP_KERNEL);
6259                         if (!ch) {
6260                                 ret = -ENOMEM;
6261                                 goto err_free;
6262                         }
6263
6264                         INIT_LIST_HEAD(&ch->list);
6265                         ch->num_queue_pairs =
6266                                 vsi->tc_config.tc_info[i].qcount;
6267                         ch->base_queue =
6268                                 vsi->tc_config.tc_info[i].qoffset;
6269
6270                         /* Bandwidth limit through tc interface is in bytes/s,
6271                          * change to Mbit/s
6272                          */
6273                         max_rate = vsi->mqprio_qopt.max_rate[i];
6274                         do_div(max_rate, I40E_BW_MBPS_DIVISOR);
6275                         ch->max_tx_rate = max_rate;
6276
6277                         list_add_tail(&ch->list, &vsi->ch_list);
6278
6279                         ret = i40e_create_queue_channel(vsi, ch);
6280                         if (ret) {
6281                                 dev_err(&vsi->back->pdev->dev,
6282                                         "Failed creating queue channel with TC%d: queues %d\n",
6283                                         i, ch->num_queue_pairs);
6284                                 goto err_free;
6285                         }
6286                         vsi->tc_seid_map[i] = ch->seid;
6287                 }
6288         }
6289         return ret;
6290
6291 err_free:
6292         i40e_remove_queue_channels(vsi);
6293         return ret;
6294 }
6295
6296 /**
6297  * i40e_veb_config_tc - Configure TCs for given VEB
6298  * @veb: given VEB
6299  * @enabled_tc: TC bitmap
6300  *
6301  * Configures given TC bitmap for VEB (switching) element
6302  **/
6303 int i40e_veb_config_tc(struct i40e_veb *veb, u8 enabled_tc)
6304 {
6305         struct i40e_aqc_configure_switching_comp_bw_config_data bw_data = {0};
6306         struct i40e_pf *pf = veb->pf;
6307         int ret = 0;
6308         int i;
6309
6310         /* No TCs or already enabled TCs just return */
6311         if (!enabled_tc || veb->enabled_tc == enabled_tc)
6312                 return ret;
6313
6314         bw_data.tc_valid_bits = enabled_tc;
6315         /* bw_data.absolute_credits is not set (relative) */
6316
6317         /* Enable ETS TCs with equal BW Share for now */
6318         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
6319                 if (enabled_tc & BIT(i))
6320                         bw_data.tc_bw_share_credits[i] = 1;
6321         }
6322
6323         ret = i40e_aq_config_switch_comp_bw_config(&pf->hw, veb->seid,
6324                                                    &bw_data, NULL);
6325         if (ret) {
6326                 dev_info(&pf->pdev->dev,
6327                          "VEB bw config failed, err %s aq_err %s\n",
6328                          i40e_stat_str(&pf->hw, ret),
6329                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6330                 goto out;
6331         }
6332
6333         /* Update the BW information */
6334         ret = i40e_veb_get_bw_info(veb);
6335         if (ret) {
6336                 dev_info(&pf->pdev->dev,
6337                          "Failed getting veb bw config, err %s aq_err %s\n",
6338                          i40e_stat_str(&pf->hw, ret),
6339                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6340         }
6341
6342 out:
6343         return ret;
6344 }
6345
6346 #ifdef CONFIG_I40E_DCB
6347 /**
6348  * i40e_dcb_reconfigure - Reconfigure all VEBs and VSIs
6349  * @pf: PF struct
6350  *
6351  * Reconfigure VEB/VSIs on a given PF; it is assumed that
6352  * the caller would've quiesce all the VSIs before calling
6353  * this function
6354  **/
6355 static void i40e_dcb_reconfigure(struct i40e_pf *pf)
6356 {
6357         u8 tc_map = 0;
6358         int ret;
6359         u8 v;
6360
6361         /* Enable the TCs available on PF to all VEBs */
6362         tc_map = i40e_pf_get_tc_map(pf);
6363         for (v = 0; v < I40E_MAX_VEB; v++) {
6364                 if (!pf->veb[v])
6365                         continue;
6366                 ret = i40e_veb_config_tc(pf->veb[v], tc_map);
6367                 if (ret) {
6368                         dev_info(&pf->pdev->dev,
6369                                  "Failed configuring TC for VEB seid=%d\n",
6370                                  pf->veb[v]->seid);
6371                         /* Will try to configure as many components */
6372                 }
6373         }
6374
6375         /* Update each VSI */
6376         for (v = 0; v < pf->num_alloc_vsi; v++) {
6377                 if (!pf->vsi[v])
6378                         continue;
6379
6380                 /* - Enable all TCs for the LAN VSI
6381                  * - For all others keep them at TC0 for now
6382                  */
6383                 if (v == pf->lan_vsi)
6384                         tc_map = i40e_pf_get_tc_map(pf);
6385                 else
6386                         tc_map = I40E_DEFAULT_TRAFFIC_CLASS;
6387
6388                 ret = i40e_vsi_config_tc(pf->vsi[v], tc_map);
6389                 if (ret) {
6390                         dev_info(&pf->pdev->dev,
6391                                  "Failed configuring TC for VSI seid=%d\n",
6392                                  pf->vsi[v]->seid);
6393                         /* Will try to configure as many components */
6394                 } else {
6395                         /* Re-configure VSI vectors based on updated TC map */
6396                         i40e_vsi_map_rings_to_vectors(pf->vsi[v]);
6397                         if (pf->vsi[v]->netdev)
6398                                 i40e_dcbnl_set_all(pf->vsi[v]);
6399                 }
6400         }
6401 }
6402
6403 /**
6404  * i40e_resume_port_tx - Resume port Tx
6405  * @pf: PF struct
6406  *
6407  * Resume a port's Tx and issue a PF reset in case of failure to
6408  * resume.
6409  **/
6410 static int i40e_resume_port_tx(struct i40e_pf *pf)
6411 {
6412         struct i40e_hw *hw = &pf->hw;
6413         int ret;
6414
6415         ret = i40e_aq_resume_port_tx(hw, NULL);
6416         if (ret) {
6417                 dev_info(&pf->pdev->dev,
6418                          "Resume Port Tx failed, err %s aq_err %s\n",
6419                           i40e_stat_str(&pf->hw, ret),
6420                           i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6421                 /* Schedule PF reset to recover */
6422                 set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
6423                 i40e_service_event_schedule(pf);
6424         }
6425
6426         return ret;
6427 }
6428
6429 /**
6430  * i40e_init_pf_dcb - Initialize DCB configuration
6431  * @pf: PF being configured
6432  *
6433  * Query the current DCB configuration and cache it
6434  * in the hardware structure
6435  **/
6436 static int i40e_init_pf_dcb(struct i40e_pf *pf)
6437 {
6438         struct i40e_hw *hw = &pf->hw;
6439         int err = 0;
6440
6441         /* Do not enable DCB for SW1 and SW2 images even if the FW is capable
6442          * Also do not enable DCBx if FW LLDP agent is disabled
6443          */
6444         if ((pf->hw_features & I40E_HW_NO_DCB_SUPPORT) ||
6445             (pf->flags & I40E_FLAG_DISABLE_FW_LLDP))
6446                 goto out;
6447
6448         /* Get the initial DCB configuration */
6449         err = i40e_init_dcb(hw);
6450         if (!err) {
6451                 /* Device/Function is not DCBX capable */
6452                 if ((!hw->func_caps.dcb) ||
6453                     (hw->dcbx_status == I40E_DCBX_STATUS_DISABLED)) {
6454                         dev_info(&pf->pdev->dev,
6455                                  "DCBX offload is not supported or is disabled for this PF.\n");
6456                 } else {
6457                         /* When status is not DISABLED then DCBX in FW */
6458                         pf->dcbx_cap = DCB_CAP_DCBX_LLD_MANAGED |
6459                                        DCB_CAP_DCBX_VER_IEEE;
6460
6461                         pf->flags |= I40E_FLAG_DCB_CAPABLE;
6462                         /* Enable DCB tagging only when more than one TC
6463                          * or explicitly disable if only one TC
6464                          */
6465                         if (i40e_dcb_get_num_tc(&hw->local_dcbx_config) > 1)
6466                                 pf->flags |= I40E_FLAG_DCB_ENABLED;
6467                         else
6468                                 pf->flags &= ~I40E_FLAG_DCB_ENABLED;
6469                         dev_dbg(&pf->pdev->dev,
6470                                 "DCBX offload is supported for this PF.\n");
6471                 }
6472         } else if (pf->hw.aq.asq_last_status == I40E_AQ_RC_EPERM) {
6473                 dev_info(&pf->pdev->dev, "FW LLDP disabled for this PF.\n");
6474                 pf->flags |= I40E_FLAG_DISABLE_FW_LLDP;
6475         } else {
6476                 dev_info(&pf->pdev->dev,
6477                          "Query for DCB configuration failed, err %s aq_err %s\n",
6478                          i40e_stat_str(&pf->hw, err),
6479                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6480         }
6481
6482 out:
6483         return err;
6484 }
6485 #endif /* CONFIG_I40E_DCB */
6486 #define SPEED_SIZE 14
6487 #define FC_SIZE 8
6488 /**
6489  * i40e_print_link_message - print link up or down
6490  * @vsi: the VSI for which link needs a message
6491  * @isup: true of link is up, false otherwise
6492  */
6493 void i40e_print_link_message(struct i40e_vsi *vsi, bool isup)
6494 {
6495         enum i40e_aq_link_speed new_speed;
6496         struct i40e_pf *pf = vsi->back;
6497         char *speed = "Unknown";
6498         char *fc = "Unknown";
6499         char *fec = "";
6500         char *req_fec = "";
6501         char *an = "";
6502
6503         new_speed = pf->hw.phy.link_info.link_speed;
6504
6505         if ((vsi->current_isup == isup) && (vsi->current_speed == new_speed))
6506                 return;
6507         vsi->current_isup = isup;
6508         vsi->current_speed = new_speed;
6509         if (!isup) {
6510                 netdev_info(vsi->netdev, "NIC Link is Down\n");
6511                 return;
6512         }
6513
6514         /* Warn user if link speed on NPAR enabled partition is not at
6515          * least 10GB
6516          */
6517         if (pf->hw.func_caps.npar_enable &&
6518             (pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_1GB ||
6519              pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_100MB))
6520                 netdev_warn(vsi->netdev,
6521                             "The partition detected link speed that is less than 10Gbps\n");
6522
6523         switch (pf->hw.phy.link_info.link_speed) {
6524         case I40E_LINK_SPEED_40GB:
6525                 speed = "40 G";
6526                 break;
6527         case I40E_LINK_SPEED_20GB:
6528                 speed = "20 G";
6529                 break;
6530         case I40E_LINK_SPEED_25GB:
6531                 speed = "25 G";
6532                 break;
6533         case I40E_LINK_SPEED_10GB:
6534                 speed = "10 G";
6535                 break;
6536         case I40E_LINK_SPEED_1GB:
6537                 speed = "1000 M";
6538                 break;
6539         case I40E_LINK_SPEED_100MB:
6540                 speed = "100 M";
6541                 break;
6542         default:
6543                 break;
6544         }
6545
6546         switch (pf->hw.fc.current_mode) {
6547         case I40E_FC_FULL:
6548                 fc = "RX/TX";
6549                 break;
6550         case I40E_FC_TX_PAUSE:
6551                 fc = "TX";
6552                 break;
6553         case I40E_FC_RX_PAUSE:
6554                 fc = "RX";
6555                 break;
6556         default:
6557                 fc = "None";
6558                 break;
6559         }
6560
6561         if (pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_25GB) {
6562                 req_fec = ", Requested FEC: None";
6563                 fec = ", FEC: None";
6564                 an = ", Autoneg: False";
6565
6566                 if (pf->hw.phy.link_info.an_info & I40E_AQ_AN_COMPLETED)
6567                         an = ", Autoneg: True";
6568
6569                 if (pf->hw.phy.link_info.fec_info &
6570                     I40E_AQ_CONFIG_FEC_KR_ENA)
6571                         fec = ", FEC: CL74 FC-FEC/BASE-R";
6572                 else if (pf->hw.phy.link_info.fec_info &
6573                          I40E_AQ_CONFIG_FEC_RS_ENA)
6574                         fec = ", FEC: CL108 RS-FEC";
6575
6576                 /* 'CL108 RS-FEC' should be displayed when RS is requested, or
6577                  * both RS and FC are requested
6578                  */
6579                 if (vsi->back->hw.phy.link_info.req_fec_info &
6580                     (I40E_AQ_REQUEST_FEC_KR | I40E_AQ_REQUEST_FEC_RS)) {
6581                         if (vsi->back->hw.phy.link_info.req_fec_info &
6582                             I40E_AQ_REQUEST_FEC_RS)
6583                                 req_fec = ", Requested FEC: CL108 RS-FEC";
6584                         else
6585                                 req_fec = ", Requested FEC: CL74 FC-FEC/BASE-R";
6586                 }
6587         }
6588
6589         netdev_info(vsi->netdev, "NIC Link is Up, %sbps Full Duplex%s%s%s, Flow Control: %s\n",
6590                     speed, req_fec, fec, an, fc);
6591 }
6592
6593 /**
6594  * i40e_up_complete - Finish the last steps of bringing up a connection
6595  * @vsi: the VSI being configured
6596  **/
6597 static int i40e_up_complete(struct i40e_vsi *vsi)
6598 {
6599         struct i40e_pf *pf = vsi->back;
6600         int err;
6601
6602         if (pf->flags & I40E_FLAG_MSIX_ENABLED)
6603                 i40e_vsi_configure_msix(vsi);
6604         else
6605                 i40e_configure_msi_and_legacy(vsi);
6606
6607         /* start rings */
6608         err = i40e_vsi_start_rings(vsi);
6609         if (err)
6610                 return err;
6611
6612         clear_bit(__I40E_VSI_DOWN, vsi->state);
6613         i40e_napi_enable_all(vsi);
6614         i40e_vsi_enable_irq(vsi);
6615
6616         if ((pf->hw.phy.link_info.link_info & I40E_AQ_LINK_UP) &&
6617             (vsi->netdev)) {
6618                 i40e_print_link_message(vsi, true);
6619                 netif_tx_start_all_queues(vsi->netdev);
6620                 netif_carrier_on(vsi->netdev);
6621         }
6622
6623         /* replay FDIR SB filters */
6624         if (vsi->type == I40E_VSI_FDIR) {
6625                 /* reset fd counters */
6626                 pf->fd_add_err = 0;
6627                 pf->fd_atr_cnt = 0;
6628                 i40e_fdir_filter_restore(vsi);
6629         }
6630
6631         /* On the next run of the service_task, notify any clients of the new
6632          * opened netdev
6633          */
6634         set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state);
6635         i40e_service_event_schedule(pf);
6636
6637         return 0;
6638 }
6639
6640 /**
6641  * i40e_vsi_reinit_locked - Reset the VSI
6642  * @vsi: the VSI being configured
6643  *
6644  * Rebuild the ring structs after some configuration
6645  * has changed, e.g. MTU size.
6646  **/
6647 static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi)
6648 {
6649         struct i40e_pf *pf = vsi->back;
6650
6651         WARN_ON(in_interrupt());
6652         while (test_and_set_bit(__I40E_CONFIG_BUSY, pf->state))
6653                 usleep_range(1000, 2000);
6654         i40e_down(vsi);
6655
6656         i40e_up(vsi);
6657         clear_bit(__I40E_CONFIG_BUSY, pf->state);
6658 }
6659
6660 /**
6661  * i40e_up - Bring the connection back up after being down
6662  * @vsi: the VSI being configured
6663  **/
6664 int i40e_up(struct i40e_vsi *vsi)
6665 {
6666         int err;
6667
6668         err = i40e_vsi_configure(vsi);
6669         if (!err)
6670                 err = i40e_up_complete(vsi);
6671
6672         return err;
6673 }
6674
6675 /**
6676  * i40e_force_link_state - Force the link status
6677  * @pf: board private structure
6678  * @is_up: whether the link state should be forced up or down
6679  **/
6680 static i40e_status i40e_force_link_state(struct i40e_pf *pf, bool is_up)
6681 {
6682         struct i40e_aq_get_phy_abilities_resp abilities;
6683         struct i40e_aq_set_phy_config config = {0};
6684         struct i40e_hw *hw = &pf->hw;
6685         i40e_status err;
6686         u64 mask;
6687         u8 speed;
6688
6689         /* Card might've been put in an unstable state by other drivers
6690          * and applications, which causes incorrect speed values being
6691          * set on startup. In order to clear speed registers, we call
6692          * get_phy_capabilities twice, once to get initial state of
6693          * available speeds, and once to get current PHY config.
6694          */
6695         err = i40e_aq_get_phy_capabilities(hw, false, true, &abilities,
6696                                            NULL);
6697         if (err) {
6698                 dev_err(&pf->pdev->dev,
6699                         "failed to get phy cap., ret =  %s last_status =  %s\n",
6700                         i40e_stat_str(hw, err),
6701                         i40e_aq_str(hw, hw->aq.asq_last_status));
6702                 return err;
6703         }
6704         speed = abilities.link_speed;
6705
6706         /* Get the current phy config */
6707         err = i40e_aq_get_phy_capabilities(hw, false, false, &abilities,
6708                                            NULL);
6709         if (err) {
6710                 dev_err(&pf->pdev->dev,
6711                         "failed to get phy cap., ret =  %s last_status =  %s\n",
6712                         i40e_stat_str(hw, err),
6713                         i40e_aq_str(hw, hw->aq.asq_last_status));
6714                 return err;
6715         }
6716
6717         /* If link needs to go up, but was not forced to go down,
6718          * and its speed values are OK, no need for a flap
6719          */
6720         if (is_up && abilities.phy_type != 0 && abilities.link_speed != 0)
6721                 return I40E_SUCCESS;
6722
6723         /* To force link we need to set bits for all supported PHY types,
6724          * but there are now more than 32, so we need to split the bitmap
6725          * across two fields.
6726          */
6727         mask = I40E_PHY_TYPES_BITMASK;
6728         config.phy_type = is_up ? cpu_to_le32((u32)(mask & 0xffffffff)) : 0;
6729         config.phy_type_ext = is_up ? (u8)((mask >> 32) & 0xff) : 0;
6730         /* Copy the old settings, except of phy_type */
6731         config.abilities = abilities.abilities;
6732         if (abilities.link_speed != 0)
6733                 config.link_speed = abilities.link_speed;
6734         else
6735                 config.link_speed = speed;
6736         config.eee_capability = abilities.eee_capability;
6737         config.eeer = abilities.eeer_val;
6738         config.low_power_ctrl = abilities.d3_lpan;
6739         config.fec_config = abilities.fec_cfg_curr_mod_ext_info &
6740                             I40E_AQ_PHY_FEC_CONFIG_MASK;
6741         err = i40e_aq_set_phy_config(hw, &config, NULL);
6742
6743         if (err) {
6744                 dev_err(&pf->pdev->dev,
6745                         "set phy config ret =  %s last_status =  %s\n",
6746                         i40e_stat_str(&pf->hw, err),
6747                         i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6748                 return err;
6749         }
6750
6751         /* Update the link info */
6752         err = i40e_update_link_info(hw);
6753         if (err) {
6754                 /* Wait a little bit (on 40G cards it sometimes takes a really
6755                  * long time for link to come back from the atomic reset)
6756                  * and try once more
6757                  */
6758                 msleep(1000);
6759                 i40e_update_link_info(hw);
6760         }
6761
6762         i40e_aq_set_link_restart_an(hw, true, NULL);
6763
6764         return I40E_SUCCESS;
6765 }
6766
6767 /**
6768  * i40e_down - Shutdown the connection processing
6769  * @vsi: the VSI being stopped
6770  **/
6771 void i40e_down(struct i40e_vsi *vsi)
6772 {
6773         int i;
6774
6775         /* It is assumed that the caller of this function
6776          * sets the vsi->state __I40E_VSI_DOWN bit.
6777          */
6778         if (vsi->netdev) {
6779                 netif_carrier_off(vsi->netdev);
6780                 netif_tx_disable(vsi->netdev);
6781         }
6782         i40e_vsi_disable_irq(vsi);
6783         i40e_vsi_stop_rings(vsi);
6784         if (vsi->type == I40E_VSI_MAIN &&
6785             vsi->back->flags & I40E_FLAG_LINK_DOWN_ON_CLOSE_ENABLED)
6786                 i40e_force_link_state(vsi->back, false);
6787         i40e_napi_disable_all(vsi);
6788
6789         for (i = 0; i < vsi->num_queue_pairs; i++) {
6790                 i40e_clean_tx_ring(vsi->tx_rings[i]);
6791                 if (i40e_enabled_xdp_vsi(vsi))
6792                         i40e_clean_tx_ring(vsi->xdp_rings[i]);
6793                 i40e_clean_rx_ring(vsi->rx_rings[i]);
6794         }
6795
6796 }
6797
6798 /**
6799  * i40e_validate_mqprio_qopt- validate queue mapping info
6800  * @vsi: the VSI being configured
6801  * @mqprio_qopt: queue parametrs
6802  **/
6803 static int i40e_validate_mqprio_qopt(struct i40e_vsi *vsi,
6804                                      struct tc_mqprio_qopt_offload *mqprio_qopt)
6805 {
6806         u64 sum_max_rate = 0;
6807         u64 max_rate = 0;
6808         int i;
6809
6810         if (mqprio_qopt->qopt.offset[0] != 0 ||
6811             mqprio_qopt->qopt.num_tc < 1 ||
6812             mqprio_qopt->qopt.num_tc > I40E_MAX_TRAFFIC_CLASS)
6813                 return -EINVAL;
6814         for (i = 0; ; i++) {
6815                 if (!mqprio_qopt->qopt.count[i])
6816                         return -EINVAL;
6817                 if (mqprio_qopt->min_rate[i]) {
6818                         dev_err(&vsi->back->pdev->dev,
6819                                 "Invalid min tx rate (greater than 0) specified\n");
6820                         return -EINVAL;
6821                 }
6822                 max_rate = mqprio_qopt->max_rate[i];
6823                 do_div(max_rate, I40E_BW_MBPS_DIVISOR);
6824                 sum_max_rate += max_rate;
6825
6826                 if (i >= mqprio_qopt->qopt.num_tc - 1)
6827                         break;
6828                 if (mqprio_qopt->qopt.offset[i + 1] !=
6829                     (mqprio_qopt->qopt.offset[i] + mqprio_qopt->qopt.count[i]))
6830                         return -EINVAL;
6831         }
6832         if (vsi->num_queue_pairs <
6833             (mqprio_qopt->qopt.offset[i] + mqprio_qopt->qopt.count[i])) {
6834                 dev_err(&vsi->back->pdev->dev,
6835                         "Failed to create traffic channel, insufficient number of queues.\n");
6836                 return -EINVAL;
6837         }
6838         if (sum_max_rate > i40e_get_link_speed(vsi)) {
6839                 dev_err(&vsi->back->pdev->dev,
6840                         "Invalid max tx rate specified\n");
6841                 return -EINVAL;
6842         }
6843         return 0;
6844 }
6845
6846 /**
6847  * i40e_vsi_set_default_tc_config - set default values for tc configuration
6848  * @vsi: the VSI being configured
6849  **/
6850 static void i40e_vsi_set_default_tc_config(struct i40e_vsi *vsi)
6851 {
6852         u16 qcount;
6853         int i;
6854
6855         /* Only TC0 is enabled */
6856         vsi->tc_config.numtc = 1;
6857         vsi->tc_config.enabled_tc = 1;
6858         qcount = min_t(int, vsi->alloc_queue_pairs,
6859                        i40e_pf_get_max_q_per_tc(vsi->back));
6860         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
6861                 /* For the TC that is not enabled set the offset to to default
6862                  * queue and allocate one queue for the given TC.
6863                  */
6864                 vsi->tc_config.tc_info[i].qoffset = 0;
6865                 if (i == 0)
6866                         vsi->tc_config.tc_info[i].qcount = qcount;
6867                 else
6868                         vsi->tc_config.tc_info[i].qcount = 1;
6869                 vsi->tc_config.tc_info[i].netdev_tc = 0;
6870         }
6871 }
6872
6873 /**
6874  * i40e_setup_tc - configure multiple traffic classes
6875  * @netdev: net device to configure
6876  * @type_data: tc offload data
6877  **/
6878 static int i40e_setup_tc(struct net_device *netdev, void *type_data)
6879 {
6880         struct tc_mqprio_qopt_offload *mqprio_qopt = type_data;
6881         struct i40e_netdev_priv *np = netdev_priv(netdev);
6882         struct i40e_vsi *vsi = np->vsi;
6883         struct i40e_pf *pf = vsi->back;
6884         u8 enabled_tc = 0, num_tc, hw;
6885         bool need_reset = false;
6886         int old_queue_pairs;
6887         int ret = -EINVAL;
6888         u16 mode;
6889         int i;
6890
6891         old_queue_pairs = vsi->num_queue_pairs;
6892         num_tc = mqprio_qopt->qopt.num_tc;
6893         hw = mqprio_qopt->qopt.hw;
6894         mode = mqprio_qopt->mode;
6895         if (!hw) {
6896                 pf->flags &= ~I40E_FLAG_TC_MQPRIO;
6897                 memcpy(&vsi->mqprio_qopt, mqprio_qopt, sizeof(*mqprio_qopt));
6898                 goto config_tc;
6899         }
6900
6901         /* Check if MFP enabled */
6902         if (pf->flags & I40E_FLAG_MFP_ENABLED) {
6903                 netdev_info(netdev,
6904                             "Configuring TC not supported in MFP mode\n");
6905                 return ret;
6906         }
6907         switch (mode) {
6908         case TC_MQPRIO_MODE_DCB:
6909                 pf->flags &= ~I40E_FLAG_TC_MQPRIO;
6910
6911                 /* Check if DCB enabled to continue */
6912                 if (!(pf->flags & I40E_FLAG_DCB_ENABLED)) {
6913                         netdev_info(netdev,
6914                                     "DCB is not enabled for adapter\n");
6915                         return ret;
6916                 }
6917
6918                 /* Check whether tc count is within enabled limit */
6919                 if (num_tc > i40e_pf_get_num_tc(pf)) {
6920                         netdev_info(netdev,
6921                                     "TC count greater than enabled on link for adapter\n");
6922                         return ret;
6923                 }
6924                 break;
6925         case TC_MQPRIO_MODE_CHANNEL:
6926                 if (pf->flags & I40E_FLAG_DCB_ENABLED) {
6927                         netdev_info(netdev,
6928                                     "Full offload of TC Mqprio options is not supported when DCB is enabled\n");
6929                         return ret;
6930                 }
6931                 if (!(pf->flags & I40E_FLAG_MSIX_ENABLED))
6932                         return ret;
6933                 ret = i40e_validate_mqprio_qopt(vsi, mqprio_qopt);
6934                 if (ret)
6935                         return ret;
6936                 memcpy(&vsi->mqprio_qopt, mqprio_qopt,
6937                        sizeof(*mqprio_qopt));
6938                 pf->flags |= I40E_FLAG_TC_MQPRIO;
6939                 pf->flags &= ~I40E_FLAG_DCB_ENABLED;
6940                 break;
6941         default:
6942                 return -EINVAL;
6943         }
6944
6945 config_tc:
6946         /* Generate TC map for number of tc requested */
6947         for (i = 0; i < num_tc; i++)
6948                 enabled_tc |= BIT(i);
6949
6950         /* Requesting same TC configuration as already enabled */
6951         if (enabled_tc == vsi->tc_config.enabled_tc &&
6952             mode != TC_MQPRIO_MODE_CHANNEL)
6953                 return 0;
6954
6955         /* Quiesce VSI queues */
6956         i40e_quiesce_vsi(vsi);
6957
6958         if (!hw && !(pf->flags & I40E_FLAG_TC_MQPRIO))
6959                 i40e_remove_queue_channels(vsi);
6960
6961         /* Configure VSI for enabled TCs */
6962         ret = i40e_vsi_config_tc(vsi, enabled_tc);
6963         if (ret) {
6964                 netdev_info(netdev, "Failed configuring TC for VSI seid=%d\n",
6965                             vsi->seid);
6966                 need_reset = true;
6967                 goto exit;
6968         }
6969
6970         if (pf->flags & I40E_FLAG_TC_MQPRIO) {
6971                 if (vsi->mqprio_qopt.max_rate[0]) {
6972                         u64 max_tx_rate = i40e_bw_bytes_to_mbits(vsi,
6973                                                   vsi->mqprio_qopt.max_rate[0]);
6974
6975                         ret = i40e_set_bw_limit(vsi, vsi->seid, max_tx_rate);
6976                         if (!ret) {
6977                                 u64 credits = max_tx_rate;
6978
6979                                 do_div(credits, I40E_BW_CREDIT_DIVISOR);
6980                                 dev_dbg(&vsi->back->pdev->dev,
6981                                         "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n",
6982                                         max_tx_rate,
6983                                         credits,
6984                                         vsi->seid);
6985                         } else {
6986                                 need_reset = true;
6987                                 goto exit;
6988                         }
6989                 }
6990                 ret = i40e_configure_queue_channels(vsi);
6991                 if (ret) {
6992                         vsi->num_queue_pairs = old_queue_pairs;
6993                         netdev_info(netdev,
6994                                     "Failed configuring queue channels\n");
6995                         need_reset = true;
6996                         goto exit;
6997                 }
6998         }
6999
7000 exit:
7001         /* Reset the configuration data to defaults, only TC0 is enabled */
7002         if (need_reset) {
7003                 i40e_vsi_set_default_tc_config(vsi);
7004                 need_reset = false;
7005         }
7006
7007         /* Unquiesce VSI */
7008         i40e_unquiesce_vsi(vsi);
7009         return ret;
7010 }
7011
7012 /**
7013  * i40e_set_cld_element - sets cloud filter element data
7014  * @filter: cloud filter rule
7015  * @cld: ptr to cloud filter element data
7016  *
7017  * This is helper function to copy data into cloud filter element
7018  **/
7019 static inline void
7020 i40e_set_cld_element(struct i40e_cloud_filter *filter,
7021                      struct i40e_aqc_cloud_filters_element_data *cld)
7022 {
7023         int i, j;
7024         u32 ipa;
7025
7026         memset(cld, 0, sizeof(*cld));
7027         ether_addr_copy(cld->outer_mac, filter->dst_mac);
7028         ether_addr_copy(cld->inner_mac, filter->src_mac);
7029
7030         if (filter->n_proto != ETH_P_IP && filter->n_proto != ETH_P_IPV6)
7031                 return;
7032
7033         if (filter->n_proto == ETH_P_IPV6) {
7034 #define IPV6_MAX_INDEX  (ARRAY_SIZE(filter->dst_ipv6) - 1)
7035                 for (i = 0, j = 0; i < ARRAY_SIZE(filter->dst_ipv6);
7036                      i++, j += 2) {
7037                         ipa = be32_to_cpu(filter->dst_ipv6[IPV6_MAX_INDEX - i]);
7038                         ipa = cpu_to_le32(ipa);
7039                         memcpy(&cld->ipaddr.raw_v6.data[j], &ipa, sizeof(ipa));
7040                 }
7041         } else {
7042                 ipa = be32_to_cpu(filter->dst_ipv4);
7043                 memcpy(&cld->ipaddr.v4.data, &ipa, sizeof(ipa));
7044         }
7045
7046         cld->inner_vlan = cpu_to_le16(ntohs(filter->vlan_id));
7047
7048         /* tenant_id is not supported by FW now, once the support is enabled
7049          * fill the cld->tenant_id with cpu_to_le32(filter->tenant_id)
7050          */
7051         if (filter->tenant_id)
7052                 return;
7053 }
7054
7055 /**
7056  * i40e_add_del_cloud_filter - Add/del cloud filter
7057  * @vsi: pointer to VSI
7058  * @filter: cloud filter rule
7059  * @add: if true, add, if false, delete
7060  *
7061  * Add or delete a cloud filter for a specific flow spec.
7062  * Returns 0 if the filter were successfully added.
7063  **/
7064 int i40e_add_del_cloud_filter(struct i40e_vsi *vsi,
7065                               struct i40e_cloud_filter *filter, bool add)
7066 {
7067         struct i40e_aqc_cloud_filters_element_data cld_filter;
7068         struct i40e_pf *pf = vsi->back;
7069         int ret;
7070         static const u16 flag_table[128] = {
7071                 [I40E_CLOUD_FILTER_FLAGS_OMAC]  =
7072                         I40E_AQC_ADD_CLOUD_FILTER_OMAC,
7073                 [I40E_CLOUD_FILTER_FLAGS_IMAC]  =
7074                         I40E_AQC_ADD_CLOUD_FILTER_IMAC,
7075                 [I40E_CLOUD_FILTER_FLAGS_IMAC_IVLAN]  =
7076                         I40E_AQC_ADD_CLOUD_FILTER_IMAC_IVLAN,
7077                 [I40E_CLOUD_FILTER_FLAGS_IMAC_TEN_ID] =
7078                         I40E_AQC_ADD_CLOUD_FILTER_IMAC_TEN_ID,
7079                 [I40E_CLOUD_FILTER_FLAGS_OMAC_TEN_ID_IMAC] =
7080                         I40E_AQC_ADD_CLOUD_FILTER_OMAC_TEN_ID_IMAC,
7081                 [I40E_CLOUD_FILTER_FLAGS_IMAC_IVLAN_TEN_ID] =
7082                         I40E_AQC_ADD_CLOUD_FILTER_IMAC_IVLAN_TEN_ID,
7083                 [I40E_CLOUD_FILTER_FLAGS_IIP] =
7084                         I40E_AQC_ADD_CLOUD_FILTER_IIP,
7085         };
7086
7087         if (filter->flags >= ARRAY_SIZE(flag_table))
7088                 return I40E_ERR_CONFIG;
7089
7090         memset(&cld_filter, 0, sizeof(cld_filter));
7091
7092         /* copy element needed to add cloud filter from filter */
7093         i40e_set_cld_element(filter, &cld_filter);
7094
7095         if (filter->tunnel_type != I40E_CLOUD_TNL_TYPE_NONE)
7096                 cld_filter.flags = cpu_to_le16(filter->tunnel_type <<
7097                                              I40E_AQC_ADD_CLOUD_TNL_TYPE_SHIFT);
7098
7099         if (filter->n_proto == ETH_P_IPV6)
7100                 cld_filter.flags |= cpu_to_le16(flag_table[filter->flags] |
7101                                                 I40E_AQC_ADD_CLOUD_FLAGS_IPV6);
7102         else
7103                 cld_filter.flags |= cpu_to_le16(flag_table[filter->flags] |
7104                                                 I40E_AQC_ADD_CLOUD_FLAGS_IPV4);
7105
7106         if (add)
7107                 ret = i40e_aq_add_cloud_filters(&pf->hw, filter->seid,
7108                                                 &cld_filter, 1);
7109         else
7110                 ret = i40e_aq_rem_cloud_filters(&pf->hw, filter->seid,
7111                                                 &cld_filter, 1);
7112         if (ret)
7113                 dev_dbg(&pf->pdev->dev,
7114                         "Failed to %s cloud filter using l4 port %u, err %d aq_err %d\n",
7115                         add ? "add" : "delete", filter->dst_port, ret,
7116                         pf->hw.aq.asq_last_status);
7117         else
7118                 dev_info(&pf->pdev->dev,
7119                          "%s cloud filter for VSI: %d\n",
7120                          add ? "Added" : "Deleted", filter->seid);
7121         return ret;
7122 }
7123
7124 /**
7125  * i40e_add_del_cloud_filter_big_buf - Add/del cloud filter using big_buf
7126  * @vsi: pointer to VSI
7127  * @filter: cloud filter rule
7128  * @add: if true, add, if false, delete
7129  *
7130  * Add or delete a cloud filter for a specific flow spec using big buffer.
7131  * Returns 0 if the filter were successfully added.
7132  **/
7133 int i40e_add_del_cloud_filter_big_buf(struct i40e_vsi *vsi,
7134                                       struct i40e_cloud_filter *filter,
7135                                       bool add)
7136 {
7137         struct i40e_aqc_cloud_filters_element_bb cld_filter;
7138         struct i40e_pf *pf = vsi->back;
7139         int ret;
7140
7141         /* Both (src/dst) valid mac_addr are not supported */
7142         if ((is_valid_ether_addr(filter->dst_mac) &&
7143              is_valid_ether_addr(filter->src_mac)) ||
7144             (is_multicast_ether_addr(filter->dst_mac) &&
7145              is_multicast_ether_addr(filter->src_mac)))
7146                 return -EOPNOTSUPP;
7147
7148         /* Big buffer cloud filter needs 'L4 port' to be non-zero. Also, UDP
7149          * ports are not supported via big buffer now.
7150          */
7151         if (!filter->dst_port || filter->ip_proto == IPPROTO_UDP)
7152                 return -EOPNOTSUPP;
7153
7154         /* adding filter using src_port/src_ip is not supported at this stage */
7155         if (filter->src_port ||
7156             (filter->src_ipv4 && filter->n_proto != ETH_P_IPV6) ||
7157             !ipv6_addr_any(&filter->ip.v6.src_ip6))
7158                 return -EOPNOTSUPP;
7159
7160         memset(&cld_filter, 0, sizeof(cld_filter));
7161
7162         /* copy element needed to add cloud filter from filter */
7163         i40e_set_cld_element(filter, &cld_filter.element);
7164
7165         if (is_valid_ether_addr(filter->dst_mac) ||
7166             is_valid_ether_addr(filter->src_mac) ||
7167             is_multicast_ether_addr(filter->dst_mac) ||
7168             is_multicast_ether_addr(filter->src_mac)) {
7169                 /* MAC + IP : unsupported mode */
7170                 if (filter->dst_ipv4)
7171                         return -EOPNOTSUPP;
7172
7173                 /* since we validated that L4 port must be valid before
7174                  * we get here, start with respective "flags" value
7175                  * and update if vlan is present or not
7176                  */
7177                 cld_filter.element.flags =
7178                         cpu_to_le16(I40E_AQC_ADD_CLOUD_FILTER_MAC_PORT);
7179
7180                 if (filter->vlan_id) {
7181                         cld_filter.element.flags =
7182                         cpu_to_le16(I40E_AQC_ADD_CLOUD_FILTER_MAC_VLAN_PORT);
7183                 }
7184
7185         } else if ((filter->dst_ipv4 && filter->n_proto != ETH_P_IPV6) ||
7186                    !ipv6_addr_any(&filter->ip.v6.dst_ip6)) {
7187                 cld_filter.element.flags =
7188                                 cpu_to_le16(I40E_AQC_ADD_CLOUD_FILTER_IP_PORT);
7189                 if (filter->n_proto == ETH_P_IPV6)
7190                         cld_filter.element.flags |=
7191                                 cpu_to_le16(I40E_AQC_ADD_CLOUD_FLAGS_IPV6);
7192                 else
7193                         cld_filter.element.flags |=
7194                                 cpu_to_le16(I40E_AQC_ADD_CLOUD_FLAGS_IPV4);
7195         } else {
7196                 dev_err(&pf->pdev->dev,
7197                         "either mac or ip has to be valid for cloud filter\n");
7198                 return -EINVAL;
7199         }
7200
7201         /* Now copy L4 port in Byte 6..7 in general fields */
7202         cld_filter.general_fields[I40E_AQC_ADD_CLOUD_FV_FLU_0X16_WORD0] =
7203                                                 be16_to_cpu(filter->dst_port);
7204
7205         if (add) {
7206                 /* Validate current device switch mode, change if necessary */
7207                 ret = i40e_validate_and_set_switch_mode(vsi);
7208                 if (ret) {
7209                         dev_err(&pf->pdev->dev,
7210                                 "failed to set switch mode, ret %d\n",
7211                                 ret);
7212                         return ret;
7213                 }
7214
7215                 ret = i40e_aq_add_cloud_filters_bb(&pf->hw, filter->seid,
7216                                                    &cld_filter, 1);
7217         } else {
7218                 ret = i40e_aq_rem_cloud_filters_bb(&pf->hw, filter->seid,
7219                                                    &cld_filter, 1);
7220         }
7221
7222         if (ret)
7223                 dev_dbg(&pf->pdev->dev,
7224                         "Failed to %s cloud filter(big buffer) err %d aq_err %d\n",
7225                         add ? "add" : "delete", ret, pf->hw.aq.asq_last_status);
7226         else
7227                 dev_info(&pf->pdev->dev,
7228                          "%s cloud filter for VSI: %d, L4 port: %d\n",
7229                          add ? "add" : "delete", filter->seid,
7230                          ntohs(filter->dst_port));
7231         return ret;
7232 }
7233
7234 /**
7235  * i40e_parse_cls_flower - Parse tc flower filters provided by kernel
7236  * @vsi: Pointer to VSI
7237  * @cls_flower: Pointer to struct tc_cls_flower_offload
7238  * @filter: Pointer to cloud filter structure
7239  *
7240  **/
7241 static int i40e_parse_cls_flower(struct i40e_vsi *vsi,
7242                                  struct tc_cls_flower_offload *f,
7243                                  struct i40e_cloud_filter *filter)
7244 {
7245         u16 n_proto_mask = 0, n_proto_key = 0, addr_type = 0;
7246         struct i40e_pf *pf = vsi->back;
7247         u8 field_flags = 0;
7248
7249         if (f->dissector->used_keys &
7250             ~(BIT(FLOW_DISSECTOR_KEY_CONTROL) |
7251               BIT(FLOW_DISSECTOR_KEY_BASIC) |
7252               BIT(FLOW_DISSECTOR_KEY_ETH_ADDRS) |
7253               BIT(FLOW_DISSECTOR_KEY_VLAN) |
7254               BIT(FLOW_DISSECTOR_KEY_IPV4_ADDRS) |
7255               BIT(FLOW_DISSECTOR_KEY_IPV6_ADDRS) |
7256               BIT(FLOW_DISSECTOR_KEY_PORTS) |
7257               BIT(FLOW_DISSECTOR_KEY_ENC_KEYID))) {
7258                 dev_err(&pf->pdev->dev, "Unsupported key used: 0x%x\n",
7259                         f->dissector->used_keys);
7260                 return -EOPNOTSUPP;
7261         }
7262
7263         if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_ENC_KEYID)) {
7264                 struct flow_dissector_key_keyid *key =
7265                         skb_flow_dissector_target(f->dissector,
7266                                                   FLOW_DISSECTOR_KEY_ENC_KEYID,
7267                                                   f->key);
7268
7269                 struct flow_dissector_key_keyid *mask =
7270                         skb_flow_dissector_target(f->dissector,
7271                                                   FLOW_DISSECTOR_KEY_ENC_KEYID,
7272                                                   f->mask);
7273
7274                 if (mask->keyid != 0)
7275                         field_flags |= I40E_CLOUD_FIELD_TEN_ID;
7276
7277                 filter->tenant_id = be32_to_cpu(key->keyid);
7278         }
7279
7280         if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_BASIC)) {
7281                 struct flow_dissector_key_basic *key =
7282                         skb_flow_dissector_target(f->dissector,
7283                                                   FLOW_DISSECTOR_KEY_BASIC,
7284                                                   f->key);
7285
7286                 struct flow_dissector_key_basic *mask =
7287                         skb_flow_dissector_target(f->dissector,
7288                                                   FLOW_DISSECTOR_KEY_BASIC,
7289                                                   f->mask);
7290
7291                 n_proto_key = ntohs(key->n_proto);
7292                 n_proto_mask = ntohs(mask->n_proto);
7293
7294                 if (n_proto_key == ETH_P_ALL) {
7295                         n_proto_key = 0;
7296                         n_proto_mask = 0;
7297                 }
7298                 filter->n_proto = n_proto_key & n_proto_mask;
7299                 filter->ip_proto = key->ip_proto;
7300         }
7301
7302         if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_ETH_ADDRS)) {
7303                 struct flow_dissector_key_eth_addrs *key =
7304                         skb_flow_dissector_target(f->dissector,
7305                                                   FLOW_DISSECTOR_KEY_ETH_ADDRS,
7306                                                   f->key);
7307
7308                 struct flow_dissector_key_eth_addrs *mask =
7309                         skb_flow_dissector_target(f->dissector,
7310                                                   FLOW_DISSECTOR_KEY_ETH_ADDRS,
7311                                                   f->mask);
7312
7313                 /* use is_broadcast and is_zero to check for all 0xf or 0 */
7314                 if (!is_zero_ether_addr(mask->dst)) {
7315                         if (is_broadcast_ether_addr(mask->dst)) {
7316                                 field_flags |= I40E_CLOUD_FIELD_OMAC;
7317                         } else {
7318                                 dev_err(&pf->pdev->dev, "Bad ether dest mask %pM\n",
7319                                         mask->dst);
7320                                 return I40E_ERR_CONFIG;
7321                         }
7322                 }
7323
7324                 if (!is_zero_ether_addr(mask->src)) {
7325                         if (is_broadcast_ether_addr(mask->src)) {
7326                                 field_flags |= I40E_CLOUD_FIELD_IMAC;
7327                         } else {
7328                                 dev_err(&pf->pdev->dev, "Bad ether src mask %pM\n",
7329                                         mask->src);
7330                                 return I40E_ERR_CONFIG;
7331                         }
7332                 }
7333                 ether_addr_copy(filter->dst_mac, key->dst);
7334                 ether_addr_copy(filter->src_mac, key->src);
7335         }
7336
7337         if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_VLAN)) {
7338                 struct flow_dissector_key_vlan *key =
7339                         skb_flow_dissector_target(f->dissector,
7340                                                   FLOW_DISSECTOR_KEY_VLAN,
7341                                                   f->key);
7342                 struct flow_dissector_key_vlan *mask =
7343                         skb_flow_dissector_target(f->dissector,
7344                                                   FLOW_DISSECTOR_KEY_VLAN,
7345                                                   f->mask);
7346
7347                 if (mask->vlan_id) {
7348                         if (mask->vlan_id == VLAN_VID_MASK) {
7349                                 field_flags |= I40E_CLOUD_FIELD_IVLAN;
7350
7351                         } else {
7352                                 dev_err(&pf->pdev->dev, "Bad vlan mask 0x%04x\n",
7353                                         mask->vlan_id);
7354                                 return I40E_ERR_CONFIG;
7355                         }
7356                 }
7357
7358                 filter->vlan_id = cpu_to_be16(key->vlan_id);
7359         }
7360
7361         if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_CONTROL)) {
7362                 struct flow_dissector_key_control *key =
7363                         skb_flow_dissector_target(f->dissector,
7364                                                   FLOW_DISSECTOR_KEY_CONTROL,
7365                                                   f->key);
7366
7367                 addr_type = key->addr_type;
7368         }
7369
7370         if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) {
7371                 struct flow_dissector_key_ipv4_addrs *key =
7372                         skb_flow_dissector_target(f->dissector,
7373                                                   FLOW_DISSECTOR_KEY_IPV4_ADDRS,
7374                                                   f->key);
7375                 struct flow_dissector_key_ipv4_addrs *mask =
7376                         skb_flow_dissector_target(f->dissector,
7377                                                   FLOW_DISSECTOR_KEY_IPV4_ADDRS,
7378                                                   f->mask);
7379
7380                 if (mask->dst) {
7381                         if (mask->dst == cpu_to_be32(0xffffffff)) {
7382                                 field_flags |= I40E_CLOUD_FIELD_IIP;
7383                         } else {
7384                                 dev_err(&pf->pdev->dev, "Bad ip dst mask %pI4b\n",
7385                                         &mask->dst);
7386                                 return I40E_ERR_CONFIG;
7387                         }
7388                 }
7389
7390                 if (mask->src) {
7391                         if (mask->src == cpu_to_be32(0xffffffff)) {
7392                                 field_flags |= I40E_CLOUD_FIELD_IIP;
7393                         } else {
7394                                 dev_err(&pf->pdev->dev, "Bad ip src mask %pI4b\n",
7395                                         &mask->src);
7396                                 return I40E_ERR_CONFIG;
7397                         }
7398                 }
7399
7400                 if (field_flags & I40E_CLOUD_FIELD_TEN_ID) {
7401                         dev_err(&pf->pdev->dev, "Tenant id not allowed for ip filter\n");
7402                         return I40E_ERR_CONFIG;
7403                 }
7404                 filter->dst_ipv4 = key->dst;
7405                 filter->src_ipv4 = key->src;
7406         }
7407
7408         if (addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) {
7409                 struct flow_dissector_key_ipv6_addrs *key =
7410                         skb_flow_dissector_target(f->dissector,
7411                                                   FLOW_DISSECTOR_KEY_IPV6_ADDRS,
7412                                                   f->key);
7413                 struct flow_dissector_key_ipv6_addrs *mask =
7414                         skb_flow_dissector_target(f->dissector,
7415                                                   FLOW_DISSECTOR_KEY_IPV6_ADDRS,
7416                                                   f->mask);
7417
7418                 /* src and dest IPV6 address should not be LOOPBACK
7419                  * (0:0:0:0:0:0:0:1), which can be represented as ::1
7420                  */
7421                 if (ipv6_addr_loopback(&key->dst) ||
7422                     ipv6_addr_loopback(&key->src)) {
7423                         dev_err(&pf->pdev->dev,
7424                                 "Bad ipv6, addr is LOOPBACK\n");
7425                         return I40E_ERR_CONFIG;
7426                 }
7427                 if (!ipv6_addr_any(&mask->dst) || !ipv6_addr_any(&mask->src))
7428                         field_flags |= I40E_CLOUD_FIELD_IIP;
7429
7430                 memcpy(&filter->src_ipv6, &key->src.s6_addr32,
7431                        sizeof(filter->src_ipv6));
7432                 memcpy(&filter->dst_ipv6, &key->dst.s6_addr32,
7433                        sizeof(filter->dst_ipv6));
7434         }
7435
7436         if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_PORTS)) {
7437                 struct flow_dissector_key_ports *key =
7438                         skb_flow_dissector_target(f->dissector,
7439                                                   FLOW_DISSECTOR_KEY_PORTS,
7440                                                   f->key);
7441                 struct flow_dissector_key_ports *mask =
7442                         skb_flow_dissector_target(f->dissector,
7443                                                   FLOW_DISSECTOR_KEY_PORTS,
7444                                                   f->mask);
7445
7446                 if (mask->src) {
7447                         if (mask->src == cpu_to_be16(0xffff)) {
7448                                 field_flags |= I40E_CLOUD_FIELD_IIP;
7449                         } else {
7450                                 dev_err(&pf->pdev->dev, "Bad src port mask 0x%04x\n",
7451                                         be16_to_cpu(mask->src));
7452                                 return I40E_ERR_CONFIG;
7453                         }
7454                 }
7455
7456                 if (mask->dst) {
7457                         if (mask->dst == cpu_to_be16(0xffff)) {
7458                                 field_flags |= I40E_CLOUD_FIELD_IIP;
7459                         } else {
7460                                 dev_err(&pf->pdev->dev, "Bad dst port mask 0x%04x\n",
7461                                         be16_to_cpu(mask->dst));
7462                                 return I40E_ERR_CONFIG;
7463                         }
7464                 }
7465
7466                 filter->dst_port = key->dst;
7467                 filter->src_port = key->src;
7468
7469                 switch (filter->ip_proto) {
7470                 case IPPROTO_TCP:
7471                 case IPPROTO_UDP:
7472                         break;
7473                 default:
7474                         dev_err(&pf->pdev->dev,
7475                                 "Only UDP and TCP transport are supported\n");
7476                         return -EINVAL;
7477                 }
7478         }
7479         filter->flags = field_flags;
7480         return 0;
7481 }
7482
7483 /**
7484  * i40e_handle_tclass: Forward to a traffic class on the device
7485  * @vsi: Pointer to VSI
7486  * @tc: traffic class index on the device
7487  * @filter: Pointer to cloud filter structure
7488  *
7489  **/
7490 static int i40e_handle_tclass(struct i40e_vsi *vsi, u32 tc,
7491                               struct i40e_cloud_filter *filter)
7492 {
7493         struct i40e_channel *ch, *ch_tmp;
7494
7495         /* direct to a traffic class on the same device */
7496         if (tc == 0) {
7497                 filter->seid = vsi->seid;
7498                 return 0;
7499         } else if (vsi->tc_config.enabled_tc & BIT(tc)) {
7500                 if (!filter->dst_port) {
7501                         dev_err(&vsi->back->pdev->dev,
7502                                 "Specify destination port to direct to traffic class that is not default\n");
7503                         return -EINVAL;
7504                 }
7505                 if (list_empty(&vsi->ch_list))
7506                         return -EINVAL;
7507                 list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list,
7508                                          list) {
7509                         if (ch->seid == vsi->tc_seid_map[tc])
7510                                 filter->seid = ch->seid;
7511                 }
7512                 return 0;
7513         }
7514         dev_err(&vsi->back->pdev->dev, "TC is not enabled\n");
7515         return -EINVAL;
7516 }
7517
7518 /**
7519  * i40e_configure_clsflower - Configure tc flower filters
7520  * @vsi: Pointer to VSI
7521  * @cls_flower: Pointer to struct tc_cls_flower_offload
7522  *
7523  **/
7524 static int i40e_configure_clsflower(struct i40e_vsi *vsi,
7525                                     struct tc_cls_flower_offload *cls_flower)
7526 {
7527         int tc = tc_classid_to_hwtc(vsi->netdev, cls_flower->classid);
7528         struct i40e_cloud_filter *filter = NULL;
7529         struct i40e_pf *pf = vsi->back;
7530         int err = 0;
7531
7532         if (tc < 0) {
7533                 dev_err(&vsi->back->pdev->dev, "Invalid traffic class\n");
7534                 return -EOPNOTSUPP;
7535         }
7536
7537         if (!tc) {
7538                 dev_err(&pf->pdev->dev, "Unable to add filter because of invalid destination");
7539                 return -EINVAL;
7540         }
7541
7542         if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state) ||
7543             test_bit(__I40E_RESET_INTR_RECEIVED, pf->state))
7544                 return -EBUSY;
7545
7546         if (pf->fdir_pf_active_filters ||
7547             (!hlist_empty(&pf->fdir_filter_list))) {
7548                 dev_err(&vsi->back->pdev->dev,
7549                         "Flow Director Sideband filters exists, turn ntuple off to configure cloud filters\n");
7550                 return -EINVAL;
7551         }
7552
7553         if (vsi->back->flags & I40E_FLAG_FD_SB_ENABLED) {
7554                 dev_err(&vsi->back->pdev->dev,
7555                         "Disable Flow Director Sideband, configuring Cloud filters via tc-flower\n");
7556                 vsi->back->flags &= ~I40E_FLAG_FD_SB_ENABLED;
7557                 vsi->back->flags |= I40E_FLAG_FD_SB_TO_CLOUD_FILTER;
7558         }
7559
7560         filter = kzalloc(sizeof(*filter), GFP_KERNEL);
7561         if (!filter)
7562                 return -ENOMEM;
7563
7564         filter->cookie = cls_flower->cookie;
7565
7566         err = i40e_parse_cls_flower(vsi, cls_flower, filter);
7567         if (err < 0)
7568                 goto err;
7569
7570         err = i40e_handle_tclass(vsi, tc, filter);
7571         if (err < 0)
7572                 goto err;
7573
7574         /* Add cloud filter */
7575         if (filter->dst_port)
7576                 err = i40e_add_del_cloud_filter_big_buf(vsi, filter, true);
7577         else
7578                 err = i40e_add_del_cloud_filter(vsi, filter, true);
7579
7580         if (err) {
7581                 dev_err(&pf->pdev->dev, "Failed to add cloud filter, err %d\n",
7582                         err);
7583                 goto err;
7584         }
7585
7586         /* add filter to the ordered list */
7587         INIT_HLIST_NODE(&filter->cloud_node);
7588
7589         hlist_add_head(&filter->cloud_node, &pf->cloud_filter_list);
7590
7591         pf->num_cloud_filters++;
7592
7593         return err;
7594 err:
7595         kfree(filter);
7596         return err;
7597 }
7598
7599 /**
7600  * i40e_find_cloud_filter - Find the could filter in the list
7601  * @vsi: Pointer to VSI
7602  * @cookie: filter specific cookie
7603  *
7604  **/
7605 static struct i40e_cloud_filter *i40e_find_cloud_filter(struct i40e_vsi *vsi,
7606                                                         unsigned long *cookie)
7607 {
7608         struct i40e_cloud_filter *filter = NULL;
7609         struct hlist_node *node2;
7610
7611         hlist_for_each_entry_safe(filter, node2,
7612                                   &vsi->back->cloud_filter_list, cloud_node)
7613                 if (!memcmp(cookie, &filter->cookie, sizeof(filter->cookie)))
7614                         return filter;
7615         return NULL;
7616 }
7617
7618 /**
7619  * i40e_delete_clsflower - Remove tc flower filters
7620  * @vsi: Pointer to VSI
7621  * @cls_flower: Pointer to struct tc_cls_flower_offload
7622  *
7623  **/
7624 static int i40e_delete_clsflower(struct i40e_vsi *vsi,
7625                                  struct tc_cls_flower_offload *cls_flower)
7626 {
7627         struct i40e_cloud_filter *filter = NULL;
7628         struct i40e_pf *pf = vsi->back;
7629         int err = 0;
7630
7631         filter = i40e_find_cloud_filter(vsi, &cls_flower->cookie);
7632
7633         if (!filter)
7634                 return -EINVAL;
7635
7636         hash_del(&filter->cloud_node);
7637
7638         if (filter->dst_port)
7639                 err = i40e_add_del_cloud_filter_big_buf(vsi, filter, false);
7640         else
7641                 err = i40e_add_del_cloud_filter(vsi, filter, false);
7642
7643         kfree(filter);
7644         if (err) {
7645                 dev_err(&pf->pdev->dev,
7646                         "Failed to delete cloud filter, err %s\n",
7647                         i40e_stat_str(&pf->hw, err));
7648                 return i40e_aq_rc_to_posix(err, pf->hw.aq.asq_last_status);
7649         }
7650
7651         pf->num_cloud_filters--;
7652         if (!pf->num_cloud_filters)
7653                 if ((pf->flags & I40E_FLAG_FD_SB_TO_CLOUD_FILTER) &&
7654                     !(pf->flags & I40E_FLAG_FD_SB_INACTIVE)) {
7655                         pf->flags |= I40E_FLAG_FD_SB_ENABLED;
7656                         pf->flags &= ~I40E_FLAG_FD_SB_TO_CLOUD_FILTER;
7657                         pf->flags &= ~I40E_FLAG_FD_SB_INACTIVE;
7658                 }
7659         return 0;
7660 }
7661
7662 /**
7663  * i40e_setup_tc_cls_flower - flower classifier offloads
7664  * @netdev: net device to configure
7665  * @type_data: offload data
7666  **/
7667 static int i40e_setup_tc_cls_flower(struct i40e_netdev_priv *np,
7668                                     struct tc_cls_flower_offload *cls_flower)
7669 {
7670         struct i40e_vsi *vsi = np->vsi;
7671
7672         switch (cls_flower->command) {
7673         case TC_CLSFLOWER_REPLACE:
7674                 return i40e_configure_clsflower(vsi, cls_flower);
7675         case TC_CLSFLOWER_DESTROY:
7676                 return i40e_delete_clsflower(vsi, cls_flower);
7677         case TC_CLSFLOWER_STATS:
7678                 return -EOPNOTSUPP;
7679         default:
7680                 return -EOPNOTSUPP;
7681         }
7682 }
7683
7684 static int i40e_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
7685                                   void *cb_priv)
7686 {
7687         struct i40e_netdev_priv *np = cb_priv;
7688
7689         if (!tc_cls_can_offload_and_chain0(np->vsi->netdev, type_data))
7690                 return -EOPNOTSUPP;
7691
7692         switch (type) {
7693         case TC_SETUP_CLSFLOWER:
7694                 return i40e_setup_tc_cls_flower(np, type_data);
7695
7696         default:
7697                 return -EOPNOTSUPP;
7698         }
7699 }
7700
7701 static int i40e_setup_tc_block(struct net_device *dev,
7702                                struct tc_block_offload *f)
7703 {
7704         struct i40e_netdev_priv *np = netdev_priv(dev);
7705
7706         if (f->binder_type != TCF_BLOCK_BINDER_TYPE_CLSACT_INGRESS)
7707                 return -EOPNOTSUPP;
7708
7709         switch (f->command) {
7710         case TC_BLOCK_BIND:
7711                 return tcf_block_cb_register(f->block, i40e_setup_tc_block_cb,
7712                                              np, np, f->extack);
7713         case TC_BLOCK_UNBIND:
7714                 tcf_block_cb_unregister(f->block, i40e_setup_tc_block_cb, np);
7715                 return 0;
7716         default:
7717                 return -EOPNOTSUPP;
7718         }
7719 }
7720
7721 static int __i40e_setup_tc(struct net_device *netdev, enum tc_setup_type type,
7722                            void *type_data)
7723 {
7724         switch (type) {
7725         case TC_SETUP_QDISC_MQPRIO:
7726                 return i40e_setup_tc(netdev, type_data);
7727         case TC_SETUP_BLOCK:
7728                 return i40e_setup_tc_block(netdev, type_data);
7729         default:
7730                 return -EOPNOTSUPP;
7731         }
7732 }
7733
7734 /**
7735  * i40e_open - Called when a network interface is made active
7736  * @netdev: network interface device structure
7737  *
7738  * The open entry point is called when a network interface is made
7739  * active by the system (IFF_UP).  At this point all resources needed
7740  * for transmit and receive operations are allocated, the interrupt
7741  * handler is registered with the OS, the netdev watchdog subtask is
7742  * enabled, and the stack is notified that the interface is ready.
7743  *
7744  * Returns 0 on success, negative value on failure
7745  **/
7746 int i40e_open(struct net_device *netdev)
7747 {
7748         struct i40e_netdev_priv *np = netdev_priv(netdev);
7749         struct i40e_vsi *vsi = np->vsi;
7750         struct i40e_pf *pf = vsi->back;
7751         int err;
7752
7753         /* disallow open during test or if eeprom is broken */
7754         if (test_bit(__I40E_TESTING, pf->state) ||
7755             test_bit(__I40E_BAD_EEPROM, pf->state))
7756                 return -EBUSY;
7757
7758         netif_carrier_off(netdev);
7759
7760         if (i40e_force_link_state(pf, true))
7761                 return -EAGAIN;
7762
7763         err = i40e_vsi_open(vsi);
7764         if (err)
7765                 return err;
7766
7767         /* configure global TSO hardware offload settings */
7768         wr32(&pf->hw, I40E_GLLAN_TSOMSK_F, be32_to_cpu(TCP_FLAG_PSH |
7769                                                        TCP_FLAG_FIN) >> 16);
7770         wr32(&pf->hw, I40E_GLLAN_TSOMSK_M, be32_to_cpu(TCP_FLAG_PSH |
7771                                                        TCP_FLAG_FIN |
7772                                                        TCP_FLAG_CWR) >> 16);
7773         wr32(&pf->hw, I40E_GLLAN_TSOMSK_L, be32_to_cpu(TCP_FLAG_CWR) >> 16);
7774
7775         udp_tunnel_get_rx_info(netdev);
7776
7777         return 0;
7778 }
7779
7780 /**
7781  * i40e_netif_set_realnum_tx_rx_queues - Update number of tx/rx queues
7782  * @vsi: vsi structure
7783  *
7784  * This updates netdev's number of tx/rx queues
7785  *
7786  * Returns status of setting tx/rx queues
7787  **/
7788 static int i40e_netif_set_realnum_tx_rx_queues(struct i40e_vsi *vsi)
7789 {
7790         int ret;
7791
7792         ret = netif_set_real_num_rx_queues(vsi->netdev,
7793                                            vsi->num_queue_pairs);
7794         if (ret)
7795                 return ret;
7796
7797         return netif_set_real_num_tx_queues(vsi->netdev,
7798                                             vsi->num_queue_pairs);
7799 }
7800
7801 /**
7802  * i40e_vsi_open -
7803  * @vsi: the VSI to open
7804  *
7805  * Finish initialization of the VSI.
7806  *
7807  * Returns 0 on success, negative value on failure
7808  *
7809  * Note: expects to be called while under rtnl_lock()
7810  **/
7811 int i40e_vsi_open(struct i40e_vsi *vsi)
7812 {
7813         struct i40e_pf *pf = vsi->back;
7814         char int_name[I40E_INT_NAME_STR_LEN];
7815         int err;
7816
7817         /* allocate descriptors */
7818         err = i40e_vsi_setup_tx_resources(vsi);
7819         if (err)
7820                 goto err_setup_tx;
7821         err = i40e_vsi_setup_rx_resources(vsi);
7822         if (err)
7823                 goto err_setup_rx;
7824
7825         err = i40e_vsi_configure(vsi);
7826         if (err)
7827                 goto err_setup_rx;
7828
7829         if (vsi->netdev) {
7830                 snprintf(int_name, sizeof(int_name) - 1, "%s-%s",
7831                          dev_driver_string(&pf->pdev->dev), vsi->netdev->name);
7832                 err = i40e_vsi_request_irq(vsi, int_name);
7833                 if (err)
7834                         goto err_setup_rx;
7835
7836                 /* Notify the stack of the actual queue counts. */
7837                 err = i40e_netif_set_realnum_tx_rx_queues(vsi);
7838                 if (err)
7839                         goto err_set_queues;
7840
7841         } else if (vsi->type == I40E_VSI_FDIR) {
7842                 snprintf(int_name, sizeof(int_name) - 1, "%s-%s:fdir",
7843                          dev_driver_string(&pf->pdev->dev),
7844                          dev_name(&pf->pdev->dev));
7845                 err = i40e_vsi_request_irq(vsi, int_name);
7846                 if (err)
7847                         goto err_setup_rx;
7848
7849         } else {
7850                 err = -EINVAL;
7851                 goto err_setup_rx;
7852         }
7853
7854         err = i40e_up_complete(vsi);
7855         if (err)
7856                 goto err_up_complete;
7857
7858         return 0;
7859
7860 err_up_complete:
7861         i40e_down(vsi);
7862 err_set_queues:
7863         i40e_vsi_free_irq(vsi);
7864 err_setup_rx:
7865         i40e_vsi_free_rx_resources(vsi);
7866 err_setup_tx:
7867         i40e_vsi_free_tx_resources(vsi);
7868         if (vsi == pf->vsi[pf->lan_vsi])
7869                 i40e_do_reset(pf, I40E_PF_RESET_FLAG, true);
7870
7871         return err;
7872 }
7873
7874 /**
7875  * i40e_fdir_filter_exit - Cleans up the Flow Director accounting
7876  * @pf: Pointer to PF
7877  *
7878  * This function destroys the hlist where all the Flow Director
7879  * filters were saved.
7880  **/
7881 static void i40e_fdir_filter_exit(struct i40e_pf *pf)
7882 {
7883         struct i40e_fdir_filter *filter;
7884         struct i40e_flex_pit *pit_entry, *tmp;
7885         struct hlist_node *node2;
7886
7887         hlist_for_each_entry_safe(filter, node2,
7888                                   &pf->fdir_filter_list, fdir_node) {
7889                 hlist_del(&filter->fdir_node);
7890                 kfree(filter);
7891         }
7892
7893         list_for_each_entry_safe(pit_entry, tmp, &pf->l3_flex_pit_list, list) {
7894                 list_del(&pit_entry->list);
7895                 kfree(pit_entry);
7896         }
7897         INIT_LIST_HEAD(&pf->l3_flex_pit_list);
7898
7899         list_for_each_entry_safe(pit_entry, tmp, &pf->l4_flex_pit_list, list) {
7900                 list_del(&pit_entry->list);
7901                 kfree(pit_entry);
7902         }
7903         INIT_LIST_HEAD(&pf->l4_flex_pit_list);
7904
7905         pf->fdir_pf_active_filters = 0;
7906         pf->fd_tcp4_filter_cnt = 0;
7907         pf->fd_udp4_filter_cnt = 0;
7908         pf->fd_sctp4_filter_cnt = 0;
7909         pf->fd_ip4_filter_cnt = 0;
7910
7911         /* Reprogram the default input set for TCP/IPv4 */
7912         i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_TCP,
7913                                 I40E_L3_SRC_MASK | I40E_L3_DST_MASK |
7914                                 I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
7915
7916         /* Reprogram the default input set for UDP/IPv4 */
7917         i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_UDP,
7918                                 I40E_L3_SRC_MASK | I40E_L3_DST_MASK |
7919                                 I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
7920
7921         /* Reprogram the default input set for SCTP/IPv4 */
7922         i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_SCTP,
7923                                 I40E_L3_SRC_MASK | I40E_L3_DST_MASK |
7924                                 I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
7925
7926         /* Reprogram the default input set for Other/IPv4 */
7927         i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_OTHER,
7928                                 I40E_L3_SRC_MASK | I40E_L3_DST_MASK);
7929
7930         i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_FRAG_IPV4,
7931                                 I40E_L3_SRC_MASK | I40E_L3_DST_MASK);
7932 }
7933
7934 /**
7935  * i40e_cloud_filter_exit - Cleans up the cloud filters
7936  * @pf: Pointer to PF
7937  *
7938  * This function destroys the hlist where all the cloud filters
7939  * were saved.
7940  **/
7941 static void i40e_cloud_filter_exit(struct i40e_pf *pf)
7942 {
7943         struct i40e_cloud_filter *cfilter;
7944         struct hlist_node *node;
7945
7946         hlist_for_each_entry_safe(cfilter, node,
7947                                   &pf->cloud_filter_list, cloud_node) {
7948                 hlist_del(&cfilter->cloud_node);
7949                 kfree(cfilter);
7950         }
7951         pf->num_cloud_filters = 0;
7952
7953         if ((pf->flags & I40E_FLAG_FD_SB_TO_CLOUD_FILTER) &&
7954             !(pf->flags & I40E_FLAG_FD_SB_INACTIVE)) {
7955                 pf->flags |= I40E_FLAG_FD_SB_ENABLED;
7956                 pf->flags &= ~I40E_FLAG_FD_SB_TO_CLOUD_FILTER;
7957                 pf->flags &= ~I40E_FLAG_FD_SB_INACTIVE;
7958         }
7959 }
7960
7961 /**
7962  * i40e_close - Disables a network interface
7963  * @netdev: network interface device structure
7964  *
7965  * The close entry point is called when an interface is de-activated
7966  * by the OS.  The hardware is still under the driver's control, but
7967  * this netdev interface is disabled.
7968  *
7969  * Returns 0, this is not allowed to fail
7970  **/
7971 int i40e_close(struct net_device *netdev)
7972 {
7973         struct i40e_netdev_priv *np = netdev_priv(netdev);
7974         struct i40e_vsi *vsi = np->vsi;
7975
7976         i40e_vsi_close(vsi);
7977
7978         return 0;
7979 }
7980
7981 /**
7982  * i40e_do_reset - Start a PF or Core Reset sequence
7983  * @pf: board private structure
7984  * @reset_flags: which reset is requested
7985  * @lock_acquired: indicates whether or not the lock has been acquired
7986  * before this function was called.
7987  *
7988  * The essential difference in resets is that the PF Reset
7989  * doesn't clear the packet buffers, doesn't reset the PE
7990  * firmware, and doesn't bother the other PFs on the chip.
7991  **/
7992 void i40e_do_reset(struct i40e_pf *pf, u32 reset_flags, bool lock_acquired)
7993 {
7994         u32 val;
7995
7996         WARN_ON(in_interrupt());
7997
7998
7999         /* do the biggest reset indicated */
8000         if (reset_flags & BIT_ULL(__I40E_GLOBAL_RESET_REQUESTED)) {
8001
8002                 /* Request a Global Reset
8003                  *
8004                  * This will start the chip's countdown to the actual full
8005                  * chip reset event, and a warning interrupt to be sent
8006                  * to all PFs, including the requestor.  Our handler
8007                  * for the warning interrupt will deal with the shutdown
8008                  * and recovery of the switch setup.
8009                  */
8010                 dev_dbg(&pf->pdev->dev, "GlobalR requested\n");
8011                 val = rd32(&pf->hw, I40E_GLGEN_RTRIG);
8012                 val |= I40E_GLGEN_RTRIG_GLOBR_MASK;
8013                 wr32(&pf->hw, I40E_GLGEN_RTRIG, val);
8014
8015         } else if (reset_flags & BIT_ULL(__I40E_CORE_RESET_REQUESTED)) {
8016
8017                 /* Request a Core Reset
8018                  *
8019                  * Same as Global Reset, except does *not* include the MAC/PHY
8020                  */
8021                 dev_dbg(&pf->pdev->dev, "CoreR requested\n");
8022                 val = rd32(&pf->hw, I40E_GLGEN_RTRIG);
8023                 val |= I40E_GLGEN_RTRIG_CORER_MASK;
8024                 wr32(&pf->hw, I40E_GLGEN_RTRIG, val);
8025                 i40e_flush(&pf->hw);
8026
8027         } else if (reset_flags & I40E_PF_RESET_FLAG) {
8028
8029                 /* Request a PF Reset
8030                  *
8031                  * Resets only the PF-specific registers
8032                  *
8033                  * This goes directly to the tear-down and rebuild of
8034                  * the switch, since we need to do all the recovery as
8035                  * for the Core Reset.
8036                  */
8037                 dev_dbg(&pf->pdev->dev, "PFR requested\n");
8038                 i40e_handle_reset_warning(pf, lock_acquired);
8039
8040         } else if (reset_flags & I40E_PF_RESET_AND_REBUILD_FLAG) {
8041                 /* Request a PF Reset
8042                  *
8043                  * Resets PF and reinitializes PFs VSI.
8044                  */
8045                 i40e_prep_for_reset(pf, lock_acquired);
8046                 i40e_reset_and_rebuild(pf, true, lock_acquired);
8047
8048         } else if (reset_flags & BIT_ULL(__I40E_REINIT_REQUESTED)) {
8049                 int v;
8050
8051                 /* Find the VSI(s) that requested a re-init */
8052                 dev_info(&pf->pdev->dev,
8053                          "VSI reinit requested\n");
8054                 for (v = 0; v < pf->num_alloc_vsi; v++) {
8055                         struct i40e_vsi *vsi = pf->vsi[v];
8056
8057                         if (vsi != NULL &&
8058                             test_and_clear_bit(__I40E_VSI_REINIT_REQUESTED,
8059                                                vsi->state))
8060                                 i40e_vsi_reinit_locked(pf->vsi[v]);
8061                 }
8062         } else if (reset_flags & BIT_ULL(__I40E_DOWN_REQUESTED)) {
8063                 int v;
8064
8065                 /* Find the VSI(s) that needs to be brought down */
8066                 dev_info(&pf->pdev->dev, "VSI down requested\n");
8067                 for (v = 0; v < pf->num_alloc_vsi; v++) {
8068                         struct i40e_vsi *vsi = pf->vsi[v];
8069
8070                         if (vsi != NULL &&
8071                             test_and_clear_bit(__I40E_VSI_DOWN_REQUESTED,
8072                                                vsi->state)) {
8073                                 set_bit(__I40E_VSI_DOWN, vsi->state);
8074                                 i40e_down(vsi);
8075                         }
8076                 }
8077         } else {
8078                 dev_info(&pf->pdev->dev,
8079                          "bad reset request 0x%08x\n", reset_flags);
8080         }
8081 }
8082
8083 #ifdef CONFIG_I40E_DCB
8084 /**
8085  * i40e_dcb_need_reconfig - Check if DCB needs reconfig
8086  * @pf: board private structure
8087  * @old_cfg: current DCB config
8088  * @new_cfg: new DCB config
8089  **/
8090 bool i40e_dcb_need_reconfig(struct i40e_pf *pf,
8091                             struct i40e_dcbx_config *old_cfg,
8092                             struct i40e_dcbx_config *new_cfg)
8093 {
8094         bool need_reconfig = false;
8095
8096         /* Check if ETS configuration has changed */
8097         if (memcmp(&new_cfg->etscfg,
8098                    &old_cfg->etscfg,
8099                    sizeof(new_cfg->etscfg))) {
8100                 /* If Priority Table has changed reconfig is needed */
8101                 if (memcmp(&new_cfg->etscfg.prioritytable,
8102                            &old_cfg->etscfg.prioritytable,
8103                            sizeof(new_cfg->etscfg.prioritytable))) {
8104                         need_reconfig = true;
8105                         dev_dbg(&pf->pdev->dev, "ETS UP2TC changed.\n");
8106                 }
8107
8108                 if (memcmp(&new_cfg->etscfg.tcbwtable,
8109                            &old_cfg->etscfg.tcbwtable,
8110                            sizeof(new_cfg->etscfg.tcbwtable)))
8111                         dev_dbg(&pf->pdev->dev, "ETS TC BW Table changed.\n");
8112
8113                 if (memcmp(&new_cfg->etscfg.tsatable,
8114                            &old_cfg->etscfg.tsatable,
8115                            sizeof(new_cfg->etscfg.tsatable)))
8116                         dev_dbg(&pf->pdev->dev, "ETS TSA Table changed.\n");
8117         }
8118
8119         /* Check if PFC configuration has changed */
8120         if (memcmp(&new_cfg->pfc,
8121                    &old_cfg->pfc,
8122                    sizeof(new_cfg->pfc))) {
8123                 need_reconfig = true;
8124                 dev_dbg(&pf->pdev->dev, "PFC config change detected.\n");
8125         }
8126
8127         /* Check if APP Table has changed */
8128         if (memcmp(&new_cfg->app,
8129                    &old_cfg->app,
8130                    sizeof(new_cfg->app))) {
8131                 need_reconfig = true;
8132                 dev_dbg(&pf->pdev->dev, "APP Table change detected.\n");
8133         }
8134
8135         dev_dbg(&pf->pdev->dev, "dcb need_reconfig=%d\n", need_reconfig);
8136         return need_reconfig;
8137 }
8138
8139 /**
8140  * i40e_handle_lldp_event - Handle LLDP Change MIB event
8141  * @pf: board private structure
8142  * @e: event info posted on ARQ
8143  **/
8144 static int i40e_handle_lldp_event(struct i40e_pf *pf,
8145                                   struct i40e_arq_event_info *e)
8146 {
8147         struct i40e_aqc_lldp_get_mib *mib =
8148                 (struct i40e_aqc_lldp_get_mib *)&e->desc.params.raw;
8149         struct i40e_hw *hw = &pf->hw;
8150         struct i40e_dcbx_config tmp_dcbx_cfg;
8151         bool need_reconfig = false;
8152         int ret = 0;
8153         u8 type;
8154
8155         /* Not DCB capable or capability disabled */
8156         if (!(pf->flags & I40E_FLAG_DCB_CAPABLE))
8157                 return ret;
8158
8159         /* Ignore if event is not for Nearest Bridge */
8160         type = ((mib->type >> I40E_AQ_LLDP_BRIDGE_TYPE_SHIFT)
8161                 & I40E_AQ_LLDP_BRIDGE_TYPE_MASK);
8162         dev_dbg(&pf->pdev->dev, "LLDP event mib bridge type 0x%x\n", type);
8163         if (type != I40E_AQ_LLDP_BRIDGE_TYPE_NEAREST_BRIDGE)
8164                 return ret;
8165
8166         /* Check MIB Type and return if event for Remote MIB update */
8167         type = mib->type & I40E_AQ_LLDP_MIB_TYPE_MASK;
8168         dev_dbg(&pf->pdev->dev,
8169                 "LLDP event mib type %s\n", type ? "remote" : "local");
8170         if (type == I40E_AQ_LLDP_MIB_REMOTE) {
8171                 /* Update the remote cached instance and return */
8172                 ret = i40e_aq_get_dcb_config(hw, I40E_AQ_LLDP_MIB_REMOTE,
8173                                 I40E_AQ_LLDP_BRIDGE_TYPE_NEAREST_BRIDGE,
8174                                 &hw->remote_dcbx_config);
8175                 goto exit;
8176         }
8177
8178         /* Store the old configuration */
8179         tmp_dcbx_cfg = hw->local_dcbx_config;
8180
8181         /* Reset the old DCBx configuration data */
8182         memset(&hw->local_dcbx_config, 0, sizeof(hw->local_dcbx_config));
8183         /* Get updated DCBX data from firmware */
8184         ret = i40e_get_dcb_config(&pf->hw);
8185         if (ret) {
8186                 dev_info(&pf->pdev->dev,
8187                          "Failed querying DCB configuration data from firmware, err %s aq_err %s\n",
8188                          i40e_stat_str(&pf->hw, ret),
8189                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
8190                 goto exit;
8191         }
8192
8193         /* No change detected in DCBX configs */
8194         if (!memcmp(&tmp_dcbx_cfg, &hw->local_dcbx_config,
8195                     sizeof(tmp_dcbx_cfg))) {
8196                 dev_dbg(&pf->pdev->dev, "No change detected in DCBX configuration.\n");
8197                 goto exit;
8198         }
8199
8200         need_reconfig = i40e_dcb_need_reconfig(pf, &tmp_dcbx_cfg,
8201                                                &hw->local_dcbx_config);
8202
8203         i40e_dcbnl_flush_apps(pf, &tmp_dcbx_cfg, &hw->local_dcbx_config);
8204
8205         if (!need_reconfig)
8206                 goto exit;
8207
8208         /* Enable DCB tagging only when more than one TC */
8209         if (i40e_dcb_get_num_tc(&hw->local_dcbx_config) > 1)
8210                 pf->flags |= I40E_FLAG_DCB_ENABLED;
8211         else
8212                 pf->flags &= ~I40E_FLAG_DCB_ENABLED;
8213
8214         set_bit(__I40E_PORT_SUSPENDED, pf->state);
8215         /* Reconfiguration needed quiesce all VSIs */
8216         i40e_pf_quiesce_all_vsi(pf);
8217
8218         /* Changes in configuration update VEB/VSI */
8219         i40e_dcb_reconfigure(pf);
8220
8221         ret = i40e_resume_port_tx(pf);
8222
8223         clear_bit(__I40E_PORT_SUSPENDED, pf->state);
8224         /* In case of error no point in resuming VSIs */
8225         if (ret)
8226                 goto exit;
8227
8228         /* Wait for the PF's queues to be disabled */
8229         ret = i40e_pf_wait_queues_disabled(pf);
8230         if (ret) {
8231                 /* Schedule PF reset to recover */
8232                 set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
8233                 i40e_service_event_schedule(pf);
8234         } else {
8235                 i40e_pf_unquiesce_all_vsi(pf);
8236         set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state);
8237         set_bit(__I40E_CLIENT_L2_CHANGE, pf->state);
8238         }
8239
8240 exit:
8241         return ret;
8242 }
8243 #endif /* CONFIG_I40E_DCB */
8244
8245 /**
8246  * i40e_do_reset_safe - Protected reset path for userland calls.
8247  * @pf: board private structure
8248  * @reset_flags: which reset is requested
8249  *
8250  **/
8251 void i40e_do_reset_safe(struct i40e_pf *pf, u32 reset_flags)
8252 {
8253         rtnl_lock();
8254         i40e_do_reset(pf, reset_flags, true);
8255         rtnl_unlock();
8256 }
8257
8258 /**
8259  * i40e_handle_lan_overflow_event - Handler for LAN queue overflow event
8260  * @pf: board private structure
8261  * @e: event info posted on ARQ
8262  *
8263  * Handler for LAN Queue Overflow Event generated by the firmware for PF
8264  * and VF queues
8265  **/
8266 static void i40e_handle_lan_overflow_event(struct i40e_pf *pf,
8267                                            struct i40e_arq_event_info *e)
8268 {
8269         struct i40e_aqc_lan_overflow *data =
8270                 (struct i40e_aqc_lan_overflow *)&e->desc.params.raw;
8271         u32 queue = le32_to_cpu(data->prtdcb_rupto);
8272         u32 qtx_ctl = le32_to_cpu(data->otx_ctl);
8273         struct i40e_hw *hw = &pf->hw;
8274         struct i40e_vf *vf;
8275         u16 vf_id;
8276
8277         dev_dbg(&pf->pdev->dev, "overflow Rx Queue Number = %d QTX_CTL=0x%08x\n",
8278                 queue, qtx_ctl);
8279
8280         /* Queue belongs to VF, find the VF and issue VF reset */
8281         if (((qtx_ctl & I40E_QTX_CTL_PFVF_Q_MASK)
8282             >> I40E_QTX_CTL_PFVF_Q_SHIFT) == I40E_QTX_CTL_VF_QUEUE) {
8283                 vf_id = (u16)((qtx_ctl & I40E_QTX_CTL_VFVM_INDX_MASK)
8284                          >> I40E_QTX_CTL_VFVM_INDX_SHIFT);
8285                 vf_id -= hw->func_caps.vf_base_id;
8286                 vf = &pf->vf[vf_id];
8287                 i40e_vc_notify_vf_reset(vf);
8288                 /* Allow VF to process pending reset notification */
8289                 msleep(20);
8290                 i40e_reset_vf(vf, false);
8291         }
8292 }
8293
8294 /**
8295  * i40e_get_cur_guaranteed_fd_count - Get the consumed guaranteed FD filters
8296  * @pf: board private structure
8297  **/
8298 u32 i40e_get_cur_guaranteed_fd_count(struct i40e_pf *pf)
8299 {
8300         u32 val, fcnt_prog;
8301
8302         val = rd32(&pf->hw, I40E_PFQF_FDSTAT);
8303         fcnt_prog = (val & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK);
8304         return fcnt_prog;
8305 }
8306
8307 /**
8308  * i40e_get_current_fd_count - Get total FD filters programmed for this PF
8309  * @pf: board private structure
8310  **/
8311 u32 i40e_get_current_fd_count(struct i40e_pf *pf)
8312 {
8313         u32 val, fcnt_prog;
8314
8315         val = rd32(&pf->hw, I40E_PFQF_FDSTAT);
8316         fcnt_prog = (val & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK) +
8317                     ((val & I40E_PFQF_FDSTAT_BEST_CNT_MASK) >>
8318                       I40E_PFQF_FDSTAT_BEST_CNT_SHIFT);
8319         return fcnt_prog;
8320 }
8321
8322 /**
8323  * i40e_get_global_fd_count - Get total FD filters programmed on device
8324  * @pf: board private structure
8325  **/
8326 u32 i40e_get_global_fd_count(struct i40e_pf *pf)
8327 {
8328         u32 val, fcnt_prog;
8329
8330         val = rd32(&pf->hw, I40E_GLQF_FDCNT_0);
8331         fcnt_prog = (val & I40E_GLQF_FDCNT_0_GUARANT_CNT_MASK) +
8332                     ((val & I40E_GLQF_FDCNT_0_BESTCNT_MASK) >>
8333                      I40E_GLQF_FDCNT_0_BESTCNT_SHIFT);
8334         return fcnt_prog;
8335 }
8336
8337 /**
8338  * i40e_reenable_fdir_sb - Restore FDir SB capability
8339  * @pf: board private structure
8340  **/
8341 static void i40e_reenable_fdir_sb(struct i40e_pf *pf)
8342 {
8343         if (test_and_clear_bit(__I40E_FD_SB_AUTO_DISABLED, pf->state))
8344                 if ((pf->flags & I40E_FLAG_FD_SB_ENABLED) &&
8345                     (I40E_DEBUG_FD & pf->hw.debug_mask))
8346                         dev_info(&pf->pdev->dev, "FD Sideband/ntuple is being enabled since we have space in the table now\n");
8347 }
8348
8349 /**
8350  * i40e_reenable_fdir_atr - Restore FDir ATR capability
8351  * @pf: board private structure
8352  **/
8353 static void i40e_reenable_fdir_atr(struct i40e_pf *pf)
8354 {
8355         if (test_and_clear_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state)) {
8356                 /* ATR uses the same filtering logic as SB rules. It only
8357                  * functions properly if the input set mask is at the default
8358                  * settings. It is safe to restore the default input set
8359                  * because there are no active TCPv4 filter rules.
8360                  */
8361                 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_TCP,
8362                                         I40E_L3_SRC_MASK | I40E_L3_DST_MASK |
8363                                         I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
8364
8365                 if ((pf->flags & I40E_FLAG_FD_ATR_ENABLED) &&
8366                     (I40E_DEBUG_FD & pf->hw.debug_mask))
8367                         dev_info(&pf->pdev->dev, "ATR is being enabled since we have space in the table and there are no conflicting ntuple rules\n");
8368         }
8369 }
8370
8371 /**
8372  * i40e_delete_invalid_filter - Delete an invalid FDIR filter
8373  * @pf: board private structure
8374  * @filter: FDir filter to remove
8375  */
8376 static void i40e_delete_invalid_filter(struct i40e_pf *pf,
8377                                        struct i40e_fdir_filter *filter)
8378 {
8379         /* Update counters */
8380         pf->fdir_pf_active_filters--;
8381         pf->fd_inv = 0;
8382
8383         switch (filter->flow_type) {
8384         case TCP_V4_FLOW:
8385                 pf->fd_tcp4_filter_cnt--;
8386                 break;
8387         case UDP_V4_FLOW:
8388                 pf->fd_udp4_filter_cnt--;
8389                 break;
8390         case SCTP_V4_FLOW:
8391                 pf->fd_sctp4_filter_cnt--;
8392                 break;
8393         case IP_USER_FLOW:
8394                 switch (filter->ip4_proto) {
8395                 case IPPROTO_TCP:
8396                         pf->fd_tcp4_filter_cnt--;
8397                         break;
8398                 case IPPROTO_UDP:
8399                         pf->fd_udp4_filter_cnt--;
8400                         break;
8401                 case IPPROTO_SCTP:
8402                         pf->fd_sctp4_filter_cnt--;
8403                         break;
8404                 case IPPROTO_IP:
8405                         pf->fd_ip4_filter_cnt--;
8406                         break;
8407                 }
8408                 break;
8409         }
8410
8411         /* Remove the filter from the list and free memory */
8412         hlist_del(&filter->fdir_node);
8413         kfree(filter);
8414 }
8415
8416 /**
8417  * i40e_fdir_check_and_reenable - Function to reenabe FD ATR or SB if disabled
8418  * @pf: board private structure
8419  **/
8420 void i40e_fdir_check_and_reenable(struct i40e_pf *pf)
8421 {
8422         struct i40e_fdir_filter *filter;
8423         u32 fcnt_prog, fcnt_avail;
8424         struct hlist_node *node;
8425
8426         if (test_bit(__I40E_FD_FLUSH_REQUESTED, pf->state))
8427                 return;
8428
8429         /* Check if we have enough room to re-enable FDir SB capability. */
8430         fcnt_prog = i40e_get_global_fd_count(pf);
8431         fcnt_avail = pf->fdir_pf_filter_count;
8432         if ((fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM)) ||
8433             (pf->fd_add_err == 0) ||
8434             (i40e_get_current_atr_cnt(pf) < pf->fd_atr_cnt))
8435                 i40e_reenable_fdir_sb(pf);
8436
8437         /* We should wait for even more space before re-enabling ATR.
8438          * Additionally, we cannot enable ATR as long as we still have TCP SB
8439          * rules active.
8440          */
8441         if ((fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM_FOR_ATR)) &&
8442             (pf->fd_tcp4_filter_cnt == 0))
8443                 i40e_reenable_fdir_atr(pf);
8444
8445         /* if hw had a problem adding a filter, delete it */
8446         if (pf->fd_inv > 0) {
8447                 hlist_for_each_entry_safe(filter, node,
8448                                           &pf->fdir_filter_list, fdir_node)
8449                         if (filter->fd_id == pf->fd_inv)
8450                                 i40e_delete_invalid_filter(pf, filter);
8451         }
8452 }
8453
8454 #define I40E_MIN_FD_FLUSH_INTERVAL 10
8455 #define I40E_MIN_FD_FLUSH_SB_ATR_UNSTABLE 30
8456 /**
8457  * i40e_fdir_flush_and_replay - Function to flush all FD filters and replay SB
8458  * @pf: board private structure
8459  **/
8460 static void i40e_fdir_flush_and_replay(struct i40e_pf *pf)
8461 {
8462         unsigned long min_flush_time;
8463         int flush_wait_retry = 50;
8464         bool disable_atr = false;
8465         int fd_room;
8466         int reg;
8467
8468         if (!time_after(jiffies, pf->fd_flush_timestamp +
8469                                  (I40E_MIN_FD_FLUSH_INTERVAL * HZ)))
8470                 return;
8471
8472         /* If the flush is happening too quick and we have mostly SB rules we
8473          * should not re-enable ATR for some time.
8474          */
8475         min_flush_time = pf->fd_flush_timestamp +
8476                          (I40E_MIN_FD_FLUSH_SB_ATR_UNSTABLE * HZ);
8477         fd_room = pf->fdir_pf_filter_count - pf->fdir_pf_active_filters;
8478
8479         if (!(time_after(jiffies, min_flush_time)) &&
8480             (fd_room < I40E_FDIR_BUFFER_HEAD_ROOM_FOR_ATR)) {
8481                 if (I40E_DEBUG_FD & pf->hw.debug_mask)
8482                         dev_info(&pf->pdev->dev, "ATR disabled, not enough FD filter space.\n");
8483                 disable_atr = true;
8484         }
8485
8486         pf->fd_flush_timestamp = jiffies;
8487         set_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state);
8488         /* flush all filters */
8489         wr32(&pf->hw, I40E_PFQF_CTL_1,
8490              I40E_PFQF_CTL_1_CLEARFDTABLE_MASK);
8491         i40e_flush(&pf->hw);
8492         pf->fd_flush_cnt++;
8493         pf->fd_add_err = 0;
8494         do {
8495                 /* Check FD flush status every 5-6msec */
8496                 usleep_range(5000, 6000);
8497                 reg = rd32(&pf->hw, I40E_PFQF_CTL_1);
8498                 if (!(reg & I40E_PFQF_CTL_1_CLEARFDTABLE_MASK))
8499                         break;
8500         } while (flush_wait_retry--);
8501         if (reg & I40E_PFQF_CTL_1_CLEARFDTABLE_MASK) {
8502                 dev_warn(&pf->pdev->dev, "FD table did not flush, needs more time\n");
8503         } else {
8504                 /* replay sideband filters */
8505                 i40e_fdir_filter_restore(pf->vsi[pf->lan_vsi]);
8506                 if (!disable_atr && !pf->fd_tcp4_filter_cnt)
8507                         clear_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state);
8508                 clear_bit(__I40E_FD_FLUSH_REQUESTED, pf->state);
8509                 if (I40E_DEBUG_FD & pf->hw.debug_mask)
8510                         dev_info(&pf->pdev->dev, "FD Filter table flushed and FD-SB replayed.\n");
8511         }
8512 }
8513
8514 /**
8515  * i40e_get_current_atr_count - Get the count of total FD ATR filters programmed
8516  * @pf: board private structure
8517  **/
8518 u32 i40e_get_current_atr_cnt(struct i40e_pf *pf)
8519 {
8520         return i40e_get_current_fd_count(pf) - pf->fdir_pf_active_filters;
8521 }
8522
8523 /* We can see up to 256 filter programming desc in transit if the filters are
8524  * being applied really fast; before we see the first
8525  * filter miss error on Rx queue 0. Accumulating enough error messages before
8526  * reacting will make sure we don't cause flush too often.
8527  */
8528 #define I40E_MAX_FD_PROGRAM_ERROR 256
8529
8530 /**
8531  * i40e_fdir_reinit_subtask - Worker thread to reinit FDIR filter table
8532  * @pf: board private structure
8533  **/
8534 static void i40e_fdir_reinit_subtask(struct i40e_pf *pf)
8535 {
8536
8537         /* if interface is down do nothing */
8538         if (test_bit(__I40E_DOWN, pf->state))
8539                 return;
8540
8541         if (test_bit(__I40E_FD_FLUSH_REQUESTED, pf->state))
8542                 i40e_fdir_flush_and_replay(pf);
8543
8544         i40e_fdir_check_and_reenable(pf);
8545
8546 }
8547
8548 /**
8549  * i40e_vsi_link_event - notify VSI of a link event
8550  * @vsi: vsi to be notified
8551  * @link_up: link up or down
8552  **/
8553 static void i40e_vsi_link_event(struct i40e_vsi *vsi, bool link_up)
8554 {
8555         if (!vsi || test_bit(__I40E_VSI_DOWN, vsi->state))
8556                 return;
8557
8558         switch (vsi->type) {
8559         case I40E_VSI_MAIN:
8560                 if (!vsi->netdev || !vsi->netdev_registered)
8561                         break;
8562
8563                 if (link_up) {
8564                         netif_carrier_on(vsi->netdev);
8565                         netif_tx_wake_all_queues(vsi->netdev);
8566                 } else {
8567                         netif_carrier_off(vsi->netdev);
8568                         netif_tx_stop_all_queues(vsi->netdev);
8569                 }
8570                 break;
8571
8572         case I40E_VSI_SRIOV:
8573         case I40E_VSI_VMDQ2:
8574         case I40E_VSI_CTRL:
8575         case I40E_VSI_IWARP:
8576         case I40E_VSI_MIRROR:
8577         default:
8578                 /* there is no notification for other VSIs */
8579                 break;
8580         }
8581 }
8582
8583 /**
8584  * i40e_veb_link_event - notify elements on the veb of a link event
8585  * @veb: veb to be notified
8586  * @link_up: link up or down
8587  **/
8588 static void i40e_veb_link_event(struct i40e_veb *veb, bool link_up)
8589 {
8590         struct i40e_pf *pf;
8591         int i;
8592
8593         if (!veb || !veb->pf)
8594                 return;
8595         pf = veb->pf;
8596
8597         /* depth first... */
8598         for (i = 0; i < I40E_MAX_VEB; i++)
8599                 if (pf->veb[i] && (pf->veb[i]->uplink_seid == veb->seid))
8600                         i40e_veb_link_event(pf->veb[i], link_up);
8601
8602         /* ... now the local VSIs */
8603         for (i = 0; i < pf->num_alloc_vsi; i++)
8604                 if (pf->vsi[i] && (pf->vsi[i]->uplink_seid == veb->seid))
8605                         i40e_vsi_link_event(pf->vsi[i], link_up);
8606 }
8607
8608 /**
8609  * i40e_link_event - Update netif_carrier status
8610  * @pf: board private structure
8611  **/
8612 static void i40e_link_event(struct i40e_pf *pf)
8613 {
8614         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
8615         u8 new_link_speed, old_link_speed;
8616         i40e_status status;
8617         bool new_link, old_link;
8618
8619         /* save off old link status information */
8620         pf->hw.phy.link_info_old = pf->hw.phy.link_info;
8621
8622         /* set this to force the get_link_status call to refresh state */
8623         pf->hw.phy.get_link_info = true;
8624
8625         old_link = (pf->hw.phy.link_info_old.link_info & I40E_AQ_LINK_UP);
8626
8627         status = i40e_get_link_status(&pf->hw, &new_link);
8628
8629         /* On success, disable temp link polling */
8630         if (status == I40E_SUCCESS) {
8631                 clear_bit(__I40E_TEMP_LINK_POLLING, pf->state);
8632         } else {
8633                 /* Enable link polling temporarily until i40e_get_link_status
8634                  * returns I40E_SUCCESS
8635                  */
8636                 set_bit(__I40E_TEMP_LINK_POLLING, pf->state);
8637                 dev_dbg(&pf->pdev->dev, "couldn't get link state, status: %d\n",
8638                         status);
8639                 return;
8640         }
8641
8642         old_link_speed = pf->hw.phy.link_info_old.link_speed;
8643         new_link_speed = pf->hw.phy.link_info.link_speed;
8644
8645         if (new_link == old_link &&
8646             new_link_speed == old_link_speed &&
8647             (test_bit(__I40E_VSI_DOWN, vsi->state) ||
8648              new_link == netif_carrier_ok(vsi->netdev)))
8649                 return;
8650
8651         i40e_print_link_message(vsi, new_link);
8652
8653         /* Notify the base of the switch tree connected to
8654          * the link.  Floating VEBs are not notified.
8655          */
8656         if (pf->lan_veb != I40E_NO_VEB && pf->veb[pf->lan_veb])
8657                 i40e_veb_link_event(pf->veb[pf->lan_veb], new_link);
8658         else
8659                 i40e_vsi_link_event(vsi, new_link);
8660
8661         if (pf->vf)
8662                 i40e_vc_notify_link_state(pf);
8663
8664         if (pf->flags & I40E_FLAG_PTP)
8665                 i40e_ptp_set_increment(pf);
8666 }
8667
8668 /**
8669  * i40e_watchdog_subtask - periodic checks not using event driven response
8670  * @pf: board private structure
8671  **/
8672 static void i40e_watchdog_subtask(struct i40e_pf *pf)
8673 {
8674         int i;
8675
8676         /* if interface is down do nothing */
8677         if (test_bit(__I40E_DOWN, pf->state) ||
8678             test_bit(__I40E_CONFIG_BUSY, pf->state))
8679                 return;
8680
8681         /* make sure we don't do these things too often */
8682         if (time_before(jiffies, (pf->service_timer_previous +
8683                                   pf->service_timer_period)))
8684                 return;
8685         pf->service_timer_previous = jiffies;
8686
8687         if ((pf->flags & I40E_FLAG_LINK_POLLING_ENABLED) ||
8688             test_bit(__I40E_TEMP_LINK_POLLING, pf->state))
8689                 i40e_link_event(pf);
8690
8691         /* Update the stats for active netdevs so the network stack
8692          * can look at updated numbers whenever it cares to
8693          */
8694         for (i = 0; i < pf->num_alloc_vsi; i++)
8695                 if (pf->vsi[i] && pf->vsi[i]->netdev)
8696                         i40e_update_stats(pf->vsi[i]);
8697
8698         if (pf->flags & I40E_FLAG_VEB_STATS_ENABLED) {
8699                 /* Update the stats for the active switching components */
8700                 for (i = 0; i < I40E_MAX_VEB; i++)
8701                         if (pf->veb[i])
8702                                 i40e_update_veb_stats(pf->veb[i]);
8703         }
8704
8705         i40e_ptp_rx_hang(pf);
8706         i40e_ptp_tx_hang(pf);
8707 }
8708
8709 /**
8710  * i40e_reset_subtask - Set up for resetting the device and driver
8711  * @pf: board private structure
8712  **/
8713 static void i40e_reset_subtask(struct i40e_pf *pf)
8714 {
8715         u32 reset_flags = 0;
8716
8717         if (test_bit(__I40E_REINIT_REQUESTED, pf->state)) {
8718                 reset_flags |= BIT(__I40E_REINIT_REQUESTED);
8719                 clear_bit(__I40E_REINIT_REQUESTED, pf->state);
8720         }
8721         if (test_bit(__I40E_PF_RESET_REQUESTED, pf->state)) {
8722                 reset_flags |= BIT(__I40E_PF_RESET_REQUESTED);
8723                 clear_bit(__I40E_PF_RESET_REQUESTED, pf->state);
8724         }
8725         if (test_bit(__I40E_CORE_RESET_REQUESTED, pf->state)) {
8726                 reset_flags |= BIT(__I40E_CORE_RESET_REQUESTED);
8727                 clear_bit(__I40E_CORE_RESET_REQUESTED, pf->state);
8728         }
8729         if (test_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state)) {
8730                 reset_flags |= BIT(__I40E_GLOBAL_RESET_REQUESTED);
8731                 clear_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state);
8732         }
8733         if (test_bit(__I40E_DOWN_REQUESTED, pf->state)) {
8734                 reset_flags |= BIT(__I40E_DOWN_REQUESTED);
8735                 clear_bit(__I40E_DOWN_REQUESTED, pf->state);
8736         }
8737
8738         /* If there's a recovery already waiting, it takes
8739          * precedence before starting a new reset sequence.
8740          */
8741         if (test_bit(__I40E_RESET_INTR_RECEIVED, pf->state)) {
8742                 i40e_prep_for_reset(pf, false);
8743                 i40e_reset(pf);
8744                 i40e_rebuild(pf, false, false);
8745         }
8746
8747         /* If we're already down or resetting, just bail */
8748         if (reset_flags &&
8749             !test_bit(__I40E_DOWN, pf->state) &&
8750             !test_bit(__I40E_CONFIG_BUSY, pf->state)) {
8751                 i40e_do_reset(pf, reset_flags, false);
8752         }
8753 }
8754
8755 /**
8756  * i40e_handle_link_event - Handle link event
8757  * @pf: board private structure
8758  * @e: event info posted on ARQ
8759  **/
8760 static void i40e_handle_link_event(struct i40e_pf *pf,
8761                                    struct i40e_arq_event_info *e)
8762 {
8763         struct i40e_aqc_get_link_status *status =
8764                 (struct i40e_aqc_get_link_status *)&e->desc.params.raw;
8765
8766         /* Do a new status request to re-enable LSE reporting
8767          * and load new status information into the hw struct
8768          * This completely ignores any state information
8769          * in the ARQ event info, instead choosing to always
8770          * issue the AQ update link status command.
8771          */
8772         i40e_link_event(pf);
8773
8774         /* Check if module meets thermal requirements */
8775         if (status->phy_type == I40E_PHY_TYPE_NOT_SUPPORTED_HIGH_TEMP) {
8776                 dev_err(&pf->pdev->dev,
8777                         "Rx/Tx is disabled on this device because the module does not meet thermal requirements.\n");
8778                 dev_err(&pf->pdev->dev,
8779                         "Refer to the Intel(R) Ethernet Adapters and Devices User Guide for a list of supported modules.\n");
8780         } else {
8781                 /* check for unqualified module, if link is down, suppress
8782                  * the message if link was forced to be down.
8783                  */
8784                 if ((status->link_info & I40E_AQ_MEDIA_AVAILABLE) &&
8785                     (!(status->an_info & I40E_AQ_QUALIFIED_MODULE)) &&
8786                     (!(status->link_info & I40E_AQ_LINK_UP)) &&
8787                     (!(pf->flags & I40E_FLAG_LINK_DOWN_ON_CLOSE_ENABLED))) {
8788                         dev_err(&pf->pdev->dev,
8789                                 "Rx/Tx is disabled on this device because an unsupported SFP module type was detected.\n");
8790                         dev_err(&pf->pdev->dev,
8791                                 "Refer to the Intel(R) Ethernet Adapters and Devices User Guide for a list of supported modules.\n");
8792                 }
8793         }
8794 }
8795
8796 /**
8797  * i40e_clean_adminq_subtask - Clean the AdminQ rings
8798  * @pf: board private structure
8799  **/
8800 static void i40e_clean_adminq_subtask(struct i40e_pf *pf)
8801 {
8802         struct i40e_arq_event_info event;
8803         struct i40e_hw *hw = &pf->hw;
8804         u16 pending, i = 0;
8805         i40e_status ret;
8806         u16 opcode;
8807         u32 oldval;
8808         u32 val;
8809
8810         /* Do not run clean AQ when PF reset fails */
8811         if (test_bit(__I40E_RESET_FAILED, pf->state))
8812                 return;
8813
8814         /* check for error indications */
8815         val = rd32(&pf->hw, pf->hw.aq.arq.len);
8816         oldval = val;
8817         if (val & I40E_PF_ARQLEN_ARQVFE_MASK) {
8818                 if (hw->debug_mask & I40E_DEBUG_AQ)
8819                         dev_info(&pf->pdev->dev, "ARQ VF Error detected\n");
8820                 val &= ~I40E_PF_ARQLEN_ARQVFE_MASK;
8821         }
8822         if (val & I40E_PF_ARQLEN_ARQOVFL_MASK) {
8823                 if (hw->debug_mask & I40E_DEBUG_AQ)
8824                         dev_info(&pf->pdev->dev, "ARQ Overflow Error detected\n");
8825                 val &= ~I40E_PF_ARQLEN_ARQOVFL_MASK;
8826                 pf->arq_overflows++;
8827         }
8828         if (val & I40E_PF_ARQLEN_ARQCRIT_MASK) {
8829                 if (hw->debug_mask & I40E_DEBUG_AQ)
8830                         dev_info(&pf->pdev->dev, "ARQ Critical Error detected\n");
8831                 val &= ~I40E_PF_ARQLEN_ARQCRIT_MASK;
8832         }
8833         if (oldval != val)
8834                 wr32(&pf->hw, pf->hw.aq.arq.len, val);
8835
8836         val = rd32(&pf->hw, pf->hw.aq.asq.len);
8837         oldval = val;
8838         if (val & I40E_PF_ATQLEN_ATQVFE_MASK) {
8839                 if (pf->hw.debug_mask & I40E_DEBUG_AQ)
8840                         dev_info(&pf->pdev->dev, "ASQ VF Error detected\n");
8841                 val &= ~I40E_PF_ATQLEN_ATQVFE_MASK;
8842         }
8843         if (val & I40E_PF_ATQLEN_ATQOVFL_MASK) {
8844                 if (pf->hw.debug_mask & I40E_DEBUG_AQ)
8845                         dev_info(&pf->pdev->dev, "ASQ Overflow Error detected\n");
8846                 val &= ~I40E_PF_ATQLEN_ATQOVFL_MASK;
8847         }
8848         if (val & I40E_PF_ATQLEN_ATQCRIT_MASK) {
8849                 if (pf->hw.debug_mask & I40E_DEBUG_AQ)
8850                         dev_info(&pf->pdev->dev, "ASQ Critical Error detected\n");
8851                 val &= ~I40E_PF_ATQLEN_ATQCRIT_MASK;
8852         }
8853         if (oldval != val)
8854                 wr32(&pf->hw, pf->hw.aq.asq.len, val);
8855
8856         event.buf_len = I40E_MAX_AQ_BUF_SIZE;
8857         event.msg_buf = kzalloc(event.buf_len, GFP_KERNEL);
8858         if (!event.msg_buf)
8859                 return;
8860
8861         do {
8862                 ret = i40e_clean_arq_element(hw, &event, &pending);
8863                 if (ret == I40E_ERR_ADMIN_QUEUE_NO_WORK)
8864                         break;
8865                 else if (ret) {
8866                         dev_info(&pf->pdev->dev, "ARQ event error %d\n", ret);
8867                         break;
8868                 }
8869
8870                 opcode = le16_to_cpu(event.desc.opcode);
8871                 switch (opcode) {
8872
8873                 case i40e_aqc_opc_get_link_status:
8874                         i40e_handle_link_event(pf, &event);
8875                         break;
8876                 case i40e_aqc_opc_send_msg_to_pf:
8877                         ret = i40e_vc_process_vf_msg(pf,
8878                                         le16_to_cpu(event.desc.retval),
8879                                         le32_to_cpu(event.desc.cookie_high),
8880                                         le32_to_cpu(event.desc.cookie_low),
8881                                         event.msg_buf,
8882                                         event.msg_len);
8883                         break;
8884                 case i40e_aqc_opc_lldp_update_mib:
8885                         dev_dbg(&pf->pdev->dev, "ARQ: Update LLDP MIB event received\n");
8886 #ifdef CONFIG_I40E_DCB
8887                         rtnl_lock();
8888                         ret = i40e_handle_lldp_event(pf, &event);
8889                         rtnl_unlock();
8890 #endif /* CONFIG_I40E_DCB */
8891                         break;
8892                 case i40e_aqc_opc_event_lan_overflow:
8893                         dev_dbg(&pf->pdev->dev, "ARQ LAN queue overflow event received\n");
8894                         i40e_handle_lan_overflow_event(pf, &event);
8895                         break;
8896                 case i40e_aqc_opc_send_msg_to_peer:
8897                         dev_info(&pf->pdev->dev, "ARQ: Msg from other pf\n");
8898                         break;
8899                 case i40e_aqc_opc_nvm_erase:
8900                 case i40e_aqc_opc_nvm_update:
8901                 case i40e_aqc_opc_oem_post_update:
8902                         i40e_debug(&pf->hw, I40E_DEBUG_NVM,
8903                                    "ARQ NVM operation 0x%04x completed\n",
8904                                    opcode);
8905                         break;
8906                 default:
8907                         dev_info(&pf->pdev->dev,
8908                                  "ARQ: Unknown event 0x%04x ignored\n",
8909                                  opcode);
8910                         break;
8911                 }
8912         } while (i++ < pf->adminq_work_limit);
8913
8914         if (i < pf->adminq_work_limit)
8915                 clear_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state);
8916
8917         /* re-enable Admin queue interrupt cause */
8918         val = rd32(hw, I40E_PFINT_ICR0_ENA);
8919         val |=  I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
8920         wr32(hw, I40E_PFINT_ICR0_ENA, val);
8921         i40e_flush(hw);
8922
8923         kfree(event.msg_buf);
8924 }
8925
8926 /**
8927  * i40e_verify_eeprom - make sure eeprom is good to use
8928  * @pf: board private structure
8929  **/
8930 static void i40e_verify_eeprom(struct i40e_pf *pf)
8931 {
8932         int err;
8933
8934         err = i40e_diag_eeprom_test(&pf->hw);
8935         if (err) {
8936                 /* retry in case of garbage read */
8937                 err = i40e_diag_eeprom_test(&pf->hw);
8938                 if (err) {
8939                         dev_info(&pf->pdev->dev, "eeprom check failed (%d), Tx/Rx traffic disabled\n",
8940                                  err);
8941                         set_bit(__I40E_BAD_EEPROM, pf->state);
8942                 }
8943         }
8944
8945         if (!err && test_bit(__I40E_BAD_EEPROM, pf->state)) {
8946                 dev_info(&pf->pdev->dev, "eeprom check passed, Tx/Rx traffic enabled\n");
8947                 clear_bit(__I40E_BAD_EEPROM, pf->state);
8948         }
8949 }
8950
8951 /**
8952  * i40e_enable_pf_switch_lb
8953  * @pf: pointer to the PF structure
8954  *
8955  * enable switch loop back or die - no point in a return value
8956  **/
8957 static void i40e_enable_pf_switch_lb(struct i40e_pf *pf)
8958 {
8959         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
8960         struct i40e_vsi_context ctxt;
8961         int ret;
8962
8963         ctxt.seid = pf->main_vsi_seid;
8964         ctxt.pf_num = pf->hw.pf_id;
8965         ctxt.vf_num = 0;
8966         ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL);
8967         if (ret) {
8968                 dev_info(&pf->pdev->dev,
8969                          "couldn't get PF vsi config, err %s aq_err %s\n",
8970                          i40e_stat_str(&pf->hw, ret),
8971                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
8972                 return;
8973         }
8974         ctxt.flags = I40E_AQ_VSI_TYPE_PF;
8975         ctxt.info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
8976         ctxt.info.switch_id |= cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
8977
8978         ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
8979         if (ret) {
8980                 dev_info(&pf->pdev->dev,
8981                          "update vsi switch failed, err %s aq_err %s\n",
8982                          i40e_stat_str(&pf->hw, ret),
8983                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
8984         }
8985 }
8986
8987 /**
8988  * i40e_disable_pf_switch_lb
8989  * @pf: pointer to the PF structure
8990  *
8991  * disable switch loop back or die - no point in a return value
8992  **/
8993 static void i40e_disable_pf_switch_lb(struct i40e_pf *pf)
8994 {
8995         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
8996         struct i40e_vsi_context ctxt;
8997         int ret;
8998
8999         ctxt.seid = pf->main_vsi_seid;
9000         ctxt.pf_num = pf->hw.pf_id;
9001         ctxt.vf_num = 0;
9002         ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL);
9003         if (ret) {
9004                 dev_info(&pf->pdev->dev,
9005                          "couldn't get PF vsi config, err %s aq_err %s\n",
9006                          i40e_stat_str(&pf->hw, ret),
9007                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
9008                 return;
9009         }
9010         ctxt.flags = I40E_AQ_VSI_TYPE_PF;
9011         ctxt.info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
9012         ctxt.info.switch_id &= ~cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
9013
9014         ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
9015         if (ret) {
9016                 dev_info(&pf->pdev->dev,
9017                          "update vsi switch failed, err %s aq_err %s\n",
9018                          i40e_stat_str(&pf->hw, ret),
9019                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
9020         }
9021 }
9022
9023 /**
9024  * i40e_config_bridge_mode - Configure the HW bridge mode
9025  * @veb: pointer to the bridge instance
9026  *
9027  * Configure the loop back mode for the LAN VSI that is downlink to the
9028  * specified HW bridge instance. It is expected this function is called
9029  * when a new HW bridge is instantiated.
9030  **/
9031 static void i40e_config_bridge_mode(struct i40e_veb *veb)
9032 {
9033         struct i40e_pf *pf = veb->pf;
9034
9035         if (pf->hw.debug_mask & I40E_DEBUG_LAN)
9036                 dev_info(&pf->pdev->dev, "enabling bridge mode: %s\n",
9037                          veb->bridge_mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB");
9038         if (veb->bridge_mode & BRIDGE_MODE_VEPA)
9039                 i40e_disable_pf_switch_lb(pf);
9040         else
9041                 i40e_enable_pf_switch_lb(pf);
9042 }
9043
9044 /**
9045  * i40e_reconstitute_veb - rebuild the VEB and anything connected to it
9046  * @veb: pointer to the VEB instance
9047  *
9048  * This is a recursive function that first builds the attached VSIs then
9049  * recurses in to build the next layer of VEB.  We track the connections
9050  * through our own index numbers because the seid's from the HW could
9051  * change across the reset.
9052  **/
9053 static int i40e_reconstitute_veb(struct i40e_veb *veb)
9054 {
9055         struct i40e_vsi *ctl_vsi = NULL;
9056         struct i40e_pf *pf = veb->pf;
9057         int v, veb_idx;
9058         int ret;
9059
9060         /* build VSI that owns this VEB, temporarily attached to base VEB */
9061         for (v = 0; v < pf->num_alloc_vsi && !ctl_vsi; v++) {
9062                 if (pf->vsi[v] &&
9063                     pf->vsi[v]->veb_idx == veb->idx &&
9064                     pf->vsi[v]->flags & I40E_VSI_FLAG_VEB_OWNER) {
9065                         ctl_vsi = pf->vsi[v];
9066                         break;
9067                 }
9068         }
9069         if (!ctl_vsi) {
9070                 dev_info(&pf->pdev->dev,
9071                          "missing owner VSI for veb_idx %d\n", veb->idx);
9072                 ret = -ENOENT;
9073                 goto end_reconstitute;
9074         }
9075         if (ctl_vsi != pf->vsi[pf->lan_vsi])
9076                 ctl_vsi->uplink_seid = pf->vsi[pf->lan_vsi]->uplink_seid;
9077         ret = i40e_add_vsi(ctl_vsi);
9078         if (ret) {
9079                 dev_info(&pf->pdev->dev,
9080                          "rebuild of veb_idx %d owner VSI failed: %d\n",
9081                          veb->idx, ret);
9082                 goto end_reconstitute;
9083         }
9084         i40e_vsi_reset_stats(ctl_vsi);
9085
9086         /* create the VEB in the switch and move the VSI onto the VEB */
9087         ret = i40e_add_veb(veb, ctl_vsi);
9088         if (ret)
9089                 goto end_reconstitute;
9090
9091         if (pf->flags & I40E_FLAG_VEB_MODE_ENABLED)
9092                 veb->bridge_mode = BRIDGE_MODE_VEB;
9093         else
9094                 veb->bridge_mode = BRIDGE_MODE_VEPA;
9095         i40e_config_bridge_mode(veb);
9096
9097         /* create the remaining VSIs attached to this VEB */
9098         for (v = 0; v < pf->num_alloc_vsi; v++) {
9099                 if (!pf->vsi[v] || pf->vsi[v] == ctl_vsi)
9100                         continue;
9101
9102                 if (pf->vsi[v]->veb_idx == veb->idx) {
9103                         struct i40e_vsi *vsi = pf->vsi[v];
9104
9105                         vsi->uplink_seid = veb->seid;
9106                         ret = i40e_add_vsi(vsi);
9107                         if (ret) {
9108                                 dev_info(&pf->pdev->dev,
9109                                          "rebuild of vsi_idx %d failed: %d\n",
9110                                          v, ret);
9111                                 goto end_reconstitute;
9112                         }
9113                         i40e_vsi_reset_stats(vsi);
9114                 }
9115         }
9116
9117         /* create any VEBs attached to this VEB - RECURSION */
9118         for (veb_idx = 0; veb_idx < I40E_MAX_VEB; veb_idx++) {
9119                 if (pf->veb[veb_idx] && pf->veb[veb_idx]->veb_idx == veb->idx) {
9120                         pf->veb[veb_idx]->uplink_seid = veb->seid;
9121                         ret = i40e_reconstitute_veb(pf->veb[veb_idx]);
9122                         if (ret)
9123                                 break;
9124                 }
9125         }
9126
9127 end_reconstitute:
9128         return ret;
9129 }
9130
9131 /**
9132  * i40e_get_capabilities - get info about the HW
9133  * @pf: the PF struct
9134  **/
9135 static int i40e_get_capabilities(struct i40e_pf *pf,
9136                                  enum i40e_admin_queue_opc list_type)
9137 {
9138         struct i40e_aqc_list_capabilities_element_resp *cap_buf;
9139         u16 data_size;
9140         int buf_len;
9141         int err;
9142
9143         buf_len = 40 * sizeof(struct i40e_aqc_list_capabilities_element_resp);
9144         do {
9145                 cap_buf = kzalloc(buf_len, GFP_KERNEL);
9146                 if (!cap_buf)
9147                         return -ENOMEM;
9148
9149                 /* this loads the data into the hw struct for us */
9150                 err = i40e_aq_discover_capabilities(&pf->hw, cap_buf, buf_len,
9151                                                     &data_size, list_type,
9152                                                     NULL);
9153                 /* data loaded, buffer no longer needed */
9154                 kfree(cap_buf);
9155
9156                 if (pf->hw.aq.asq_last_status == I40E_AQ_RC_ENOMEM) {
9157                         /* retry with a larger buffer */
9158                         buf_len = data_size;
9159                 } else if (pf->hw.aq.asq_last_status != I40E_AQ_RC_OK || err) {
9160                         dev_info(&pf->pdev->dev,
9161                                  "capability discovery failed, err %s aq_err %s\n",
9162                                  i40e_stat_str(&pf->hw, err),
9163                                  i40e_aq_str(&pf->hw,
9164                                              pf->hw.aq.asq_last_status));
9165                         return -ENODEV;
9166                 }
9167         } while (err);
9168
9169         if (pf->hw.debug_mask & I40E_DEBUG_USER) {
9170                 if (list_type == i40e_aqc_opc_list_func_capabilities) {
9171                         dev_info(&pf->pdev->dev,
9172                                  "pf=%d, num_vfs=%d, msix_pf=%d, msix_vf=%d, fd_g=%d, fd_b=%d, pf_max_q=%d num_vsi=%d\n",
9173                                  pf->hw.pf_id, pf->hw.func_caps.num_vfs,
9174                                  pf->hw.func_caps.num_msix_vectors,
9175                                  pf->hw.func_caps.num_msix_vectors_vf,
9176                                  pf->hw.func_caps.fd_filters_guaranteed,
9177                                  pf->hw.func_caps.fd_filters_best_effort,
9178                                  pf->hw.func_caps.num_tx_qp,
9179                                  pf->hw.func_caps.num_vsis);
9180                 } else if (list_type == i40e_aqc_opc_list_dev_capabilities) {
9181                         dev_info(&pf->pdev->dev,
9182                                  "switch_mode=0x%04x, function_valid=0x%08x\n",
9183                                  pf->hw.dev_caps.switch_mode,
9184                                  pf->hw.dev_caps.valid_functions);
9185                         dev_info(&pf->pdev->dev,
9186                                  "SR-IOV=%d, num_vfs for all function=%u\n",
9187                                  pf->hw.dev_caps.sr_iov_1_1,
9188                                  pf->hw.dev_caps.num_vfs);
9189                         dev_info(&pf->pdev->dev,
9190                                  "num_vsis=%u, num_rx:%u, num_tx=%u\n",
9191                                  pf->hw.dev_caps.num_vsis,
9192                                  pf->hw.dev_caps.num_rx_qp,
9193                                  pf->hw.dev_caps.num_tx_qp);
9194                 }
9195         }
9196         if (list_type == i40e_aqc_opc_list_func_capabilities) {
9197 #define DEF_NUM_VSI (1 + (pf->hw.func_caps.fcoe ? 1 : 0) \
9198                        + pf->hw.func_caps.num_vfs)
9199                 if (pf->hw.revision_id == 0 &&
9200                     pf->hw.func_caps.num_vsis < DEF_NUM_VSI) {
9201                         dev_info(&pf->pdev->dev,
9202                                  "got num_vsis %d, setting num_vsis to %d\n",
9203                                  pf->hw.func_caps.num_vsis, DEF_NUM_VSI);
9204                         pf->hw.func_caps.num_vsis = DEF_NUM_VSI;
9205                 }
9206         }
9207         return 0;
9208 }
9209
9210 static int i40e_vsi_clear(struct i40e_vsi *vsi);
9211
9212 /**
9213  * i40e_fdir_sb_setup - initialize the Flow Director resources for Sideband
9214  * @pf: board private structure
9215  **/
9216 static void i40e_fdir_sb_setup(struct i40e_pf *pf)
9217 {
9218         struct i40e_vsi *vsi;
9219
9220         /* quick workaround for an NVM issue that leaves a critical register
9221          * uninitialized
9222          */
9223         if (!rd32(&pf->hw, I40E_GLQF_HKEY(0))) {
9224                 static const u32 hkey[] = {
9225                         0xe640d33f, 0xcdfe98ab, 0x73fa7161, 0x0d7a7d36,
9226                         0xeacb7d61, 0xaa4f05b6, 0x9c5c89ed, 0xfc425ddb,
9227                         0xa4654832, 0xfc7461d4, 0x8f827619, 0xf5c63c21,
9228                         0x95b3a76d};
9229                 int i;
9230
9231                 for (i = 0; i <= I40E_GLQF_HKEY_MAX_INDEX; i++)
9232                         wr32(&pf->hw, I40E_GLQF_HKEY(i), hkey[i]);
9233         }
9234
9235         if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED))
9236                 return;
9237
9238         /* find existing VSI and see if it needs configuring */
9239         vsi = i40e_find_vsi_by_type(pf, I40E_VSI_FDIR);
9240
9241         /* create a new VSI if none exists */
9242         if (!vsi) {
9243                 vsi = i40e_vsi_setup(pf, I40E_VSI_FDIR,
9244                                      pf->vsi[pf->lan_vsi]->seid, 0);
9245                 if (!vsi) {
9246                         dev_info(&pf->pdev->dev, "Couldn't create FDir VSI\n");
9247                         pf->flags &= ~I40E_FLAG_FD_SB_ENABLED;
9248                         pf->flags |= I40E_FLAG_FD_SB_INACTIVE;
9249                         return;
9250                 }
9251         }
9252
9253         i40e_vsi_setup_irqhandler(vsi, i40e_fdir_clean_ring);
9254 }
9255
9256 /**
9257  * i40e_fdir_teardown - release the Flow Director resources
9258  * @pf: board private structure
9259  **/
9260 static void i40e_fdir_teardown(struct i40e_pf *pf)
9261 {
9262         struct i40e_vsi *vsi;
9263
9264         i40e_fdir_filter_exit(pf);
9265         vsi = i40e_find_vsi_by_type(pf, I40E_VSI_FDIR);
9266         if (vsi)
9267                 i40e_vsi_release(vsi);
9268 }
9269
9270 /**
9271  * i40e_rebuild_cloud_filters - Rebuilds cloud filters for VSIs
9272  * @vsi: PF main vsi
9273  * @seid: seid of main or channel VSIs
9274  *
9275  * Rebuilds cloud filters associated with main VSI and channel VSIs if they
9276  * existed before reset
9277  **/
9278 static int i40e_rebuild_cloud_filters(struct i40e_vsi *vsi, u16 seid)
9279 {
9280         struct i40e_cloud_filter *cfilter;
9281         struct i40e_pf *pf = vsi->back;
9282         struct hlist_node *node;
9283         i40e_status ret;
9284
9285         /* Add cloud filters back if they exist */
9286         hlist_for_each_entry_safe(cfilter, node, &pf->cloud_filter_list,
9287                                   cloud_node) {
9288                 if (cfilter->seid != seid)
9289                         continue;
9290
9291                 if (cfilter->dst_port)
9292                         ret = i40e_add_del_cloud_filter_big_buf(vsi, cfilter,
9293                                                                 true);
9294                 else
9295                         ret = i40e_add_del_cloud_filter(vsi, cfilter, true);
9296
9297                 if (ret) {
9298                         dev_dbg(&pf->pdev->dev,
9299                                 "Failed to rebuild cloud filter, err %s aq_err %s\n",
9300                                 i40e_stat_str(&pf->hw, ret),
9301                                 i40e_aq_str(&pf->hw,
9302                                             pf->hw.aq.asq_last_status));
9303                         return ret;
9304                 }
9305         }
9306         return 0;
9307 }
9308
9309 /**
9310  * i40e_rebuild_channels - Rebuilds channel VSIs if they existed before reset
9311  * @vsi: PF main vsi
9312  *
9313  * Rebuilds channel VSIs if they existed before reset
9314  **/
9315 static int i40e_rebuild_channels(struct i40e_vsi *vsi)
9316 {
9317         struct i40e_channel *ch, *ch_tmp;
9318         i40e_status ret;
9319
9320         if (list_empty(&vsi->ch_list))
9321                 return 0;
9322
9323         list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) {
9324                 if (!ch->initialized)
9325                         break;
9326                 /* Proceed with creation of channel (VMDq2) VSI */
9327                 ret = i40e_add_channel(vsi->back, vsi->uplink_seid, ch);
9328                 if (ret) {
9329                         dev_info(&vsi->back->pdev->dev,
9330                                  "failed to rebuild channels using uplink_seid %u\n",
9331                                  vsi->uplink_seid);
9332                         return ret;
9333                 }
9334                 /* Reconfigure TX queues using QTX_CTL register */
9335                 ret = i40e_channel_config_tx_ring(vsi->back, vsi, ch);
9336                 if (ret) {
9337                         dev_info(&vsi->back->pdev->dev,
9338                                  "failed to configure TX rings for channel %u\n",
9339                                  ch->seid);
9340                         return ret;
9341                 }
9342                 /* update 'next_base_queue' */
9343                 vsi->next_base_queue = vsi->next_base_queue +
9344                                                         ch->num_queue_pairs;
9345                 if (ch->max_tx_rate) {
9346                         u64 credits = ch->max_tx_rate;
9347
9348                         if (i40e_set_bw_limit(vsi, ch->seid,
9349                                               ch->max_tx_rate))
9350                                 return -EINVAL;
9351
9352                         do_div(credits, I40E_BW_CREDIT_DIVISOR);
9353                         dev_dbg(&vsi->back->pdev->dev,
9354                                 "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n",
9355                                 ch->max_tx_rate,
9356                                 credits,
9357                                 ch->seid);
9358                 }
9359                 ret = i40e_rebuild_cloud_filters(vsi, ch->seid);
9360                 if (ret) {
9361                         dev_dbg(&vsi->back->pdev->dev,
9362                                 "Failed to rebuild cloud filters for channel VSI %u\n",
9363                                 ch->seid);
9364                         return ret;
9365                 }
9366         }
9367         return 0;
9368 }
9369
9370 /**
9371  * i40e_clean_xps_state - clean xps state for every tx_ring
9372  * @vsi: ptr to the VSI
9373  **/
9374 static void i40e_clean_xps_state(struct i40e_vsi *vsi)
9375 {
9376         int i;
9377
9378         if (vsi->tx_rings)
9379                 for (i = 0; i < vsi->num_queue_pairs; i++)
9380                         if (vsi->tx_rings[i])
9381                                 clear_bit(__I40E_TX_XPS_INIT_DONE,
9382                                           vsi->tx_rings[i]->state);
9383 }
9384
9385 /**
9386  * i40e_prep_for_reset - prep for the core to reset
9387  * @pf: board private structure
9388  * @lock_acquired: indicates whether or not the lock has been acquired
9389  * before this function was called.
9390  *
9391  * Close up the VFs and other things in prep for PF Reset.
9392   **/
9393 static void i40e_prep_for_reset(struct i40e_pf *pf, bool lock_acquired)
9394 {
9395         struct i40e_hw *hw = &pf->hw;
9396         i40e_status ret = 0;
9397         u32 v;
9398
9399         clear_bit(__I40E_RESET_INTR_RECEIVED, pf->state);
9400         if (test_and_set_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
9401                 return;
9402         if (i40e_check_asq_alive(&pf->hw))
9403                 i40e_vc_notify_reset(pf);
9404
9405         dev_dbg(&pf->pdev->dev, "Tearing down internal switch for reset\n");
9406
9407         /* quiesce the VSIs and their queues that are not already DOWN */
9408         /* pf_quiesce_all_vsi modifies netdev structures -rtnl_lock needed */
9409         if (!lock_acquired)
9410                 rtnl_lock();
9411         i40e_pf_quiesce_all_vsi(pf);
9412         if (!lock_acquired)
9413                 rtnl_unlock();
9414
9415         for (v = 0; v < pf->num_alloc_vsi; v++) {
9416                 if (pf->vsi[v]) {
9417                         i40e_clean_xps_state(pf->vsi[v]);
9418                         pf->vsi[v]->seid = 0;
9419                 }
9420         }
9421
9422         i40e_shutdown_adminq(&pf->hw);
9423
9424         /* call shutdown HMC */
9425         if (hw->hmc.hmc_obj) {
9426                 ret = i40e_shutdown_lan_hmc(hw);
9427                 if (ret)
9428                         dev_warn(&pf->pdev->dev,
9429                                  "shutdown_lan_hmc failed: %d\n", ret);
9430         }
9431 }
9432
9433 /**
9434  * i40e_send_version - update firmware with driver version
9435  * @pf: PF struct
9436  */
9437 static void i40e_send_version(struct i40e_pf *pf)
9438 {
9439         struct i40e_driver_version dv;
9440
9441         dv.major_version = DRV_VERSION_MAJOR;
9442         dv.minor_version = DRV_VERSION_MINOR;
9443         dv.build_version = DRV_VERSION_BUILD;
9444         dv.subbuild_version = 0;
9445         strlcpy(dv.driver_string, DRV_VERSION, sizeof(dv.driver_string));
9446         i40e_aq_send_driver_version(&pf->hw, &dv, NULL);
9447 }
9448
9449 /**
9450  * i40e_get_oem_version - get OEM specific version information
9451  * @hw: pointer to the hardware structure
9452  **/
9453 static void i40e_get_oem_version(struct i40e_hw *hw)
9454 {
9455         u16 block_offset = 0xffff;
9456         u16 block_length = 0;
9457         u16 capabilities = 0;
9458         u16 gen_snap = 0;
9459         u16 release = 0;
9460
9461 #define I40E_SR_NVM_OEM_VERSION_PTR             0x1B
9462 #define I40E_NVM_OEM_LENGTH_OFFSET              0x00
9463 #define I40E_NVM_OEM_CAPABILITIES_OFFSET        0x01
9464 #define I40E_NVM_OEM_GEN_OFFSET                 0x02
9465 #define I40E_NVM_OEM_RELEASE_OFFSET             0x03
9466 #define I40E_NVM_OEM_CAPABILITIES_MASK          0x000F
9467 #define I40E_NVM_OEM_LENGTH                     3
9468
9469         /* Check if pointer to OEM version block is valid. */
9470         i40e_read_nvm_word(hw, I40E_SR_NVM_OEM_VERSION_PTR, &block_offset);
9471         if (block_offset == 0xffff)
9472                 return;
9473
9474         /* Check if OEM version block has correct length. */
9475         i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_LENGTH_OFFSET,
9476                            &block_length);
9477         if (block_length < I40E_NVM_OEM_LENGTH)
9478                 return;
9479
9480         /* Check if OEM version format is as expected. */
9481         i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_CAPABILITIES_OFFSET,
9482                            &capabilities);
9483         if ((capabilities & I40E_NVM_OEM_CAPABILITIES_MASK) != 0)
9484                 return;
9485
9486         i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_GEN_OFFSET,
9487                            &gen_snap);
9488         i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_RELEASE_OFFSET,
9489                            &release);
9490         hw->nvm.oem_ver = (gen_snap << I40E_OEM_SNAP_SHIFT) | release;
9491         hw->nvm.eetrack = I40E_OEM_EETRACK_ID;
9492 }
9493
9494 /**
9495  * i40e_reset - wait for core reset to finish reset, reset pf if corer not seen
9496  * @pf: board private structure
9497  **/
9498 static int i40e_reset(struct i40e_pf *pf)
9499 {
9500         struct i40e_hw *hw = &pf->hw;
9501         i40e_status ret;
9502
9503         ret = i40e_pf_reset(hw);
9504         if (ret) {
9505                 dev_info(&pf->pdev->dev, "PF reset failed, %d\n", ret);
9506                 set_bit(__I40E_RESET_FAILED, pf->state);
9507                 clear_bit(__I40E_RESET_RECOVERY_PENDING, pf->state);
9508         } else {
9509                 pf->pfr_count++;
9510         }
9511         return ret;
9512 }
9513
9514 /**
9515  * i40e_rebuild - rebuild using a saved config
9516  * @pf: board private structure
9517  * @reinit: if the Main VSI needs to re-initialized.
9518  * @lock_acquired: indicates whether or not the lock has been acquired
9519  * before this function was called.
9520  **/
9521 static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired)
9522 {
9523         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
9524         struct i40e_hw *hw = &pf->hw;
9525         i40e_status ret;
9526         u32 val;
9527         int v;
9528
9529         if (test_bit(__I40E_DOWN, pf->state))
9530                 goto clear_recovery;
9531         dev_dbg(&pf->pdev->dev, "Rebuilding internal switch\n");
9532
9533         /* rebuild the basics for the AdminQ, HMC, and initial HW switch */
9534         ret = i40e_init_adminq(&pf->hw);
9535         if (ret) {
9536                 dev_info(&pf->pdev->dev, "Rebuild AdminQ failed, err %s aq_err %s\n",
9537                          i40e_stat_str(&pf->hw, ret),
9538                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
9539                 goto clear_recovery;
9540         }
9541         i40e_get_oem_version(&pf->hw);
9542
9543         if (test_and_clear_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state)) {
9544                 /* The following delay is necessary for firmware update. */
9545                 mdelay(1000);
9546         }
9547
9548         /* re-verify the eeprom if we just had an EMP reset */
9549         if (test_and_clear_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state))
9550                 i40e_verify_eeprom(pf);
9551
9552         i40e_clear_pxe_mode(hw);
9553         ret = i40e_get_capabilities(pf, i40e_aqc_opc_list_func_capabilities);
9554         if (ret)
9555                 goto end_core_reset;
9556
9557         ret = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp,
9558                                 hw->func_caps.num_rx_qp, 0, 0);
9559         if (ret) {
9560                 dev_info(&pf->pdev->dev, "init_lan_hmc failed: %d\n", ret);
9561                 goto end_core_reset;
9562         }
9563         ret = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY);
9564         if (ret) {
9565                 dev_info(&pf->pdev->dev, "configure_lan_hmc failed: %d\n", ret);
9566                 goto end_core_reset;
9567         }
9568
9569         /* Enable FW to write a default DCB config on link-up */
9570         i40e_aq_set_dcb_parameters(hw, true, NULL);
9571
9572 #ifdef CONFIG_I40E_DCB
9573         ret = i40e_init_pf_dcb(pf);
9574         if (ret) {
9575                 dev_info(&pf->pdev->dev, "DCB init failed %d, disabled\n", ret);
9576                 pf->flags &= ~I40E_FLAG_DCB_CAPABLE;
9577                 /* Continue without DCB enabled */
9578         }
9579 #endif /* CONFIG_I40E_DCB */
9580         /* do basic switch setup */
9581         if (!lock_acquired)
9582                 rtnl_lock();
9583         ret = i40e_setup_pf_switch(pf, reinit);
9584         if (ret)
9585                 goto end_unlock;
9586
9587         /* The driver only wants link up/down and module qualification
9588          * reports from firmware.  Note the negative logic.
9589          */
9590         ret = i40e_aq_set_phy_int_mask(&pf->hw,
9591                                        ~(I40E_AQ_EVENT_LINK_UPDOWN |
9592                                          I40E_AQ_EVENT_MEDIA_NA |
9593                                          I40E_AQ_EVENT_MODULE_QUAL_FAIL), NULL);
9594         if (ret)
9595                 dev_info(&pf->pdev->dev, "set phy mask fail, err %s aq_err %s\n",
9596                          i40e_stat_str(&pf->hw, ret),
9597                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
9598
9599         /* Rebuild the VSIs and VEBs that existed before reset.
9600          * They are still in our local switch element arrays, so only
9601          * need to rebuild the switch model in the HW.
9602          *
9603          * If there were VEBs but the reconstitution failed, we'll try
9604          * try to recover minimal use by getting the basic PF VSI working.
9605          */
9606         if (vsi->uplink_seid != pf->mac_seid) {
9607                 dev_dbg(&pf->pdev->dev, "attempting to rebuild switch\n");
9608                 /* find the one VEB connected to the MAC, and find orphans */
9609                 for (v = 0; v < I40E_MAX_VEB; v++) {
9610                         if (!pf->veb[v])
9611                                 continue;
9612
9613                         if (pf->veb[v]->uplink_seid == pf->mac_seid ||
9614                             pf->veb[v]->uplink_seid == 0) {
9615                                 ret = i40e_reconstitute_veb(pf->veb[v]);
9616
9617                                 if (!ret)
9618                                         continue;
9619
9620                                 /* If Main VEB failed, we're in deep doodoo,
9621                                  * so give up rebuilding the switch and set up
9622                                  * for minimal rebuild of PF VSI.
9623                                  * If orphan failed, we'll report the error
9624                                  * but try to keep going.
9625                                  */
9626                                 if (pf->veb[v]->uplink_seid == pf->mac_seid) {
9627                                         dev_info(&pf->pdev->dev,
9628                                                  "rebuild of switch failed: %d, will try to set up simple PF connection\n",
9629                                                  ret);
9630                                         vsi->uplink_seid = pf->mac_seid;
9631                                         break;
9632                                 } else if (pf->veb[v]->uplink_seid == 0) {
9633                                         dev_info(&pf->pdev->dev,
9634                                                  "rebuild of orphan VEB failed: %d\n",
9635                                                  ret);
9636                                 }
9637                         }
9638                 }
9639         }
9640
9641         if (vsi->uplink_seid == pf->mac_seid) {
9642                 dev_dbg(&pf->pdev->dev, "attempting to rebuild PF VSI\n");
9643                 /* no VEB, so rebuild only the Main VSI */
9644                 ret = i40e_add_vsi(vsi);
9645                 if (ret) {
9646                         dev_info(&pf->pdev->dev,
9647                                  "rebuild of Main VSI failed: %d\n", ret);
9648                         goto end_unlock;
9649                 }
9650         }
9651
9652         if (vsi->mqprio_qopt.max_rate[0]) {
9653                 u64 max_tx_rate = i40e_bw_bytes_to_mbits(vsi,
9654                                                   vsi->mqprio_qopt.max_rate[0]);
9655                 u64 credits = 0;
9656
9657                 ret = i40e_set_bw_limit(vsi, vsi->seid, max_tx_rate);
9658                 if (ret)
9659                         goto end_unlock;
9660
9661                 credits = max_tx_rate;
9662                 do_div(credits, I40E_BW_CREDIT_DIVISOR);
9663                 dev_dbg(&vsi->back->pdev->dev,
9664                         "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n",
9665                         max_tx_rate,
9666                         credits,
9667                         vsi->seid);
9668         }
9669
9670         ret = i40e_rebuild_cloud_filters(vsi, vsi->seid);
9671         if (ret)
9672                 goto end_unlock;
9673
9674         /* PF Main VSI is rebuild by now, go ahead and rebuild channel VSIs
9675          * for this main VSI if they exist
9676          */
9677         ret = i40e_rebuild_channels(vsi);
9678         if (ret)
9679                 goto end_unlock;
9680
9681         /* Reconfigure hardware for allowing smaller MSS in the case
9682          * of TSO, so that we avoid the MDD being fired and causing
9683          * a reset in the case of small MSS+TSO.
9684          */
9685 #define I40E_REG_MSS          0x000E64DC
9686 #define I40E_REG_MSS_MIN_MASK 0x3FF0000
9687 #define I40E_64BYTE_MSS       0x400000
9688         val = rd32(hw, I40E_REG_MSS);
9689         if ((val & I40E_REG_MSS_MIN_MASK) > I40E_64BYTE_MSS) {
9690                 val &= ~I40E_REG_MSS_MIN_MASK;
9691                 val |= I40E_64BYTE_MSS;
9692                 wr32(hw, I40E_REG_MSS, val);
9693         }
9694
9695         if (pf->hw_features & I40E_HW_RESTART_AUTONEG) {
9696                 msleep(75);
9697                 ret = i40e_aq_set_link_restart_an(&pf->hw, true, NULL);
9698                 if (ret)
9699                         dev_info(&pf->pdev->dev, "link restart failed, err %s aq_err %s\n",
9700                                  i40e_stat_str(&pf->hw, ret),
9701                                  i40e_aq_str(&pf->hw,
9702                                              pf->hw.aq.asq_last_status));
9703         }
9704         /* reinit the misc interrupt */
9705         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
9706                 ret = i40e_setup_misc_vector(pf);
9707                 if (ret)
9708                         goto end_unlock;
9709         }
9710
9711         /* Add a filter to drop all Flow control frames from any VSI from being
9712          * transmitted. By doing so we stop a malicious VF from sending out
9713          * PAUSE or PFC frames and potentially controlling traffic for other
9714          * PF/VF VSIs.
9715          * The FW can still send Flow control frames if enabled.
9716          */
9717         i40e_add_filter_to_drop_tx_flow_control_frames(&pf->hw,
9718                                                        pf->main_vsi_seid);
9719
9720         /* restart the VSIs that were rebuilt and running before the reset */
9721         i40e_pf_unquiesce_all_vsi(pf);
9722
9723         /* Release the RTNL lock before we start resetting VFs */
9724         if (!lock_acquired)
9725                 rtnl_unlock();
9726
9727         /* Restore promiscuous settings */
9728         ret = i40e_set_promiscuous(pf, pf->cur_promisc);
9729         if (ret)
9730                 dev_warn(&pf->pdev->dev,
9731                          "Failed to restore promiscuous setting: %s, err %s aq_err %s\n",
9732                          pf->cur_promisc ? "on" : "off",
9733                          i40e_stat_str(&pf->hw, ret),
9734                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
9735
9736         i40e_reset_all_vfs(pf, true);
9737
9738         /* tell the firmware that we're starting */
9739         i40e_send_version(pf);
9740
9741         /* We've already released the lock, so don't do it again */
9742         goto end_core_reset;
9743
9744 end_unlock:
9745         if (!lock_acquired)
9746                 rtnl_unlock();
9747 end_core_reset:
9748         clear_bit(__I40E_RESET_FAILED, pf->state);
9749 clear_recovery:
9750         clear_bit(__I40E_RESET_RECOVERY_PENDING, pf->state);
9751         clear_bit(__I40E_TIMEOUT_RECOVERY_PENDING, pf->state);
9752 }
9753
9754 /**
9755  * i40e_reset_and_rebuild - reset and rebuild using a saved config
9756  * @pf: board private structure
9757  * @reinit: if the Main VSI needs to re-initialized.
9758  * @lock_acquired: indicates whether or not the lock has been acquired
9759  * before this function was called.
9760  **/
9761 static void i40e_reset_and_rebuild(struct i40e_pf *pf, bool reinit,
9762                                    bool lock_acquired)
9763 {
9764         int ret;
9765         /* Now we wait for GRST to settle out.
9766          * We don't have to delete the VEBs or VSIs from the hw switch
9767          * because the reset will make them disappear.
9768          */
9769         ret = i40e_reset(pf);
9770         if (!ret)
9771                 i40e_rebuild(pf, reinit, lock_acquired);
9772 }
9773
9774 /**
9775  * i40e_handle_reset_warning - prep for the PF to reset, reset and rebuild
9776  * @pf: board private structure
9777  *
9778  * Close up the VFs and other things in prep for a Core Reset,
9779  * then get ready to rebuild the world.
9780  * @lock_acquired: indicates whether or not the lock has been acquired
9781  * before this function was called.
9782  **/
9783 static void i40e_handle_reset_warning(struct i40e_pf *pf, bool lock_acquired)
9784 {
9785         i40e_prep_for_reset(pf, lock_acquired);
9786         i40e_reset_and_rebuild(pf, false, lock_acquired);
9787 }
9788
9789 /**
9790  * i40e_handle_mdd_event
9791  * @pf: pointer to the PF structure
9792  *
9793  * Called from the MDD irq handler to identify possibly malicious vfs
9794  **/
9795 static void i40e_handle_mdd_event(struct i40e_pf *pf)
9796 {
9797         struct i40e_hw *hw = &pf->hw;
9798         bool mdd_detected = false;
9799         bool pf_mdd_detected = false;
9800         struct i40e_vf *vf;
9801         u32 reg;
9802         int i;
9803
9804         if (!test_bit(__I40E_MDD_EVENT_PENDING, pf->state))
9805                 return;
9806
9807         /* find what triggered the MDD event */
9808         reg = rd32(hw, I40E_GL_MDET_TX);
9809         if (reg & I40E_GL_MDET_TX_VALID_MASK) {
9810                 u8 pf_num = (reg & I40E_GL_MDET_TX_PF_NUM_MASK) >>
9811                                 I40E_GL_MDET_TX_PF_NUM_SHIFT;
9812                 u16 vf_num = (reg & I40E_GL_MDET_TX_VF_NUM_MASK) >>
9813                                 I40E_GL_MDET_TX_VF_NUM_SHIFT;
9814                 u8 event = (reg & I40E_GL_MDET_TX_EVENT_MASK) >>
9815                                 I40E_GL_MDET_TX_EVENT_SHIFT;
9816                 u16 queue = ((reg & I40E_GL_MDET_TX_QUEUE_MASK) >>
9817                                 I40E_GL_MDET_TX_QUEUE_SHIFT) -
9818                                 pf->hw.func_caps.base_queue;
9819                 if (netif_msg_tx_err(pf))
9820                         dev_info(&pf->pdev->dev, "Malicious Driver Detection event 0x%02x on TX queue %d PF number 0x%02x VF number 0x%02x\n",
9821                                  event, queue, pf_num, vf_num);
9822                 wr32(hw, I40E_GL_MDET_TX, 0xffffffff);
9823                 mdd_detected = true;
9824         }
9825         reg = rd32(hw, I40E_GL_MDET_RX);
9826         if (reg & I40E_GL_MDET_RX_VALID_MASK) {
9827                 u8 func = (reg & I40E_GL_MDET_RX_FUNCTION_MASK) >>
9828                                 I40E_GL_MDET_RX_FUNCTION_SHIFT;
9829                 u8 event = (reg & I40E_GL_MDET_RX_EVENT_MASK) >>
9830                                 I40E_GL_MDET_RX_EVENT_SHIFT;
9831                 u16 queue = ((reg & I40E_GL_MDET_RX_QUEUE_MASK) >>
9832                                 I40E_GL_MDET_RX_QUEUE_SHIFT) -
9833                                 pf->hw.func_caps.base_queue;
9834                 if (netif_msg_rx_err(pf))
9835                         dev_info(&pf->pdev->dev, "Malicious Driver Detection event 0x%02x on RX queue %d of function 0x%02x\n",
9836                                  event, queue, func);
9837                 wr32(hw, I40E_GL_MDET_RX, 0xffffffff);
9838                 mdd_detected = true;
9839         }
9840
9841         if (mdd_detected) {
9842                 reg = rd32(hw, I40E_PF_MDET_TX);
9843                 if (reg & I40E_PF_MDET_TX_VALID_MASK) {
9844                         wr32(hw, I40E_PF_MDET_TX, 0xFFFF);
9845                         dev_info(&pf->pdev->dev, "TX driver issue detected, PF reset issued\n");
9846                         pf_mdd_detected = true;
9847                 }
9848                 reg = rd32(hw, I40E_PF_MDET_RX);
9849                 if (reg & I40E_PF_MDET_RX_VALID_MASK) {
9850                         wr32(hw, I40E_PF_MDET_RX, 0xFFFF);
9851                         dev_info(&pf->pdev->dev, "RX driver issue detected, PF reset issued\n");
9852                         pf_mdd_detected = true;
9853                 }
9854                 /* Queue belongs to the PF, initiate a reset */
9855                 if (pf_mdd_detected) {
9856                         set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
9857                         i40e_service_event_schedule(pf);
9858                 }
9859         }
9860
9861         /* see if one of the VFs needs its hand slapped */
9862         for (i = 0; i < pf->num_alloc_vfs && mdd_detected; i++) {
9863                 vf = &(pf->vf[i]);
9864                 reg = rd32(hw, I40E_VP_MDET_TX(i));
9865                 if (reg & I40E_VP_MDET_TX_VALID_MASK) {
9866                         wr32(hw, I40E_VP_MDET_TX(i), 0xFFFF);
9867                         vf->num_mdd_events++;
9868                         dev_info(&pf->pdev->dev, "TX driver issue detected on VF %d\n",
9869                                  i);
9870                 }
9871
9872                 reg = rd32(hw, I40E_VP_MDET_RX(i));
9873                 if (reg & I40E_VP_MDET_RX_VALID_MASK) {
9874                         wr32(hw, I40E_VP_MDET_RX(i), 0xFFFF);
9875                         vf->num_mdd_events++;
9876                         dev_info(&pf->pdev->dev, "RX driver issue detected on VF %d\n",
9877                                  i);
9878                 }
9879
9880                 if (vf->num_mdd_events > I40E_DEFAULT_NUM_MDD_EVENTS_ALLOWED) {
9881                         dev_info(&pf->pdev->dev,
9882                                  "Too many MDD events on VF %d, disabled\n", i);
9883                         dev_info(&pf->pdev->dev,
9884                                  "Use PF Control I/F to re-enable the VF\n");
9885                         set_bit(I40E_VF_STATE_DISABLED, &vf->vf_states);
9886                 }
9887         }
9888
9889         /* re-enable mdd interrupt cause */
9890         clear_bit(__I40E_MDD_EVENT_PENDING, pf->state);
9891         reg = rd32(hw, I40E_PFINT_ICR0_ENA);
9892         reg |=  I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK;
9893         wr32(hw, I40E_PFINT_ICR0_ENA, reg);
9894         i40e_flush(hw);
9895 }
9896
9897 static const char *i40e_tunnel_name(u8 type)
9898 {
9899         switch (type) {
9900         case UDP_TUNNEL_TYPE_VXLAN:
9901                 return "vxlan";
9902         case UDP_TUNNEL_TYPE_GENEVE:
9903                 return "geneve";
9904         default:
9905                 return "unknown";
9906         }
9907 }
9908
9909 /**
9910  * i40e_sync_udp_filters - Trigger a sync event for existing UDP filters
9911  * @pf: board private structure
9912  **/
9913 static void i40e_sync_udp_filters(struct i40e_pf *pf)
9914 {
9915         int i;
9916
9917         /* loop through and set pending bit for all active UDP filters */
9918         for (i = 0; i < I40E_MAX_PF_UDP_OFFLOAD_PORTS; i++) {
9919                 if (pf->udp_ports[i].port)
9920                         pf->pending_udp_bitmap |= BIT_ULL(i);
9921         }
9922
9923         set_bit(__I40E_UDP_FILTER_SYNC_PENDING, pf->state);
9924 }
9925
9926 /**
9927  * i40e_sync_udp_filters_subtask - Sync the VSI filter list with HW
9928  * @pf: board private structure
9929  **/
9930 static void i40e_sync_udp_filters_subtask(struct i40e_pf *pf)
9931 {
9932         struct i40e_hw *hw = &pf->hw;
9933         u8 filter_index, type;
9934         u16 port;
9935         int i;
9936
9937         if (!test_and_clear_bit(__I40E_UDP_FILTER_SYNC_PENDING, pf->state))
9938                 return;
9939
9940         /* acquire RTNL to maintain state of flags and port requests */
9941         rtnl_lock();
9942
9943         for (i = 0; i < I40E_MAX_PF_UDP_OFFLOAD_PORTS; i++) {
9944                 if (pf->pending_udp_bitmap & BIT_ULL(i)) {
9945                         struct i40e_udp_port_config *udp_port;
9946                         i40e_status ret = 0;
9947
9948                         udp_port = &pf->udp_ports[i];
9949                         pf->pending_udp_bitmap &= ~BIT_ULL(i);
9950
9951                         port = READ_ONCE(udp_port->port);
9952                         type = READ_ONCE(udp_port->type);
9953                         filter_index = READ_ONCE(udp_port->filter_index);
9954
9955                         /* release RTNL while we wait on AQ command */
9956                         rtnl_unlock();
9957
9958                         if (port)
9959                                 ret = i40e_aq_add_udp_tunnel(hw, port,
9960                                                              type,
9961                                                              &filter_index,
9962                                                              NULL);
9963                         else if (filter_index != I40E_UDP_PORT_INDEX_UNUSED)
9964                                 ret = i40e_aq_del_udp_tunnel(hw, filter_index,
9965                                                              NULL);
9966
9967                         /* reacquire RTNL so we can update filter_index */
9968                         rtnl_lock();
9969
9970                         if (ret) {
9971                                 dev_info(&pf->pdev->dev,
9972                                          "%s %s port %d, index %d failed, err %s aq_err %s\n",
9973                                          i40e_tunnel_name(type),
9974                                          port ? "add" : "delete",
9975                                          port,
9976                                          filter_index,
9977                                          i40e_stat_str(&pf->hw, ret),
9978                                          i40e_aq_str(&pf->hw,
9979                                                      pf->hw.aq.asq_last_status));
9980                                 if (port) {
9981                                         /* failed to add, just reset port,
9982                                          * drop pending bit for any deletion
9983                                          */
9984                                         udp_port->port = 0;
9985                                         pf->pending_udp_bitmap &= ~BIT_ULL(i);
9986                                 }
9987                         } else if (port) {
9988                                 /* record filter index on success */
9989                                 udp_port->filter_index = filter_index;
9990                         }
9991                 }
9992         }
9993
9994         rtnl_unlock();
9995 }
9996
9997 /**
9998  * i40e_service_task - Run the driver's async subtasks
9999  * @work: pointer to work_struct containing our data
10000  **/
10001 static void i40e_service_task(struct work_struct *work)
10002 {
10003         struct i40e_pf *pf = container_of(work,
10004                                           struct i40e_pf,
10005                                           service_task);
10006         unsigned long start_time = jiffies;
10007
10008         /* don't bother with service tasks if a reset is in progress */
10009         if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
10010                 return;
10011
10012         if (test_and_set_bit(__I40E_SERVICE_SCHED, pf->state))
10013                 return;
10014
10015         i40e_detect_recover_hung(pf->vsi[pf->lan_vsi]);
10016         i40e_sync_filters_subtask(pf);
10017         i40e_reset_subtask(pf);
10018         i40e_handle_mdd_event(pf);
10019         i40e_vc_process_vflr_event(pf);
10020         i40e_watchdog_subtask(pf);
10021         i40e_fdir_reinit_subtask(pf);
10022         if (test_and_clear_bit(__I40E_CLIENT_RESET, pf->state)) {
10023                 /* Client subtask will reopen next time through. */
10024                 i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], true);
10025         } else {
10026                 i40e_client_subtask(pf);
10027                 if (test_and_clear_bit(__I40E_CLIENT_L2_CHANGE,
10028                                        pf->state))
10029                         i40e_notify_client_of_l2_param_changes(
10030                                                         pf->vsi[pf->lan_vsi]);
10031         }
10032         i40e_sync_filters_subtask(pf);
10033         i40e_sync_udp_filters_subtask(pf);
10034         i40e_clean_adminq_subtask(pf);
10035
10036         /* flush memory to make sure state is correct before next watchdog */
10037         smp_mb__before_atomic();
10038         clear_bit(__I40E_SERVICE_SCHED, pf->state);
10039
10040         /* If the tasks have taken longer than one timer cycle or there
10041          * is more work to be done, reschedule the service task now
10042          * rather than wait for the timer to tick again.
10043          */
10044         if (time_after(jiffies, (start_time + pf->service_timer_period)) ||
10045             test_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state)             ||
10046             test_bit(__I40E_MDD_EVENT_PENDING, pf->state)                ||
10047             test_bit(__I40E_VFLR_EVENT_PENDING, pf->state))
10048                 i40e_service_event_schedule(pf);
10049 }
10050
10051 /**
10052  * i40e_service_timer - timer callback
10053  * @data: pointer to PF struct
10054  **/
10055 static void i40e_service_timer(struct timer_list *t)
10056 {
10057         struct i40e_pf *pf = from_timer(pf, t, service_timer);
10058
10059         mod_timer(&pf->service_timer,
10060                   round_jiffies(jiffies + pf->service_timer_period));
10061         i40e_service_event_schedule(pf);
10062 }
10063
10064 /**
10065  * i40e_set_num_rings_in_vsi - Determine number of rings in the VSI
10066  * @vsi: the VSI being configured
10067  **/
10068 static int i40e_set_num_rings_in_vsi(struct i40e_vsi *vsi)
10069 {
10070         struct i40e_pf *pf = vsi->back;
10071
10072         switch (vsi->type) {
10073         case I40E_VSI_MAIN:
10074                 vsi->alloc_queue_pairs = pf->num_lan_qps;
10075                 vsi->num_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
10076                                       I40E_REQ_DESCRIPTOR_MULTIPLE);
10077                 if (pf->flags & I40E_FLAG_MSIX_ENABLED)
10078                         vsi->num_q_vectors = pf->num_lan_msix;
10079                 else
10080                         vsi->num_q_vectors = 1;
10081
10082                 break;
10083
10084         case I40E_VSI_FDIR:
10085                 vsi->alloc_queue_pairs = 1;
10086                 vsi->num_desc = ALIGN(I40E_FDIR_RING_COUNT,
10087                                       I40E_REQ_DESCRIPTOR_MULTIPLE);
10088                 vsi->num_q_vectors = pf->num_fdsb_msix;
10089                 break;
10090
10091         case I40E_VSI_VMDQ2:
10092                 vsi->alloc_queue_pairs = pf->num_vmdq_qps;
10093                 vsi->num_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
10094                                       I40E_REQ_DESCRIPTOR_MULTIPLE);
10095                 vsi->num_q_vectors = pf->num_vmdq_msix;
10096                 break;
10097
10098         case I40E_VSI_SRIOV:
10099                 vsi->alloc_queue_pairs = pf->num_vf_qps;
10100                 vsi->num_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
10101                                       I40E_REQ_DESCRIPTOR_MULTIPLE);
10102                 break;
10103
10104         default:
10105                 WARN_ON(1);
10106                 return -ENODATA;
10107         }
10108
10109         return 0;
10110 }
10111
10112 /**
10113  * i40e_vsi_alloc_arrays - Allocate queue and vector pointer arrays for the vsi
10114  * @vsi: VSI pointer
10115  * @alloc_qvectors: a bool to specify if q_vectors need to be allocated.
10116  *
10117  * On error: returns error code (negative)
10118  * On success: returns 0
10119  **/
10120 static int i40e_vsi_alloc_arrays(struct i40e_vsi *vsi, bool alloc_qvectors)
10121 {
10122         struct i40e_ring **next_rings;
10123         int size;
10124         int ret = 0;
10125
10126         /* allocate memory for both Tx, XDP Tx and Rx ring pointers */
10127         size = sizeof(struct i40e_ring *) * vsi->alloc_queue_pairs *
10128                (i40e_enabled_xdp_vsi(vsi) ? 3 : 2);
10129         vsi->tx_rings = kzalloc(size, GFP_KERNEL);
10130         if (!vsi->tx_rings)
10131                 return -ENOMEM;
10132         next_rings = vsi->tx_rings + vsi->alloc_queue_pairs;
10133         if (i40e_enabled_xdp_vsi(vsi)) {
10134                 vsi->xdp_rings = next_rings;
10135                 next_rings += vsi->alloc_queue_pairs;
10136         }
10137         vsi->rx_rings = next_rings;
10138
10139         if (alloc_qvectors) {
10140                 /* allocate memory for q_vector pointers */
10141                 size = sizeof(struct i40e_q_vector *) * vsi->num_q_vectors;
10142                 vsi->q_vectors = kzalloc(size, GFP_KERNEL);
10143                 if (!vsi->q_vectors) {
10144                         ret = -ENOMEM;
10145                         goto err_vectors;
10146                 }
10147         }
10148         return ret;
10149
10150 err_vectors:
10151         kfree(vsi->tx_rings);
10152         return ret;
10153 }
10154
10155 /**
10156  * i40e_vsi_mem_alloc - Allocates the next available struct vsi in the PF
10157  * @pf: board private structure
10158  * @type: type of VSI
10159  *
10160  * On error: returns error code (negative)
10161  * On success: returns vsi index in PF (positive)
10162  **/
10163 static int i40e_vsi_mem_alloc(struct i40e_pf *pf, enum i40e_vsi_type type)
10164 {
10165         int ret = -ENODEV;
10166         struct i40e_vsi *vsi;
10167         int vsi_idx;
10168         int i;
10169
10170         /* Need to protect the allocation of the VSIs at the PF level */
10171         mutex_lock(&pf->switch_mutex);
10172
10173         /* VSI list may be fragmented if VSI creation/destruction has
10174          * been happening.  We can afford to do a quick scan to look
10175          * for any free VSIs in the list.
10176          *
10177          * find next empty vsi slot, looping back around if necessary
10178          */
10179         i = pf->next_vsi;
10180         while (i < pf->num_alloc_vsi && pf->vsi[i])
10181                 i++;
10182         if (i >= pf->num_alloc_vsi) {
10183                 i = 0;
10184                 while (i < pf->next_vsi && pf->vsi[i])
10185                         i++;
10186         }
10187
10188         if (i < pf->num_alloc_vsi && !pf->vsi[i]) {
10189                 vsi_idx = i;             /* Found one! */
10190         } else {
10191                 ret = -ENODEV;
10192                 goto unlock_pf;  /* out of VSI slots! */
10193         }
10194         pf->next_vsi = ++i;
10195
10196         vsi = kzalloc(sizeof(*vsi), GFP_KERNEL);
10197         if (!vsi) {
10198                 ret = -ENOMEM;
10199                 goto unlock_pf;
10200         }
10201         vsi->type = type;
10202         vsi->back = pf;
10203         set_bit(__I40E_VSI_DOWN, vsi->state);
10204         vsi->flags = 0;
10205         vsi->idx = vsi_idx;
10206         vsi->int_rate_limit = 0;
10207         vsi->rss_table_size = (vsi->type == I40E_VSI_MAIN) ?
10208                                 pf->rss_table_size : 64;
10209         vsi->netdev_registered = false;
10210         vsi->work_limit = I40E_DEFAULT_IRQ_WORK;
10211         hash_init(vsi->mac_filter_hash);
10212         vsi->irqs_ready = false;
10213
10214         ret = i40e_set_num_rings_in_vsi(vsi);
10215         if (ret)
10216                 goto err_rings;
10217
10218         ret = i40e_vsi_alloc_arrays(vsi, true);
10219         if (ret)
10220                 goto err_rings;
10221
10222         /* Setup default MSIX irq handler for VSI */
10223         i40e_vsi_setup_irqhandler(vsi, i40e_msix_clean_rings);
10224
10225         /* Initialize VSI lock */
10226         spin_lock_init(&vsi->mac_filter_hash_lock);
10227         pf->vsi[vsi_idx] = vsi;
10228         ret = vsi_idx;
10229         goto unlock_pf;
10230
10231 err_rings:
10232         pf->next_vsi = i - 1;
10233         kfree(vsi);
10234 unlock_pf:
10235         mutex_unlock(&pf->switch_mutex);
10236         return ret;
10237 }
10238
10239 /**
10240  * i40e_vsi_free_arrays - Free queue and vector pointer arrays for the VSI
10241  * @vsi: VSI pointer
10242  * @free_qvectors: a bool to specify if q_vectors need to be freed.
10243  *
10244  * On error: returns error code (negative)
10245  * On success: returns 0
10246  **/
10247 static void i40e_vsi_free_arrays(struct i40e_vsi *vsi, bool free_qvectors)
10248 {
10249         /* free the ring and vector containers */
10250         if (free_qvectors) {
10251                 kfree(vsi->q_vectors);
10252                 vsi->q_vectors = NULL;
10253         }
10254         kfree(vsi->tx_rings);
10255         vsi->tx_rings = NULL;
10256         vsi->rx_rings = NULL;
10257         vsi->xdp_rings = NULL;
10258 }
10259
10260 /**
10261  * i40e_clear_rss_config_user - clear the user configured RSS hash keys
10262  * and lookup table
10263  * @vsi: Pointer to VSI structure
10264  */
10265 static void i40e_clear_rss_config_user(struct i40e_vsi *vsi)
10266 {
10267         if (!vsi)
10268                 return;
10269
10270         kfree(vsi->rss_hkey_user);
10271         vsi->rss_hkey_user = NULL;
10272
10273         kfree(vsi->rss_lut_user);
10274         vsi->rss_lut_user = NULL;
10275 }
10276
10277 /**
10278  * i40e_vsi_clear - Deallocate the VSI provided
10279  * @vsi: the VSI being un-configured
10280  **/
10281 static int i40e_vsi_clear(struct i40e_vsi *vsi)
10282 {
10283         struct i40e_pf *pf;
10284
10285         if (!vsi)
10286                 return 0;
10287
10288         if (!vsi->back)
10289                 goto free_vsi;
10290         pf = vsi->back;
10291
10292         mutex_lock(&pf->switch_mutex);
10293         if (!pf->vsi[vsi->idx]) {
10294                 dev_err(&pf->pdev->dev, "pf->vsi[%d] is NULL, just free vsi[%d](type %d)\n",
10295                         vsi->idx, vsi->idx, vsi->type);
10296                 goto unlock_vsi;
10297         }
10298
10299         if (pf->vsi[vsi->idx] != vsi) {
10300                 dev_err(&pf->pdev->dev,
10301                         "pf->vsi[%d](type %d) != vsi[%d](type %d): no free!\n",
10302                         pf->vsi[vsi->idx]->idx,
10303                         pf->vsi[vsi->idx]->type,
10304                         vsi->idx, vsi->type);
10305                 goto unlock_vsi;
10306         }
10307
10308         /* updates the PF for this cleared vsi */
10309         i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx);
10310         i40e_put_lump(pf->irq_pile, vsi->base_vector, vsi->idx);
10311
10312         i40e_vsi_free_arrays(vsi, true);
10313         i40e_clear_rss_config_user(vsi);
10314
10315         pf->vsi[vsi->idx] = NULL;
10316         if (vsi->idx < pf->next_vsi)
10317                 pf->next_vsi = vsi->idx;
10318
10319 unlock_vsi:
10320         mutex_unlock(&pf->switch_mutex);
10321 free_vsi:
10322         kfree(vsi);
10323
10324         return 0;
10325 }
10326
10327 /**
10328  * i40e_vsi_clear_rings - Deallocates the Rx and Tx rings for the provided VSI
10329  * @vsi: the VSI being cleaned
10330  **/
10331 static void i40e_vsi_clear_rings(struct i40e_vsi *vsi)
10332 {
10333         int i;
10334
10335         if (vsi->tx_rings && vsi->tx_rings[0]) {
10336                 for (i = 0; i < vsi->alloc_queue_pairs; i++) {
10337                         kfree_rcu(vsi->tx_rings[i], rcu);
10338                         WRITE_ONCE(vsi->tx_rings[i], NULL);
10339                         WRITE_ONCE(vsi->rx_rings[i], NULL);
10340                         if (vsi->xdp_rings)
10341                                 WRITE_ONCE(vsi->xdp_rings[i], NULL);
10342                 }
10343         }
10344 }
10345
10346 /**
10347  * i40e_alloc_rings - Allocates the Rx and Tx rings for the provided VSI
10348  * @vsi: the VSI being configured
10349  **/
10350 static int i40e_alloc_rings(struct i40e_vsi *vsi)
10351 {
10352         int i, qpv = i40e_enabled_xdp_vsi(vsi) ? 3 : 2;
10353         struct i40e_pf *pf = vsi->back;
10354         struct i40e_ring *ring;
10355
10356         /* Set basic values in the rings to be used later during open() */
10357         for (i = 0; i < vsi->alloc_queue_pairs; i++) {
10358                 /* allocate space for both Tx and Rx in one shot */
10359                 ring = kcalloc(qpv, sizeof(struct i40e_ring), GFP_KERNEL);
10360                 if (!ring)
10361                         goto err_out;
10362
10363                 ring->queue_index = i;
10364                 ring->reg_idx = vsi->base_queue + i;
10365                 ring->ring_active = false;
10366                 ring->vsi = vsi;
10367                 ring->netdev = vsi->netdev;
10368                 ring->dev = &pf->pdev->dev;
10369                 ring->count = vsi->num_desc;
10370                 ring->size = 0;
10371                 ring->dcb_tc = 0;
10372                 if (vsi->back->hw_features & I40E_HW_WB_ON_ITR_CAPABLE)
10373                         ring->flags = I40E_TXR_FLAGS_WB_ON_ITR;
10374                 ring->itr_setting = pf->tx_itr_default;
10375                 WRITE_ONCE(vsi->tx_rings[i], ring++);
10376
10377                 if (!i40e_enabled_xdp_vsi(vsi))
10378                         goto setup_rx;
10379
10380                 ring->queue_index = vsi->alloc_queue_pairs + i;
10381                 ring->reg_idx = vsi->base_queue + ring->queue_index;
10382                 ring->ring_active = false;
10383                 ring->vsi = vsi;
10384                 ring->netdev = NULL;
10385                 ring->dev = &pf->pdev->dev;
10386                 ring->count = vsi->num_desc;
10387                 ring->size = 0;
10388                 ring->dcb_tc = 0;
10389                 if (vsi->back->hw_features & I40E_HW_WB_ON_ITR_CAPABLE)
10390                         ring->flags = I40E_TXR_FLAGS_WB_ON_ITR;
10391                 set_ring_xdp(ring);
10392                 ring->itr_setting = pf->tx_itr_default;
10393                 WRITE_ONCE(vsi->xdp_rings[i], ring++);
10394
10395 setup_rx:
10396                 ring->queue_index = i;
10397                 ring->reg_idx = vsi->base_queue + i;
10398                 ring->ring_active = false;
10399                 ring->vsi = vsi;
10400                 ring->netdev = vsi->netdev;
10401                 ring->dev = &pf->pdev->dev;
10402                 ring->count = vsi->num_desc;
10403                 ring->size = 0;
10404                 ring->dcb_tc = 0;
10405                 ring->itr_setting = pf->rx_itr_default;
10406                 WRITE_ONCE(vsi->rx_rings[i], ring);
10407         }
10408
10409         return 0;
10410
10411 err_out:
10412         i40e_vsi_clear_rings(vsi);
10413         return -ENOMEM;
10414 }
10415
10416 /**
10417  * i40e_reserve_msix_vectors - Reserve MSI-X vectors in the kernel
10418  * @pf: board private structure
10419  * @vectors: the number of MSI-X vectors to request
10420  *
10421  * Returns the number of vectors reserved, or error
10422  **/
10423 static int i40e_reserve_msix_vectors(struct i40e_pf *pf, int vectors)
10424 {
10425         vectors = pci_enable_msix_range(pf->pdev, pf->msix_entries,
10426                                         I40E_MIN_MSIX, vectors);
10427         if (vectors < 0) {
10428                 dev_info(&pf->pdev->dev,
10429                          "MSI-X vector reservation failed: %d\n", vectors);
10430                 vectors = 0;
10431         }
10432
10433         return vectors;
10434 }
10435
10436 /**
10437  * i40e_init_msix - Setup the MSIX capability
10438  * @pf: board private structure
10439  *
10440  * Work with the OS to set up the MSIX vectors needed.
10441  *
10442  * Returns the number of vectors reserved or negative on failure
10443  **/
10444 static int i40e_init_msix(struct i40e_pf *pf)
10445 {
10446         struct i40e_hw *hw = &pf->hw;
10447         int cpus, extra_vectors;
10448         int vectors_left;
10449         int v_budget, i;
10450         int v_actual;
10451         int iwarp_requested = 0;
10452
10453         if (!(pf->flags & I40E_FLAG_MSIX_ENABLED))
10454                 return -ENODEV;
10455
10456         /* The number of vectors we'll request will be comprised of:
10457          *   - Add 1 for "other" cause for Admin Queue events, etc.
10458          *   - The number of LAN queue pairs
10459          *      - Queues being used for RSS.
10460          *              We don't need as many as max_rss_size vectors.
10461          *              use rss_size instead in the calculation since that
10462          *              is governed by number of cpus in the system.
10463          *      - assumes symmetric Tx/Rx pairing
10464          *   - The number of VMDq pairs
10465          *   - The CPU count within the NUMA node if iWARP is enabled
10466          * Once we count this up, try the request.
10467          *
10468          * If we can't get what we want, we'll simplify to nearly nothing
10469          * and try again.  If that still fails, we punt.
10470          */
10471         vectors_left = hw->func_caps.num_msix_vectors;
10472         v_budget = 0;
10473
10474         /* reserve one vector for miscellaneous handler */
10475         if (vectors_left) {
10476                 v_budget++;
10477                 vectors_left--;
10478         }
10479
10480         /* reserve some vectors for the main PF traffic queues. Initially we
10481          * only reserve at most 50% of the available vectors, in the case that
10482          * the number of online CPUs is large. This ensures that we can enable
10483          * extra features as well. Once we've enabled the other features, we
10484          * will use any remaining vectors to reach as close as we can to the
10485          * number of online CPUs.
10486          */
10487         cpus = num_online_cpus();
10488         pf->num_lan_msix = min_t(int, cpus, vectors_left / 2);
10489         vectors_left -= pf->num_lan_msix;
10490
10491         /* reserve one vector for sideband flow director */
10492         if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
10493                 if (vectors_left) {
10494                         pf->num_fdsb_msix = 1;
10495                         v_budget++;
10496                         vectors_left--;
10497                 } else {
10498                         pf->num_fdsb_msix = 0;
10499                 }
10500         }
10501
10502         /* can we reserve enough for iWARP? */
10503         if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
10504                 iwarp_requested = pf->num_iwarp_msix;
10505
10506                 if (!vectors_left)
10507                         pf->num_iwarp_msix = 0;
10508                 else if (vectors_left < pf->num_iwarp_msix)
10509                         pf->num_iwarp_msix = 1;
10510                 v_budget += pf->num_iwarp_msix;
10511                 vectors_left -= pf->num_iwarp_msix;
10512         }
10513
10514         /* any vectors left over go for VMDq support */
10515         if (pf->flags & I40E_FLAG_VMDQ_ENABLED) {
10516                 if (!vectors_left) {
10517                         pf->num_vmdq_msix = 0;
10518                         pf->num_vmdq_qps = 0;
10519                 } else {
10520                         int vmdq_vecs_wanted =
10521                                 pf->num_vmdq_vsis * pf->num_vmdq_qps;
10522                         int vmdq_vecs =
10523                                 min_t(int, vectors_left, vmdq_vecs_wanted);
10524
10525                         /* if we're short on vectors for what's desired, we limit
10526                          * the queues per vmdq.  If this is still more than are
10527                          * available, the user will need to change the number of
10528                          * queues/vectors used by the PF later with the ethtool
10529                          * channels command
10530                          */
10531                         if (vectors_left < vmdq_vecs_wanted) {
10532                                 pf->num_vmdq_qps = 1;
10533                                 vmdq_vecs_wanted = pf->num_vmdq_vsis;
10534                                 vmdq_vecs = min_t(int,
10535                                                   vectors_left,
10536                                                   vmdq_vecs_wanted);
10537                         }
10538                         pf->num_vmdq_msix = pf->num_vmdq_qps;
10539
10540                         v_budget += vmdq_vecs;
10541                         vectors_left -= vmdq_vecs;
10542                 }
10543         }
10544
10545         /* On systems with a large number of SMP cores, we previously limited
10546          * the number of vectors for num_lan_msix to be at most 50% of the
10547          * available vectors, to allow for other features. Now, we add back
10548          * the remaining vectors. However, we ensure that the total
10549          * num_lan_msix will not exceed num_online_cpus(). To do this, we
10550          * calculate the number of vectors we can add without going over the
10551          * cap of CPUs. For systems with a small number of CPUs this will be
10552          * zero.
10553          */
10554         extra_vectors = min_t(int, cpus - pf->num_lan_msix, vectors_left);
10555         pf->num_lan_msix += extra_vectors;
10556         vectors_left -= extra_vectors;
10557
10558         WARN(vectors_left < 0,
10559              "Calculation of remaining vectors underflowed. This is an accounting bug when determining total MSI-X vectors.\n");
10560
10561         v_budget += pf->num_lan_msix;
10562         pf->msix_entries = kcalloc(v_budget, sizeof(struct msix_entry),
10563                                    GFP_KERNEL);
10564         if (!pf->msix_entries)
10565                 return -ENOMEM;
10566
10567         for (i = 0; i < v_budget; i++)
10568                 pf->msix_entries[i].entry = i;
10569         v_actual = i40e_reserve_msix_vectors(pf, v_budget);
10570
10571         if (v_actual < I40E_MIN_MSIX) {
10572                 pf->flags &= ~I40E_FLAG_MSIX_ENABLED;
10573                 kfree(pf->msix_entries);
10574                 pf->msix_entries = NULL;
10575                 pci_disable_msix(pf->pdev);
10576                 return -ENODEV;
10577
10578         } else if (v_actual == I40E_MIN_MSIX) {
10579                 /* Adjust for minimal MSIX use */
10580                 pf->num_vmdq_vsis = 0;
10581                 pf->num_vmdq_qps = 0;
10582                 pf->num_lan_qps = 1;
10583                 pf->num_lan_msix = 1;
10584
10585         } else if (v_actual != v_budget) {
10586                 /* If we have limited resources, we will start with no vectors
10587                  * for the special features and then allocate vectors to some
10588                  * of these features based on the policy and at the end disable
10589                  * the features that did not get any vectors.
10590                  */
10591                 int vec;
10592
10593                 dev_info(&pf->pdev->dev,
10594                          "MSI-X vector limit reached with %d, wanted %d, attempting to redistribute vectors\n",
10595                          v_actual, v_budget);
10596                 /* reserve the misc vector */
10597                 vec = v_actual - 1;
10598
10599                 /* Scale vector usage down */
10600                 pf->num_vmdq_msix = 1;    /* force VMDqs to only one vector */
10601                 pf->num_vmdq_vsis = 1;
10602                 pf->num_vmdq_qps = 1;
10603
10604                 /* partition out the remaining vectors */
10605                 switch (vec) {
10606                 case 2:
10607                         pf->num_lan_msix = 1;
10608                         break;
10609                 case 3:
10610                         if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
10611                                 pf->num_lan_msix = 1;
10612                                 pf->num_iwarp_msix = 1;
10613                         } else {
10614                                 pf->num_lan_msix = 2;
10615                         }
10616                         break;
10617                 default:
10618                         if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
10619                                 pf->num_iwarp_msix = min_t(int, (vec / 3),
10620                                                  iwarp_requested);
10621                                 pf->num_vmdq_vsis = min_t(int, (vec / 3),
10622                                                   I40E_DEFAULT_NUM_VMDQ_VSI);
10623                         } else {
10624                                 pf->num_vmdq_vsis = min_t(int, (vec / 2),
10625                                                   I40E_DEFAULT_NUM_VMDQ_VSI);
10626                         }
10627                         if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
10628                                 pf->num_fdsb_msix = 1;
10629                                 vec--;
10630                         }
10631                         pf->num_lan_msix = min_t(int,
10632                                (vec - (pf->num_iwarp_msix + pf->num_vmdq_vsis)),
10633                                                               pf->num_lan_msix);
10634                         pf->num_lan_qps = pf->num_lan_msix;
10635                         break;
10636                 }
10637         }
10638
10639         if ((pf->flags & I40E_FLAG_FD_SB_ENABLED) &&
10640             (pf->num_fdsb_msix == 0)) {
10641                 dev_info(&pf->pdev->dev, "Sideband Flowdir disabled, not enough MSI-X vectors\n");
10642                 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED;
10643                 pf->flags |= I40E_FLAG_FD_SB_INACTIVE;
10644         }
10645         if ((pf->flags & I40E_FLAG_VMDQ_ENABLED) &&
10646             (pf->num_vmdq_msix == 0)) {
10647                 dev_info(&pf->pdev->dev, "VMDq disabled, not enough MSI-X vectors\n");
10648                 pf->flags &= ~I40E_FLAG_VMDQ_ENABLED;
10649         }
10650
10651         if ((pf->flags & I40E_FLAG_IWARP_ENABLED) &&
10652             (pf->num_iwarp_msix == 0)) {
10653                 dev_info(&pf->pdev->dev, "IWARP disabled, not enough MSI-X vectors\n");
10654                 pf->flags &= ~I40E_FLAG_IWARP_ENABLED;
10655         }
10656         i40e_debug(&pf->hw, I40E_DEBUG_INIT,
10657                    "MSI-X vector distribution: PF %d, VMDq %d, FDSB %d, iWARP %d\n",
10658                    pf->num_lan_msix,
10659                    pf->num_vmdq_msix * pf->num_vmdq_vsis,
10660                    pf->num_fdsb_msix,
10661                    pf->num_iwarp_msix);
10662
10663         return v_actual;
10664 }
10665
10666 /**
10667  * i40e_vsi_alloc_q_vector - Allocate memory for a single interrupt vector
10668  * @vsi: the VSI being configured
10669  * @v_idx: index of the vector in the vsi struct
10670  * @cpu: cpu to be used on affinity_mask
10671  *
10672  * We allocate one q_vector.  If allocation fails we return -ENOMEM.
10673  **/
10674 static int i40e_vsi_alloc_q_vector(struct i40e_vsi *vsi, int v_idx, int cpu)
10675 {
10676         struct i40e_q_vector *q_vector;
10677
10678         /* allocate q_vector */
10679         q_vector = kzalloc(sizeof(struct i40e_q_vector), GFP_KERNEL);
10680         if (!q_vector)
10681                 return -ENOMEM;
10682
10683         q_vector->vsi = vsi;
10684         q_vector->v_idx = v_idx;
10685         cpumask_copy(&q_vector->affinity_mask, cpu_possible_mask);
10686
10687         if (vsi->netdev)
10688                 netif_napi_add(vsi->netdev, &q_vector->napi,
10689                                i40e_napi_poll, NAPI_POLL_WEIGHT);
10690
10691         /* tie q_vector and vsi together */
10692         vsi->q_vectors[v_idx] = q_vector;
10693
10694         return 0;
10695 }
10696
10697 /**
10698  * i40e_vsi_alloc_q_vectors - Allocate memory for interrupt vectors
10699  * @vsi: the VSI being configured
10700  *
10701  * We allocate one q_vector per queue interrupt.  If allocation fails we
10702  * return -ENOMEM.
10703  **/
10704 static int i40e_vsi_alloc_q_vectors(struct i40e_vsi *vsi)
10705 {
10706         struct i40e_pf *pf = vsi->back;
10707         int err, v_idx, num_q_vectors, current_cpu;
10708
10709         /* if not MSIX, give the one vector only to the LAN VSI */
10710         if (pf->flags & I40E_FLAG_MSIX_ENABLED)
10711                 num_q_vectors = vsi->num_q_vectors;
10712         else if (vsi == pf->vsi[pf->lan_vsi])
10713                 num_q_vectors = 1;
10714         else
10715                 return -EINVAL;
10716
10717         current_cpu = cpumask_first(cpu_online_mask);
10718
10719         for (v_idx = 0; v_idx < num_q_vectors; v_idx++) {
10720                 err = i40e_vsi_alloc_q_vector(vsi, v_idx, current_cpu);
10721                 if (err)
10722                         goto err_out;
10723                 current_cpu = cpumask_next(current_cpu, cpu_online_mask);
10724                 if (unlikely(current_cpu >= nr_cpu_ids))
10725                         current_cpu = cpumask_first(cpu_online_mask);
10726         }
10727
10728         return 0;
10729
10730 err_out:
10731         while (v_idx--)
10732                 i40e_free_q_vector(vsi, v_idx);
10733
10734         return err;
10735 }
10736
10737 /**
10738  * i40e_init_interrupt_scheme - Determine proper interrupt scheme
10739  * @pf: board private structure to initialize
10740  **/
10741 static int i40e_init_interrupt_scheme(struct i40e_pf *pf)
10742 {
10743         int vectors = 0;
10744         ssize_t size;
10745
10746         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
10747                 vectors = i40e_init_msix(pf);
10748                 if (vectors < 0) {
10749                         pf->flags &= ~(I40E_FLAG_MSIX_ENABLED   |
10750                                        I40E_FLAG_IWARP_ENABLED  |
10751                                        I40E_FLAG_RSS_ENABLED    |
10752                                        I40E_FLAG_DCB_CAPABLE    |
10753                                        I40E_FLAG_DCB_ENABLED    |
10754                                        I40E_FLAG_SRIOV_ENABLED  |
10755                                        I40E_FLAG_FD_SB_ENABLED  |
10756                                        I40E_FLAG_FD_ATR_ENABLED |
10757                                        I40E_FLAG_VMDQ_ENABLED);
10758                         pf->flags |= I40E_FLAG_FD_SB_INACTIVE;
10759
10760                         /* rework the queue expectations without MSIX */
10761                         i40e_determine_queue_usage(pf);
10762                 }
10763         }
10764
10765         if (!(pf->flags & I40E_FLAG_MSIX_ENABLED) &&
10766             (pf->flags & I40E_FLAG_MSI_ENABLED)) {
10767                 dev_info(&pf->pdev->dev, "MSI-X not available, trying MSI\n");
10768                 vectors = pci_enable_msi(pf->pdev);
10769                 if (vectors < 0) {
10770                         dev_info(&pf->pdev->dev, "MSI init failed - %d\n",
10771                                  vectors);
10772                         pf->flags &= ~I40E_FLAG_MSI_ENABLED;
10773                 }
10774                 vectors = 1;  /* one MSI or Legacy vector */
10775         }
10776
10777         if (!(pf->flags & (I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED)))
10778                 dev_info(&pf->pdev->dev, "MSI-X and MSI not available, falling back to Legacy IRQ\n");
10779
10780         /* set up vector assignment tracking */
10781         size = sizeof(struct i40e_lump_tracking) + (sizeof(u16) * vectors);
10782         pf->irq_pile = kzalloc(size, GFP_KERNEL);
10783         if (!pf->irq_pile)
10784                 return -ENOMEM;
10785
10786         pf->irq_pile->num_entries = vectors;
10787
10788         /* track first vector for misc interrupts, ignore return */
10789         (void)i40e_get_lump(pf, pf->irq_pile, 1, I40E_PILE_VALID_BIT - 1);
10790
10791         return 0;
10792 }
10793
10794 /**
10795  * i40e_restore_interrupt_scheme - Restore the interrupt scheme
10796  * @pf: private board data structure
10797  *
10798  * Restore the interrupt scheme that was cleared when we suspended the
10799  * device. This should be called during resume to re-allocate the q_vectors
10800  * and reacquire IRQs.
10801  */
10802 static int i40e_restore_interrupt_scheme(struct i40e_pf *pf)
10803 {
10804         int err, i;
10805
10806         /* We cleared the MSI and MSI-X flags when disabling the old interrupt
10807          * scheme. We need to re-enabled them here in order to attempt to
10808          * re-acquire the MSI or MSI-X vectors
10809          */
10810         pf->flags |= (I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED);
10811
10812         err = i40e_init_interrupt_scheme(pf);
10813         if (err)
10814                 return err;
10815
10816         /* Now that we've re-acquired IRQs, we need to remap the vectors and
10817          * rings together again.
10818          */
10819         for (i = 0; i < pf->num_alloc_vsi; i++) {
10820                 if (pf->vsi[i]) {
10821                         err = i40e_vsi_alloc_q_vectors(pf->vsi[i]);
10822                         if (err)
10823                                 goto err_unwind;
10824                         i40e_vsi_map_rings_to_vectors(pf->vsi[i]);
10825                 }
10826         }
10827
10828         err = i40e_setup_misc_vector(pf);
10829         if (err)
10830                 goto err_unwind;
10831
10832         if (pf->flags & I40E_FLAG_IWARP_ENABLED)
10833                 i40e_client_update_msix_info(pf);
10834
10835         return 0;
10836
10837 err_unwind:
10838         while (i--) {
10839                 if (pf->vsi[i])
10840                         i40e_vsi_free_q_vectors(pf->vsi[i]);
10841         }
10842
10843         return err;
10844 }
10845
10846 /**
10847  * i40e_setup_misc_vector - Setup the misc vector to handle non queue events
10848  * @pf: board private structure
10849  *
10850  * This sets up the handler for MSIX 0, which is used to manage the
10851  * non-queue interrupts, e.g. AdminQ and errors.  This is not used
10852  * when in MSI or Legacy interrupt mode.
10853  **/
10854 static int i40e_setup_misc_vector(struct i40e_pf *pf)
10855 {
10856         struct i40e_hw *hw = &pf->hw;
10857         int err = 0;
10858
10859         /* Only request the IRQ once, the first time through. */
10860         if (!test_and_set_bit(__I40E_MISC_IRQ_REQUESTED, pf->state)) {
10861                 err = request_irq(pf->msix_entries[0].vector,
10862                                   i40e_intr, 0, pf->int_name, pf);
10863                 if (err) {
10864                         clear_bit(__I40E_MISC_IRQ_REQUESTED, pf->state);
10865                         dev_info(&pf->pdev->dev,
10866                                  "request_irq for %s failed: %d\n",
10867                                  pf->int_name, err);
10868                         return -EFAULT;
10869                 }
10870         }
10871
10872         i40e_enable_misc_int_causes(pf);
10873
10874         /* associate no queues to the misc vector */
10875         wr32(hw, I40E_PFINT_LNKLST0, I40E_QUEUE_END_OF_LIST);
10876         wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), I40E_ITR_8K >> 1);
10877
10878         i40e_flush(hw);
10879
10880         i40e_irq_dynamic_enable_icr0(pf);
10881
10882         return err;
10883 }
10884
10885 /**
10886  * i40e_get_rss_aq - Get RSS keys and lut by using AQ commands
10887  * @vsi: Pointer to vsi structure
10888  * @seed: Buffter to store the hash keys
10889  * @lut: Buffer to store the lookup table entries
10890  * @lut_size: Size of buffer to store the lookup table entries
10891  *
10892  * Return 0 on success, negative on failure
10893  */
10894 static int i40e_get_rss_aq(struct i40e_vsi *vsi, const u8 *seed,
10895                            u8 *lut, u16 lut_size)
10896 {
10897         struct i40e_pf *pf = vsi->back;
10898         struct i40e_hw *hw = &pf->hw;
10899         int ret = 0;
10900
10901         if (seed) {
10902                 ret = i40e_aq_get_rss_key(hw, vsi->id,
10903                         (struct i40e_aqc_get_set_rss_key_data *)seed);
10904                 if (ret) {
10905                         dev_info(&pf->pdev->dev,
10906                                  "Cannot get RSS key, err %s aq_err %s\n",
10907                                  i40e_stat_str(&pf->hw, ret),
10908                                  i40e_aq_str(&pf->hw,
10909                                              pf->hw.aq.asq_last_status));
10910                         return ret;
10911                 }
10912         }
10913
10914         if (lut) {
10915                 bool pf_lut = vsi->type == I40E_VSI_MAIN ? true : false;
10916
10917                 ret = i40e_aq_get_rss_lut(hw, vsi->id, pf_lut, lut, lut_size);
10918                 if (ret) {
10919                         dev_info(&pf->pdev->dev,
10920                                  "Cannot get RSS lut, err %s aq_err %s\n",
10921                                  i40e_stat_str(&pf->hw, ret),
10922                                  i40e_aq_str(&pf->hw,
10923                                              pf->hw.aq.asq_last_status));
10924                         return ret;
10925                 }
10926         }
10927
10928         return ret;
10929 }
10930
10931 /**
10932  * i40e_config_rss_reg - Configure RSS keys and lut by writing registers
10933  * @vsi: Pointer to vsi structure
10934  * @seed: RSS hash seed
10935  * @lut: Lookup table
10936  * @lut_size: Lookup table size
10937  *
10938  * Returns 0 on success, negative on failure
10939  **/
10940 static int i40e_config_rss_reg(struct i40e_vsi *vsi, const u8 *seed,
10941                                const u8 *lut, u16 lut_size)
10942 {
10943         struct i40e_pf *pf = vsi->back;
10944         struct i40e_hw *hw = &pf->hw;
10945         u16 vf_id = vsi->vf_id;
10946         u8 i;
10947
10948         /* Fill out hash function seed */
10949         if (seed) {
10950                 u32 *seed_dw = (u32 *)seed;
10951
10952                 if (vsi->type == I40E_VSI_MAIN) {
10953                         for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++)
10954                                 wr32(hw, I40E_PFQF_HKEY(i), seed_dw[i]);
10955                 } else if (vsi->type == I40E_VSI_SRIOV) {
10956                         for (i = 0; i <= I40E_VFQF_HKEY1_MAX_INDEX; i++)
10957                                 wr32(hw, I40E_VFQF_HKEY1(i, vf_id), seed_dw[i]);
10958                 } else {
10959                         dev_err(&pf->pdev->dev, "Cannot set RSS seed - invalid VSI type\n");
10960                 }
10961         }
10962
10963         if (lut) {
10964                 u32 *lut_dw = (u32 *)lut;
10965
10966                 if (vsi->type == I40E_VSI_MAIN) {
10967                         if (lut_size != I40E_HLUT_ARRAY_SIZE)
10968                                 return -EINVAL;
10969                         for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++)
10970                                 wr32(hw, I40E_PFQF_HLUT(i), lut_dw[i]);
10971                 } else if (vsi->type == I40E_VSI_SRIOV) {
10972                         if (lut_size != I40E_VF_HLUT_ARRAY_SIZE)
10973                                 return -EINVAL;
10974                         for (i = 0; i <= I40E_VFQF_HLUT_MAX_INDEX; i++)
10975                                 wr32(hw, I40E_VFQF_HLUT1(i, vf_id), lut_dw[i]);
10976                 } else {
10977                         dev_err(&pf->pdev->dev, "Cannot set RSS LUT - invalid VSI type\n");
10978                 }
10979         }
10980         i40e_flush(hw);
10981
10982         return 0;
10983 }
10984
10985 /**
10986  * i40e_get_rss_reg - Get the RSS keys and lut by reading registers
10987  * @vsi: Pointer to VSI structure
10988  * @seed: Buffer to store the keys
10989  * @lut: Buffer to store the lookup table entries
10990  * @lut_size: Size of buffer to store the lookup table entries
10991  *
10992  * Returns 0 on success, negative on failure
10993  */
10994 static int i40e_get_rss_reg(struct i40e_vsi *vsi, u8 *seed,
10995                             u8 *lut, u16 lut_size)
10996 {
10997         struct i40e_pf *pf = vsi->back;
10998         struct i40e_hw *hw = &pf->hw;
10999         u16 i;
11000
11001         if (seed) {
11002                 u32 *seed_dw = (u32 *)seed;
11003
11004                 for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++)
11005                         seed_dw[i] = i40e_read_rx_ctl(hw, I40E_PFQF_HKEY(i));
11006         }
11007         if (lut) {
11008                 u32 *lut_dw = (u32 *)lut;
11009
11010                 if (lut_size != I40E_HLUT_ARRAY_SIZE)
11011                         return -EINVAL;
11012                 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++)
11013                         lut_dw[i] = rd32(hw, I40E_PFQF_HLUT(i));
11014         }
11015
11016         return 0;
11017 }
11018
11019 /**
11020  * i40e_config_rss - Configure RSS keys and lut
11021  * @vsi: Pointer to VSI structure
11022  * @seed: RSS hash seed
11023  * @lut: Lookup table
11024  * @lut_size: Lookup table size
11025  *
11026  * Returns 0 on success, negative on failure
11027  */
11028 int i40e_config_rss(struct i40e_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size)
11029 {
11030         struct i40e_pf *pf = vsi->back;
11031
11032         if (pf->hw_features & I40E_HW_RSS_AQ_CAPABLE)
11033                 return i40e_config_rss_aq(vsi, seed, lut, lut_size);
11034         else
11035                 return i40e_config_rss_reg(vsi, seed, lut, lut_size);
11036 }
11037
11038 /**
11039  * i40e_get_rss - Get RSS keys and lut
11040  * @vsi: Pointer to VSI structure
11041  * @seed: Buffer to store the keys
11042  * @lut: Buffer to store the lookup table entries
11043  * @lut_size: Size of buffer to store the lookup table entries
11044  *
11045  * Returns 0 on success, negative on failure
11046  */
11047 int i40e_get_rss(struct i40e_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size)
11048 {
11049         struct i40e_pf *pf = vsi->back;
11050
11051         if (pf->hw_features & I40E_HW_RSS_AQ_CAPABLE)
11052                 return i40e_get_rss_aq(vsi, seed, lut, lut_size);
11053         else
11054                 return i40e_get_rss_reg(vsi, seed, lut, lut_size);
11055 }
11056
11057 /**
11058  * i40e_fill_rss_lut - Fill the RSS lookup table with default values
11059  * @pf: Pointer to board private structure
11060  * @lut: Lookup table
11061  * @rss_table_size: Lookup table size
11062  * @rss_size: Range of queue number for hashing
11063  */
11064 void i40e_fill_rss_lut(struct i40e_pf *pf, u8 *lut,
11065                        u16 rss_table_size, u16 rss_size)
11066 {
11067         u16 i;
11068
11069         for (i = 0; i < rss_table_size; i++)
11070                 lut[i] = i % rss_size;
11071 }
11072
11073 /**
11074  * i40e_pf_config_rss - Prepare for RSS if used
11075  * @pf: board private structure
11076  **/
11077 static int i40e_pf_config_rss(struct i40e_pf *pf)
11078 {
11079         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
11080         u8 seed[I40E_HKEY_ARRAY_SIZE];
11081         u8 *lut;
11082         struct i40e_hw *hw = &pf->hw;
11083         u32 reg_val;
11084         u64 hena;
11085         int ret;
11086
11087         /* By default we enable TCP/UDP with IPv4/IPv6 ptypes */
11088         hena = (u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(0)) |
11089                 ((u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(1)) << 32);
11090         hena |= i40e_pf_get_default_rss_hena(pf);
11091
11092         i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), (u32)hena);
11093         i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), (u32)(hena >> 32));
11094
11095         /* Determine the RSS table size based on the hardware capabilities */
11096         reg_val = i40e_read_rx_ctl(hw, I40E_PFQF_CTL_0);
11097         reg_val = (pf->rss_table_size == 512) ?
11098                         (reg_val | I40E_PFQF_CTL_0_HASHLUTSIZE_512) :
11099                         (reg_val & ~I40E_PFQF_CTL_0_HASHLUTSIZE_512);
11100         i40e_write_rx_ctl(hw, I40E_PFQF_CTL_0, reg_val);
11101
11102         /* Determine the RSS size of the VSI */
11103         if (!vsi->rss_size) {
11104                 u16 qcount;
11105                 /* If the firmware does something weird during VSI init, we
11106                  * could end up with zero TCs. Check for that to avoid
11107                  * divide-by-zero. It probably won't pass traffic, but it also
11108                  * won't panic.
11109                  */
11110                 qcount = vsi->num_queue_pairs /
11111                          (vsi->tc_config.numtc ? vsi->tc_config.numtc : 1);
11112                 vsi->rss_size = min_t(int, pf->alloc_rss_size, qcount);
11113         }
11114         if (!vsi->rss_size)
11115                 return -EINVAL;
11116
11117         lut = kzalloc(vsi->rss_table_size, GFP_KERNEL);
11118         if (!lut)
11119                 return -ENOMEM;
11120
11121         /* Use user configured lut if there is one, otherwise use default */
11122         if (vsi->rss_lut_user)
11123                 memcpy(lut, vsi->rss_lut_user, vsi->rss_table_size);
11124         else
11125                 i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, vsi->rss_size);
11126
11127         /* Use user configured hash key if there is one, otherwise
11128          * use default.
11129          */
11130         if (vsi->rss_hkey_user)
11131                 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE);
11132         else
11133                 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE);
11134         ret = i40e_config_rss(vsi, seed, lut, vsi->rss_table_size);
11135         kfree(lut);
11136
11137         return ret;
11138 }
11139
11140 /**
11141  * i40e_reconfig_rss_queues - change number of queues for rss and rebuild
11142  * @pf: board private structure
11143  * @queue_count: the requested queue count for rss.
11144  *
11145  * returns 0 if rss is not enabled, if enabled returns the final rss queue
11146  * count which may be different from the requested queue count.
11147  * Note: expects to be called while under rtnl_lock()
11148  **/
11149 int i40e_reconfig_rss_queues(struct i40e_pf *pf, int queue_count)
11150 {
11151         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
11152         int new_rss_size;
11153
11154         if (!(pf->flags & I40E_FLAG_RSS_ENABLED))
11155                 return 0;
11156
11157         new_rss_size = min_t(int, queue_count, pf->rss_size_max);
11158
11159         if (queue_count != vsi->num_queue_pairs) {
11160                 u16 qcount;
11161
11162                 vsi->req_queue_pairs = queue_count;
11163                 i40e_prep_for_reset(pf, true);
11164
11165                 pf->alloc_rss_size = new_rss_size;
11166
11167                 i40e_reset_and_rebuild(pf, true, true);
11168
11169                 /* Discard the user configured hash keys and lut, if less
11170                  * queues are enabled.
11171                  */
11172                 if (queue_count < vsi->rss_size) {
11173                         i40e_clear_rss_config_user(vsi);
11174                         dev_dbg(&pf->pdev->dev,
11175                                 "discard user configured hash keys and lut\n");
11176                 }
11177
11178                 /* Reset vsi->rss_size, as number of enabled queues changed */
11179                 qcount = vsi->num_queue_pairs / vsi->tc_config.numtc;
11180                 vsi->rss_size = min_t(int, pf->alloc_rss_size, qcount);
11181
11182                 i40e_pf_config_rss(pf);
11183         }
11184         dev_info(&pf->pdev->dev, "User requested queue count/HW max RSS count:  %d/%d\n",
11185                  vsi->req_queue_pairs, pf->rss_size_max);
11186         return pf->alloc_rss_size;
11187 }
11188
11189 /**
11190  * i40e_get_partition_bw_setting - Retrieve BW settings for this PF partition
11191  * @pf: board private structure
11192  **/
11193 i40e_status i40e_get_partition_bw_setting(struct i40e_pf *pf)
11194 {
11195         i40e_status status;
11196         bool min_valid, max_valid;
11197         u32 max_bw, min_bw;
11198
11199         status = i40e_read_bw_from_alt_ram(&pf->hw, &max_bw, &min_bw,
11200                                            &min_valid, &max_valid);
11201
11202         if (!status) {
11203                 if (min_valid)
11204                         pf->min_bw = min_bw;
11205                 if (max_valid)
11206                         pf->max_bw = max_bw;
11207         }
11208
11209         return status;
11210 }
11211
11212 /**
11213  * i40e_set_partition_bw_setting - Set BW settings for this PF partition
11214  * @pf: board private structure
11215  **/
11216 i40e_status i40e_set_partition_bw_setting(struct i40e_pf *pf)
11217 {
11218         struct i40e_aqc_configure_partition_bw_data bw_data;
11219         i40e_status status;
11220
11221         memset(&bw_data, 0, sizeof(bw_data));
11222
11223         /* Set the valid bit for this PF */
11224         bw_data.pf_valid_bits = cpu_to_le16(BIT(pf->hw.pf_id));
11225         bw_data.max_bw[pf->hw.pf_id] = pf->max_bw & I40E_ALT_BW_VALUE_MASK;
11226         bw_data.min_bw[pf->hw.pf_id] = pf->min_bw & I40E_ALT_BW_VALUE_MASK;
11227
11228         /* Set the new bandwidths */
11229         status = i40e_aq_configure_partition_bw(&pf->hw, &bw_data, NULL);
11230
11231         return status;
11232 }
11233
11234 /**
11235  * i40e_commit_partition_bw_setting - Commit BW settings for this PF partition
11236  * @pf: board private structure
11237  **/
11238 i40e_status i40e_commit_partition_bw_setting(struct i40e_pf *pf)
11239 {
11240         /* Commit temporary BW setting to permanent NVM image */
11241         enum i40e_admin_queue_err last_aq_status;
11242         i40e_status ret;
11243         u16 nvm_word;
11244
11245         if (pf->hw.partition_id != 1) {
11246                 dev_info(&pf->pdev->dev,
11247                          "Commit BW only works on partition 1! This is partition %d",
11248                          pf->hw.partition_id);
11249                 ret = I40E_NOT_SUPPORTED;
11250                 goto bw_commit_out;
11251         }
11252
11253         /* Acquire NVM for read access */
11254         ret = i40e_acquire_nvm(&pf->hw, I40E_RESOURCE_READ);
11255         last_aq_status = pf->hw.aq.asq_last_status;
11256         if (ret) {
11257                 dev_info(&pf->pdev->dev,
11258                          "Cannot acquire NVM for read access, err %s aq_err %s\n",
11259                          i40e_stat_str(&pf->hw, ret),
11260                          i40e_aq_str(&pf->hw, last_aq_status));
11261                 goto bw_commit_out;
11262         }
11263
11264         /* Read word 0x10 of NVM - SW compatibility word 1 */
11265         ret = i40e_aq_read_nvm(&pf->hw,
11266                                I40E_SR_NVM_CONTROL_WORD,
11267                                0x10, sizeof(nvm_word), &nvm_word,
11268                                false, NULL);
11269         /* Save off last admin queue command status before releasing
11270          * the NVM
11271          */
11272         last_aq_status = pf->hw.aq.asq_last_status;
11273         i40e_release_nvm(&pf->hw);
11274         if (ret) {
11275                 dev_info(&pf->pdev->dev, "NVM read error, err %s aq_err %s\n",
11276                          i40e_stat_str(&pf->hw, ret),
11277                          i40e_aq_str(&pf->hw, last_aq_status));
11278                 goto bw_commit_out;
11279         }
11280
11281         /* Wait a bit for NVM release to complete */
11282         msleep(50);
11283
11284         /* Acquire NVM for write access */
11285         ret = i40e_acquire_nvm(&pf->hw, I40E_RESOURCE_WRITE);
11286         last_aq_status = pf->hw.aq.asq_last_status;
11287         if (ret) {
11288                 dev_info(&pf->pdev->dev,
11289                          "Cannot acquire NVM for write access, err %s aq_err %s\n",
11290                          i40e_stat_str(&pf->hw, ret),
11291                          i40e_aq_str(&pf->hw, last_aq_status));
11292                 goto bw_commit_out;
11293         }
11294         /* Write it back out unchanged to initiate update NVM,
11295          * which will force a write of the shadow (alt) RAM to
11296          * the NVM - thus storing the bandwidth values permanently.
11297          */
11298         ret = i40e_aq_update_nvm(&pf->hw,
11299                                  I40E_SR_NVM_CONTROL_WORD,
11300                                  0x10, sizeof(nvm_word),
11301                                  &nvm_word, true, 0, NULL);
11302         /* Save off last admin queue command status before releasing
11303          * the NVM
11304          */
11305         last_aq_status = pf->hw.aq.asq_last_status;
11306         i40e_release_nvm(&pf->hw);
11307         if (ret)
11308                 dev_info(&pf->pdev->dev,
11309                          "BW settings NOT SAVED, err %s aq_err %s\n",
11310                          i40e_stat_str(&pf->hw, ret),
11311                          i40e_aq_str(&pf->hw, last_aq_status));
11312 bw_commit_out:
11313
11314         return ret;
11315 }
11316
11317 /**
11318  * i40e_sw_init - Initialize general software structures (struct i40e_pf)
11319  * @pf: board private structure to initialize
11320  *
11321  * i40e_sw_init initializes the Adapter private data structure.
11322  * Fields are initialized based on PCI device information and
11323  * OS network device settings (MTU size).
11324  **/
11325 static int i40e_sw_init(struct i40e_pf *pf)
11326 {
11327         int err = 0;
11328         int size;
11329         u16 pow;
11330
11331         /* Set default capability flags */
11332         pf->flags = I40E_FLAG_RX_CSUM_ENABLED |
11333                     I40E_FLAG_MSI_ENABLED     |
11334                     I40E_FLAG_MSIX_ENABLED;
11335
11336         /* Set default ITR */
11337         pf->rx_itr_default = I40E_ITR_RX_DEF;
11338         pf->tx_itr_default = I40E_ITR_TX_DEF;
11339
11340         /* Depending on PF configurations, it is possible that the RSS
11341          * maximum might end up larger than the available queues
11342          */
11343         pf->rss_size_max = BIT(pf->hw.func_caps.rss_table_entry_width);
11344         pf->alloc_rss_size = 1;
11345         pf->rss_table_size = pf->hw.func_caps.rss_table_size;
11346         pf->rss_size_max = min_t(int, pf->rss_size_max,
11347                                  pf->hw.func_caps.num_tx_qp);
11348
11349         /* find the next higher power-of-2 of num cpus */
11350         pow = roundup_pow_of_two(num_online_cpus());
11351         pf->rss_size_max = min_t(int, pf->rss_size_max, pow);
11352
11353         if (pf->hw.func_caps.rss) {
11354                 pf->flags |= I40E_FLAG_RSS_ENABLED;
11355                 pf->alloc_rss_size = min_t(int, pf->rss_size_max,
11356                                            num_online_cpus());
11357         }
11358
11359         /* MFP mode enabled */
11360         if (pf->hw.func_caps.npar_enable || pf->hw.func_caps.flex10_enable) {
11361                 pf->flags |= I40E_FLAG_MFP_ENABLED;
11362                 dev_info(&pf->pdev->dev, "MFP mode Enabled\n");
11363                 if (i40e_get_partition_bw_setting(pf)) {
11364                         dev_warn(&pf->pdev->dev,
11365                                  "Could not get partition bw settings\n");
11366                 } else {
11367                         dev_info(&pf->pdev->dev,
11368                                  "Partition BW Min = %8.8x, Max = %8.8x\n",
11369                                  pf->min_bw, pf->max_bw);
11370
11371                         /* nudge the Tx scheduler */
11372                         i40e_set_partition_bw_setting(pf);
11373                 }
11374         }
11375
11376         if ((pf->hw.func_caps.fd_filters_guaranteed > 0) ||
11377             (pf->hw.func_caps.fd_filters_best_effort > 0)) {
11378                 pf->flags |= I40E_FLAG_FD_ATR_ENABLED;
11379                 pf->atr_sample_rate = I40E_DEFAULT_ATR_SAMPLE_RATE;
11380                 if (pf->flags & I40E_FLAG_MFP_ENABLED &&
11381                     pf->hw.num_partitions > 1)
11382                         dev_info(&pf->pdev->dev,
11383                                  "Flow Director Sideband mode Disabled in MFP mode\n");
11384                 else
11385                         pf->flags |= I40E_FLAG_FD_SB_ENABLED;
11386                 pf->fdir_pf_filter_count =
11387                                  pf->hw.func_caps.fd_filters_guaranteed;
11388                 pf->hw.fdir_shared_filter_count =
11389                                  pf->hw.func_caps.fd_filters_best_effort;
11390         }
11391
11392         if (pf->hw.mac.type == I40E_MAC_X722) {
11393                 pf->hw_features |= (I40E_HW_RSS_AQ_CAPABLE |
11394                                     I40E_HW_128_QP_RSS_CAPABLE |
11395                                     I40E_HW_ATR_EVICT_CAPABLE |
11396                                     I40E_HW_WB_ON_ITR_CAPABLE |
11397                                     I40E_HW_MULTIPLE_TCP_UDP_RSS_PCTYPE |
11398                                     I40E_HW_NO_PCI_LINK_CHECK |
11399                                     I40E_HW_USE_SET_LLDP_MIB |
11400                                     I40E_HW_GENEVE_OFFLOAD_CAPABLE |
11401                                     I40E_HW_PTP_L4_CAPABLE |
11402                                     I40E_HW_WOL_MC_MAGIC_PKT_WAKE |
11403                                     I40E_HW_OUTER_UDP_CSUM_CAPABLE);
11404
11405 #define I40E_FDEVICT_PCTYPE_DEFAULT 0xc03
11406                 if (rd32(&pf->hw, I40E_GLQF_FDEVICTENA(1)) !=
11407                     I40E_FDEVICT_PCTYPE_DEFAULT) {
11408                         dev_warn(&pf->pdev->dev,
11409                                  "FD EVICT PCTYPES are not right, disable FD HW EVICT\n");
11410                         pf->hw_features &= ~I40E_HW_ATR_EVICT_CAPABLE;
11411                 }
11412         } else if ((pf->hw.aq.api_maj_ver > 1) ||
11413                    ((pf->hw.aq.api_maj_ver == 1) &&
11414                     (pf->hw.aq.api_min_ver > 4))) {
11415                 /* Supported in FW API version higher than 1.4 */
11416                 pf->hw_features |= I40E_HW_GENEVE_OFFLOAD_CAPABLE;
11417         }
11418
11419         /* Enable HW ATR eviction if possible */
11420         if (pf->hw_features & I40E_HW_ATR_EVICT_CAPABLE)
11421                 pf->flags |= I40E_FLAG_HW_ATR_EVICT_ENABLED;
11422
11423         if ((pf->hw.mac.type == I40E_MAC_XL710) &&
11424             (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver < 33)) ||
11425             (pf->hw.aq.fw_maj_ver < 4))) {
11426                 pf->hw_features |= I40E_HW_RESTART_AUTONEG;
11427                 /* No DCB support  for FW < v4.33 */
11428                 pf->hw_features |= I40E_HW_NO_DCB_SUPPORT;
11429         }
11430
11431         /* Disable FW LLDP if FW < v4.3 */
11432         if ((pf->hw.mac.type == I40E_MAC_XL710) &&
11433             (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver < 3)) ||
11434             (pf->hw.aq.fw_maj_ver < 4)))
11435                 pf->hw_features |= I40E_HW_STOP_FW_LLDP;
11436
11437         /* Use the FW Set LLDP MIB API if FW > v4.40 */
11438         if ((pf->hw.mac.type == I40E_MAC_XL710) &&
11439             (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver >= 40)) ||
11440             (pf->hw.aq.fw_maj_ver >= 5)))
11441                 pf->hw_features |= I40E_HW_USE_SET_LLDP_MIB;
11442
11443         /* Enable PTP L4 if FW > v6.0 */
11444         if (pf->hw.mac.type == I40E_MAC_XL710 &&
11445             pf->hw.aq.fw_maj_ver >= 6)
11446                 pf->hw_features |= I40E_HW_PTP_L4_CAPABLE;
11447
11448         if (pf->hw.func_caps.vmdq && num_online_cpus() != 1) {
11449                 pf->num_vmdq_vsis = I40E_DEFAULT_NUM_VMDQ_VSI;
11450                 pf->flags |= I40E_FLAG_VMDQ_ENABLED;
11451                 pf->num_vmdq_qps = i40e_default_queues_per_vmdq(pf);
11452         }
11453
11454         if (pf->hw.func_caps.iwarp && num_online_cpus() != 1) {
11455                 pf->flags |= I40E_FLAG_IWARP_ENABLED;
11456                 /* IWARP needs one extra vector for CQP just like MISC.*/
11457                 pf->num_iwarp_msix = (int)num_online_cpus() + 1;
11458         }
11459         /* Stopping the FW LLDP engine is only supported on the
11460          * XL710 with a FW ver >= 1.7.  Also, stopping FW LLDP
11461          * engine is not supported if NPAR is functioning on this
11462          * part
11463          */
11464         if (pf->hw.mac.type == I40E_MAC_XL710 &&
11465             !pf->hw.func_caps.npar_enable &&
11466             (pf->hw.aq.api_maj_ver > 1 ||
11467              (pf->hw.aq.api_maj_ver == 1 && pf->hw.aq.api_min_ver > 6)))
11468                 pf->hw_features |= I40E_HW_STOPPABLE_FW_LLDP;
11469
11470 #ifdef CONFIG_PCI_IOV
11471         if (pf->hw.func_caps.num_vfs && pf->hw.partition_id == 1) {
11472                 pf->num_vf_qps = I40E_DEFAULT_QUEUES_PER_VF;
11473                 pf->flags |= I40E_FLAG_SRIOV_ENABLED;
11474                 pf->num_req_vfs = min_t(int,
11475                                         pf->hw.func_caps.num_vfs,
11476                                         I40E_MAX_VF_COUNT);
11477         }
11478 #endif /* CONFIG_PCI_IOV */
11479         pf->eeprom_version = 0xDEAD;
11480         pf->lan_veb = I40E_NO_VEB;
11481         pf->lan_vsi = I40E_NO_VSI;
11482
11483         /* By default FW has this off for performance reasons */
11484         pf->flags &= ~I40E_FLAG_VEB_STATS_ENABLED;
11485
11486         /* set up queue assignment tracking */
11487         size = sizeof(struct i40e_lump_tracking)
11488                 + (sizeof(u16) * pf->hw.func_caps.num_tx_qp);
11489         pf->qp_pile = kzalloc(size, GFP_KERNEL);
11490         if (!pf->qp_pile) {
11491                 err = -ENOMEM;
11492                 goto sw_init_done;
11493         }
11494         pf->qp_pile->num_entries = pf->hw.func_caps.num_tx_qp;
11495
11496         pf->tx_timeout_recovery_level = 1;
11497
11498         mutex_init(&pf->switch_mutex);
11499
11500 sw_init_done:
11501         return err;
11502 }
11503
11504 /**
11505  * i40e_set_ntuple - set the ntuple feature flag and take action
11506  * @pf: board private structure to initialize
11507  * @features: the feature set that the stack is suggesting
11508  *
11509  * returns a bool to indicate if reset needs to happen
11510  **/
11511 bool i40e_set_ntuple(struct i40e_pf *pf, netdev_features_t features)
11512 {
11513         bool need_reset = false;
11514
11515         /* Check if Flow Director n-tuple support was enabled or disabled.  If
11516          * the state changed, we need to reset.
11517          */
11518         if (features & NETIF_F_NTUPLE) {
11519                 /* Enable filters and mark for reset */
11520                 if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED))
11521                         need_reset = true;
11522                 /* enable FD_SB only if there is MSI-X vector and no cloud
11523                  * filters exist
11524                  */
11525                 if (pf->num_fdsb_msix > 0 && !pf->num_cloud_filters) {
11526                         pf->flags |= I40E_FLAG_FD_SB_ENABLED;
11527                         pf->flags &= ~I40E_FLAG_FD_SB_INACTIVE;
11528                 }
11529         } else {
11530                 /* turn off filters, mark for reset and clear SW filter list */
11531                 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
11532                         need_reset = true;
11533                         i40e_fdir_filter_exit(pf);
11534                 }
11535                 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED;
11536                 clear_bit(__I40E_FD_SB_AUTO_DISABLED, pf->state);
11537                 pf->flags |= I40E_FLAG_FD_SB_INACTIVE;
11538
11539                 /* reset fd counters */
11540                 pf->fd_add_err = 0;
11541                 pf->fd_atr_cnt = 0;
11542                 /* if ATR was auto disabled it can be re-enabled. */
11543                 if (test_and_clear_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state))
11544                         if ((pf->flags & I40E_FLAG_FD_ATR_ENABLED) &&
11545                             (I40E_DEBUG_FD & pf->hw.debug_mask))
11546                                 dev_info(&pf->pdev->dev, "ATR re-enabled.\n");
11547         }
11548         return need_reset;
11549 }
11550
11551 /**
11552  * i40e_clear_rss_lut - clear the rx hash lookup table
11553  * @vsi: the VSI being configured
11554  **/
11555 static void i40e_clear_rss_lut(struct i40e_vsi *vsi)
11556 {
11557         struct i40e_pf *pf = vsi->back;
11558         struct i40e_hw *hw = &pf->hw;
11559         u16 vf_id = vsi->vf_id;
11560         u8 i;
11561
11562         if (vsi->type == I40E_VSI_MAIN) {
11563                 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++)
11564                         wr32(hw, I40E_PFQF_HLUT(i), 0);
11565         } else if (vsi->type == I40E_VSI_SRIOV) {
11566                 for (i = 0; i <= I40E_VFQF_HLUT_MAX_INDEX; i++)
11567                         i40e_write_rx_ctl(hw, I40E_VFQF_HLUT1(i, vf_id), 0);
11568         } else {
11569                 dev_err(&pf->pdev->dev, "Cannot set RSS LUT - invalid VSI type\n");
11570         }
11571 }
11572
11573 /**
11574  * i40e_set_features - set the netdev feature flags
11575  * @netdev: ptr to the netdev being adjusted
11576  * @features: the feature set that the stack is suggesting
11577  * Note: expects to be called while under rtnl_lock()
11578  **/
11579 static int i40e_set_features(struct net_device *netdev,
11580                              netdev_features_t features)
11581 {
11582         struct i40e_netdev_priv *np = netdev_priv(netdev);
11583         struct i40e_vsi *vsi = np->vsi;
11584         struct i40e_pf *pf = vsi->back;
11585         bool need_reset;
11586
11587         if (features & NETIF_F_RXHASH && !(netdev->features & NETIF_F_RXHASH))
11588                 i40e_pf_config_rss(pf);
11589         else if (!(features & NETIF_F_RXHASH) &&
11590                  netdev->features & NETIF_F_RXHASH)
11591                 i40e_clear_rss_lut(vsi);
11592
11593         if (features & NETIF_F_HW_VLAN_CTAG_RX)
11594                 i40e_vlan_stripping_enable(vsi);
11595         else
11596                 i40e_vlan_stripping_disable(vsi);
11597
11598         if (!(features & NETIF_F_HW_TC) && pf->num_cloud_filters) {
11599                 dev_err(&pf->pdev->dev,
11600                         "Offloaded tc filters active, can't turn hw_tc_offload off");
11601                 return -EINVAL;
11602         }
11603
11604         need_reset = i40e_set_ntuple(pf, features);
11605
11606         if (need_reset)
11607                 i40e_do_reset(pf, I40E_PF_RESET_FLAG, true);
11608
11609         return 0;
11610 }
11611
11612 /**
11613  * i40e_get_udp_port_idx - Lookup a possibly offloaded for Rx UDP port
11614  * @pf: board private structure
11615  * @port: The UDP port to look up
11616  *
11617  * Returns the index number or I40E_MAX_PF_UDP_OFFLOAD_PORTS if port not found
11618  **/
11619 static u8 i40e_get_udp_port_idx(struct i40e_pf *pf, u16 port)
11620 {
11621         u8 i;
11622
11623         for (i = 0; i < I40E_MAX_PF_UDP_OFFLOAD_PORTS; i++) {
11624                 /* Do not report ports with pending deletions as
11625                  * being available.
11626                  */
11627                 if (!port && (pf->pending_udp_bitmap & BIT_ULL(i)))
11628                         continue;
11629                 if (pf->udp_ports[i].port == port)
11630                         return i;
11631         }
11632
11633         return i;
11634 }
11635
11636 /**
11637  * i40e_udp_tunnel_add - Get notifications about UDP tunnel ports that come up
11638  * @netdev: This physical port's netdev
11639  * @ti: Tunnel endpoint information
11640  **/
11641 static void i40e_udp_tunnel_add(struct net_device *netdev,
11642                                 struct udp_tunnel_info *ti)
11643 {
11644         struct i40e_netdev_priv *np = netdev_priv(netdev);
11645         struct i40e_vsi *vsi = np->vsi;
11646         struct i40e_pf *pf = vsi->back;
11647         u16 port = ntohs(ti->port);
11648         u8 next_idx;
11649         u8 idx;
11650
11651         idx = i40e_get_udp_port_idx(pf, port);
11652
11653         /* Check if port already exists */
11654         if (idx < I40E_MAX_PF_UDP_OFFLOAD_PORTS) {
11655                 netdev_info(netdev, "port %d already offloaded\n", port);
11656                 return;
11657         }
11658
11659         /* Now check if there is space to add the new port */
11660         next_idx = i40e_get_udp_port_idx(pf, 0);
11661
11662         if (next_idx == I40E_MAX_PF_UDP_OFFLOAD_PORTS) {
11663                 netdev_info(netdev, "maximum number of offloaded UDP ports reached, not adding port %d\n",
11664                             port);
11665                 return;
11666         }
11667
11668         switch (ti->type) {
11669         case UDP_TUNNEL_TYPE_VXLAN:
11670                 pf->udp_ports[next_idx].type = I40E_AQC_TUNNEL_TYPE_VXLAN;
11671                 break;
11672         case UDP_TUNNEL_TYPE_GENEVE:
11673                 if (!(pf->hw_features & I40E_HW_GENEVE_OFFLOAD_CAPABLE))
11674                         return;
11675                 pf->udp_ports[next_idx].type = I40E_AQC_TUNNEL_TYPE_NGE;
11676                 break;
11677         default:
11678                 return;
11679         }
11680
11681         /* New port: add it and mark its index in the bitmap */
11682         pf->udp_ports[next_idx].port = port;
11683         pf->udp_ports[next_idx].filter_index = I40E_UDP_PORT_INDEX_UNUSED;
11684         pf->pending_udp_bitmap |= BIT_ULL(next_idx);
11685         set_bit(__I40E_UDP_FILTER_SYNC_PENDING, pf->state);
11686 }
11687
11688 /**
11689  * i40e_udp_tunnel_del - Get notifications about UDP tunnel ports that go away
11690  * @netdev: This physical port's netdev
11691  * @ti: Tunnel endpoint information
11692  **/
11693 static void i40e_udp_tunnel_del(struct net_device *netdev,
11694                                 struct udp_tunnel_info *ti)
11695 {
11696         struct i40e_netdev_priv *np = netdev_priv(netdev);
11697         struct i40e_vsi *vsi = np->vsi;
11698         struct i40e_pf *pf = vsi->back;
11699         u16 port = ntohs(ti->port);
11700         u8 idx;
11701
11702         idx = i40e_get_udp_port_idx(pf, port);
11703
11704         /* Check if port already exists */
11705         if (idx >= I40E_MAX_PF_UDP_OFFLOAD_PORTS)
11706                 goto not_found;
11707
11708         switch (ti->type) {
11709         case UDP_TUNNEL_TYPE_VXLAN:
11710                 if (pf->udp_ports[idx].type != I40E_AQC_TUNNEL_TYPE_VXLAN)
11711                         goto not_found;
11712                 break;
11713         case UDP_TUNNEL_TYPE_GENEVE:
11714                 if (pf->udp_ports[idx].type != I40E_AQC_TUNNEL_TYPE_NGE)
11715                         goto not_found;
11716                 break;
11717         default:
11718                 goto not_found;
11719         }
11720
11721         /* if port exists, set it to 0 (mark for deletion)
11722          * and make it pending
11723          */
11724         pf->udp_ports[idx].port = 0;
11725
11726         /* Toggle pending bit instead of setting it. This way if we are
11727          * deleting a port that has yet to be added we just clear the pending
11728          * bit and don't have to worry about it.
11729          */
11730         pf->pending_udp_bitmap ^= BIT_ULL(idx);
11731         set_bit(__I40E_UDP_FILTER_SYNC_PENDING, pf->state);
11732
11733         return;
11734 not_found:
11735         netdev_warn(netdev, "UDP port %d was not found, not deleting\n",
11736                     port);
11737 }
11738
11739 static int i40e_get_phys_port_id(struct net_device *netdev,
11740                                  struct netdev_phys_item_id *ppid)
11741 {
11742         struct i40e_netdev_priv *np = netdev_priv(netdev);
11743         struct i40e_pf *pf = np->vsi->back;
11744         struct i40e_hw *hw = &pf->hw;
11745
11746         if (!(pf->hw_features & I40E_HW_PORT_ID_VALID))
11747                 return -EOPNOTSUPP;
11748
11749         ppid->id_len = min_t(int, sizeof(hw->mac.port_addr), sizeof(ppid->id));
11750         memcpy(ppid->id, hw->mac.port_addr, ppid->id_len);
11751
11752         return 0;
11753 }
11754
11755 /**
11756  * i40e_ndo_fdb_add - add an entry to the hardware database
11757  * @ndm: the input from the stack
11758  * @tb: pointer to array of nladdr (unused)
11759  * @dev: the net device pointer
11760  * @addr: the MAC address entry being added
11761  * @vid: VLAN ID
11762  * @flags: instructions from stack about fdb operation
11763  */
11764 static int i40e_ndo_fdb_add(struct ndmsg *ndm, struct nlattr *tb[],
11765                             struct net_device *dev,
11766                             const unsigned char *addr, u16 vid,
11767                             u16 flags)
11768 {
11769         struct i40e_netdev_priv *np = netdev_priv(dev);
11770         struct i40e_pf *pf = np->vsi->back;
11771         int err = 0;
11772
11773         if (!(pf->flags & I40E_FLAG_SRIOV_ENABLED))
11774                 return -EOPNOTSUPP;
11775
11776         if (vid) {
11777                 pr_info("%s: vlans aren't supported yet for dev_uc|mc_add()\n", dev->name);
11778                 return -EINVAL;
11779         }
11780
11781         /* Hardware does not support aging addresses so if a
11782          * ndm_state is given only allow permanent addresses
11783          */
11784         if (ndm->ndm_state && !(ndm->ndm_state & NUD_PERMANENT)) {
11785                 netdev_info(dev, "FDB only supports static addresses\n");
11786                 return -EINVAL;
11787         }
11788
11789         if (is_unicast_ether_addr(addr) || is_link_local_ether_addr(addr))
11790                 err = dev_uc_add_excl(dev, addr);
11791         else if (is_multicast_ether_addr(addr))
11792                 err = dev_mc_add_excl(dev, addr);
11793         else
11794                 err = -EINVAL;
11795
11796         /* Only return duplicate errors if NLM_F_EXCL is set */
11797         if (err == -EEXIST && !(flags & NLM_F_EXCL))
11798                 err = 0;
11799
11800         return err;
11801 }
11802
11803 /**
11804  * i40e_ndo_bridge_setlink - Set the hardware bridge mode
11805  * @dev: the netdev being configured
11806  * @nlh: RTNL message
11807  * @flags: bridge flags
11808  *
11809  * Inserts a new hardware bridge if not already created and
11810  * enables the bridging mode requested (VEB or VEPA). If the
11811  * hardware bridge has already been inserted and the request
11812  * is to change the mode then that requires a PF reset to
11813  * allow rebuild of the components with required hardware
11814  * bridge mode enabled.
11815  *
11816  * Note: expects to be called while under rtnl_lock()
11817  **/
11818 static int i40e_ndo_bridge_setlink(struct net_device *dev,
11819                                    struct nlmsghdr *nlh,
11820                                    u16 flags)
11821 {
11822         struct i40e_netdev_priv *np = netdev_priv(dev);
11823         struct i40e_vsi *vsi = np->vsi;
11824         struct i40e_pf *pf = vsi->back;
11825         struct i40e_veb *veb = NULL;
11826         struct nlattr *attr, *br_spec;
11827         int i, rem;
11828
11829         /* Only for PF VSI for now */
11830         if (vsi->seid != pf->vsi[pf->lan_vsi]->seid)
11831                 return -EOPNOTSUPP;
11832
11833         /* Find the HW bridge for PF VSI */
11834         for (i = 0; i < I40E_MAX_VEB && !veb; i++) {
11835                 if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid)
11836                         veb = pf->veb[i];
11837         }
11838
11839         br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC);
11840         if (!br_spec)
11841                 return -EINVAL;
11842
11843         nla_for_each_nested(attr, br_spec, rem) {
11844                 __u16 mode;
11845
11846                 if (nla_type(attr) != IFLA_BRIDGE_MODE)
11847                         continue;
11848
11849                 mode = nla_get_u16(attr);
11850                 if ((mode != BRIDGE_MODE_VEPA) &&
11851                     (mode != BRIDGE_MODE_VEB))
11852                         return -EINVAL;
11853
11854                 /* Insert a new HW bridge */
11855                 if (!veb) {
11856                         veb = i40e_veb_setup(pf, 0, vsi->uplink_seid, vsi->seid,
11857                                              vsi->tc_config.enabled_tc);
11858                         if (veb) {
11859                                 veb->bridge_mode = mode;
11860                                 i40e_config_bridge_mode(veb);
11861                         } else {
11862                                 /* No Bridge HW offload available */
11863                                 return -ENOENT;
11864                         }
11865                         break;
11866                 } else if (mode != veb->bridge_mode) {
11867                         /* Existing HW bridge but different mode needs reset */
11868                         veb->bridge_mode = mode;
11869                         /* TODO: If no VFs or VMDq VSIs, disallow VEB mode */
11870                         if (mode == BRIDGE_MODE_VEB)
11871                                 pf->flags |= I40E_FLAG_VEB_MODE_ENABLED;
11872                         else
11873                                 pf->flags &= ~I40E_FLAG_VEB_MODE_ENABLED;
11874                         i40e_do_reset(pf, I40E_PF_RESET_FLAG, true);
11875                         break;
11876                 }
11877         }
11878
11879         return 0;
11880 }
11881
11882 /**
11883  * i40e_ndo_bridge_getlink - Get the hardware bridge mode
11884  * @skb: skb buff
11885  * @pid: process id
11886  * @seq: RTNL message seq #
11887  * @dev: the netdev being configured
11888  * @filter_mask: unused
11889  * @nlflags: netlink flags passed in
11890  *
11891  * Return the mode in which the hardware bridge is operating in
11892  * i.e VEB or VEPA.
11893  **/
11894 static int i40e_ndo_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
11895                                    struct net_device *dev,
11896                                    u32 __always_unused filter_mask,
11897                                    int nlflags)
11898 {
11899         struct i40e_netdev_priv *np = netdev_priv(dev);
11900         struct i40e_vsi *vsi = np->vsi;
11901         struct i40e_pf *pf = vsi->back;
11902         struct i40e_veb *veb = NULL;
11903         int i;
11904
11905         /* Only for PF VSI for now */
11906         if (vsi->seid != pf->vsi[pf->lan_vsi]->seid)
11907                 return -EOPNOTSUPP;
11908
11909         /* Find the HW bridge for the PF VSI */
11910         for (i = 0; i < I40E_MAX_VEB && !veb; i++) {
11911                 if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid)
11912                         veb = pf->veb[i];
11913         }
11914
11915         if (!veb)
11916                 return 0;
11917
11918         return ndo_dflt_bridge_getlink(skb, pid, seq, dev, veb->bridge_mode,
11919                                        0, 0, nlflags, filter_mask, NULL);
11920 }
11921
11922 /**
11923  * i40e_features_check - Validate encapsulated packet conforms to limits
11924  * @skb: skb buff
11925  * @dev: This physical port's netdev
11926  * @features: Offload features that the stack believes apply
11927  **/
11928 static netdev_features_t i40e_features_check(struct sk_buff *skb,
11929                                              struct net_device *dev,
11930                                              netdev_features_t features)
11931 {
11932         size_t len;
11933
11934         /* No point in doing any of this if neither checksum nor GSO are
11935          * being requested for this frame.  We can rule out both by just
11936          * checking for CHECKSUM_PARTIAL
11937          */
11938         if (skb->ip_summed != CHECKSUM_PARTIAL)
11939                 return features;
11940
11941         /* We cannot support GSO if the MSS is going to be less than
11942          * 64 bytes.  If it is then we need to drop support for GSO.
11943          */
11944         if (skb_is_gso(skb) && (skb_shinfo(skb)->gso_size < 64))
11945                 features &= ~NETIF_F_GSO_MASK;
11946
11947         /* MACLEN can support at most 63 words */
11948         len = skb_network_header(skb) - skb->data;
11949         if (len & ~(63 * 2))
11950                 goto out_err;
11951
11952         /* IPLEN and EIPLEN can support at most 127 dwords */
11953         len = skb_transport_header(skb) - skb_network_header(skb);
11954         if (len & ~(127 * 4))
11955                 goto out_err;
11956
11957         if (skb->encapsulation) {
11958                 /* L4TUNLEN can support 127 words */
11959                 len = skb_inner_network_header(skb) - skb_transport_header(skb);
11960                 if (len & ~(127 * 2))
11961                         goto out_err;
11962
11963                 /* IPLEN can support at most 127 dwords */
11964                 len = skb_inner_transport_header(skb) -
11965                       skb_inner_network_header(skb);
11966                 if (len & ~(127 * 4))
11967                         goto out_err;
11968         }
11969
11970         /* No need to validate L4LEN as TCP is the only protocol with a
11971          * a flexible value and we support all possible values supported
11972          * by TCP, which is at most 15 dwords
11973          */
11974
11975         return features;
11976 out_err:
11977         return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
11978 }
11979
11980 /**
11981  * i40e_xdp_setup - add/remove an XDP program
11982  * @vsi: VSI to changed
11983  * @prog: XDP program
11984  **/
11985 static int i40e_xdp_setup(struct i40e_vsi *vsi,
11986                           struct bpf_prog *prog)
11987 {
11988         int frame_size = vsi->netdev->mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;
11989         struct i40e_pf *pf = vsi->back;
11990         struct bpf_prog *old_prog;
11991         bool need_reset;
11992         int i;
11993
11994         /* Don't allow frames that span over multiple buffers */
11995         if (frame_size > vsi->rx_buf_len)
11996                 return -EINVAL;
11997
11998         if (!i40e_enabled_xdp_vsi(vsi) && !prog)
11999                 return 0;
12000
12001         /* When turning XDP on->off/off->on we reset and rebuild the rings. */
12002         need_reset = (i40e_enabled_xdp_vsi(vsi) != !!prog);
12003
12004         if (need_reset)
12005                 i40e_prep_for_reset(pf, true);
12006
12007         old_prog = xchg(&vsi->xdp_prog, prog);
12008
12009         if (need_reset)
12010                 i40e_reset_and_rebuild(pf, true, true);
12011
12012         for (i = 0; i < vsi->num_queue_pairs; i++)
12013                 WRITE_ONCE(vsi->rx_rings[i]->xdp_prog, vsi->xdp_prog);
12014
12015         if (old_prog)
12016                 bpf_prog_put(old_prog);
12017
12018         return 0;
12019 }
12020
12021 /**
12022  * i40e_xdp - implements ndo_bpf for i40e
12023  * @dev: netdevice
12024  * @xdp: XDP command
12025  **/
12026 static int i40e_xdp(struct net_device *dev,
12027                     struct netdev_bpf *xdp)
12028 {
12029         struct i40e_netdev_priv *np = netdev_priv(dev);
12030         struct i40e_vsi *vsi = np->vsi;
12031
12032         if (vsi->type != I40E_VSI_MAIN)
12033                 return -EINVAL;
12034
12035         switch (xdp->command) {
12036         case XDP_SETUP_PROG:
12037                 return i40e_xdp_setup(vsi, xdp->prog);
12038         case XDP_QUERY_PROG:
12039                 xdp->prog_id = vsi->xdp_prog ? vsi->xdp_prog->aux->id : 0;
12040                 return 0;
12041         default:
12042                 return -EINVAL;
12043         }
12044 }
12045
12046 static const struct net_device_ops i40e_netdev_ops = {
12047         .ndo_open               = i40e_open,
12048         .ndo_stop               = i40e_close,
12049         .ndo_start_xmit         = i40e_lan_xmit_frame,
12050         .ndo_get_stats64        = i40e_get_netdev_stats_struct,
12051         .ndo_set_rx_mode        = i40e_set_rx_mode,
12052         .ndo_validate_addr      = eth_validate_addr,
12053         .ndo_set_mac_address    = i40e_set_mac,
12054         .ndo_change_mtu         = i40e_change_mtu,
12055         .ndo_do_ioctl           = i40e_ioctl,
12056         .ndo_tx_timeout         = i40e_tx_timeout,
12057         .ndo_vlan_rx_add_vid    = i40e_vlan_rx_add_vid,
12058         .ndo_vlan_rx_kill_vid   = i40e_vlan_rx_kill_vid,
12059 #ifdef CONFIG_NET_POLL_CONTROLLER
12060         .ndo_poll_controller    = i40e_netpoll,
12061 #endif
12062         .ndo_setup_tc           = __i40e_setup_tc,
12063         .ndo_set_features       = i40e_set_features,
12064         .ndo_set_vf_mac         = i40e_ndo_set_vf_mac,
12065         .ndo_set_vf_vlan        = i40e_ndo_set_vf_port_vlan,
12066         .ndo_set_vf_rate        = i40e_ndo_set_vf_bw,
12067         .ndo_get_vf_config      = i40e_ndo_get_vf_config,
12068         .ndo_set_vf_link_state  = i40e_ndo_set_vf_link_state,
12069         .ndo_set_vf_spoofchk    = i40e_ndo_set_vf_spoofchk,
12070         .ndo_set_vf_trust       = i40e_ndo_set_vf_trust,
12071         .ndo_udp_tunnel_add     = i40e_udp_tunnel_add,
12072         .ndo_udp_tunnel_del     = i40e_udp_tunnel_del,
12073         .ndo_get_phys_port_id   = i40e_get_phys_port_id,
12074         .ndo_fdb_add            = i40e_ndo_fdb_add,
12075         .ndo_features_check     = i40e_features_check,
12076         .ndo_bridge_getlink     = i40e_ndo_bridge_getlink,
12077         .ndo_bridge_setlink     = i40e_ndo_bridge_setlink,
12078         .ndo_bpf                = i40e_xdp,
12079         .ndo_xdp_xmit           = i40e_xdp_xmit,
12080 };
12081
12082 /**
12083  * i40e_config_netdev - Setup the netdev flags
12084  * @vsi: the VSI being configured
12085  *
12086  * Returns 0 on success, negative value on failure
12087  **/
12088 static int i40e_config_netdev(struct i40e_vsi *vsi)
12089 {
12090         struct i40e_pf *pf = vsi->back;
12091         struct i40e_hw *hw = &pf->hw;
12092         struct i40e_netdev_priv *np;
12093         struct net_device *netdev;
12094         u8 broadcast[ETH_ALEN];
12095         u8 mac_addr[ETH_ALEN];
12096         int etherdev_size;
12097         netdev_features_t hw_enc_features;
12098         netdev_features_t hw_features;
12099
12100         etherdev_size = sizeof(struct i40e_netdev_priv);
12101         netdev = alloc_etherdev_mq(etherdev_size, vsi->alloc_queue_pairs);
12102         if (!netdev)
12103                 return -ENOMEM;
12104
12105         vsi->netdev = netdev;
12106         np = netdev_priv(netdev);
12107         np->vsi = vsi;
12108
12109         hw_enc_features = NETIF_F_SG                    |
12110                           NETIF_F_IP_CSUM               |
12111                           NETIF_F_IPV6_CSUM             |
12112                           NETIF_F_HIGHDMA               |
12113                           NETIF_F_SOFT_FEATURES         |
12114                           NETIF_F_TSO                   |
12115                           NETIF_F_TSO_ECN               |
12116                           NETIF_F_TSO6                  |
12117                           NETIF_F_GSO_GRE               |
12118                           NETIF_F_GSO_GRE_CSUM          |
12119                           NETIF_F_GSO_PARTIAL           |
12120                           NETIF_F_GSO_IPXIP4            |
12121                           NETIF_F_GSO_IPXIP6            |
12122                           NETIF_F_GSO_UDP_TUNNEL        |
12123                           NETIF_F_GSO_UDP_TUNNEL_CSUM   |
12124                           NETIF_F_SCTP_CRC              |
12125                           NETIF_F_RXHASH                |
12126                           NETIF_F_RXCSUM                |
12127                           0;
12128
12129         if (!(pf->hw_features & I40E_HW_OUTER_UDP_CSUM_CAPABLE))
12130                 netdev->gso_partial_features |= NETIF_F_GSO_UDP_TUNNEL_CSUM;
12131
12132         netdev->gso_partial_features |= NETIF_F_GSO_GRE_CSUM;
12133
12134         netdev->hw_enc_features |= hw_enc_features;
12135
12136         /* record features VLANs can make use of */
12137         netdev->vlan_features |= hw_enc_features | NETIF_F_TSO_MANGLEID;
12138
12139         if (!(pf->flags & I40E_FLAG_MFP_ENABLED))
12140                 netdev->hw_features |= NETIF_F_NTUPLE | NETIF_F_HW_TC;
12141
12142         hw_features = hw_enc_features           |
12143                       NETIF_F_HW_VLAN_CTAG_TX   |
12144                       NETIF_F_HW_VLAN_CTAG_RX;
12145
12146         netdev->hw_features |= hw_features;
12147
12148         netdev->features |= hw_features | NETIF_F_HW_VLAN_CTAG_FILTER;
12149         netdev->hw_enc_features |= NETIF_F_TSO_MANGLEID;
12150
12151         if (vsi->type == I40E_VSI_MAIN) {
12152                 SET_NETDEV_DEV(netdev, &pf->pdev->dev);
12153                 ether_addr_copy(mac_addr, hw->mac.perm_addr);
12154                 /* The following steps are necessary for two reasons. First,
12155                  * some older NVM configurations load a default MAC-VLAN
12156                  * filter that will accept any tagged packet, and we want to
12157                  * replace this with a normal filter. Additionally, it is
12158                  * possible our MAC address was provided by the platform using
12159                  * Open Firmware or similar.
12160                  *
12161                  * Thus, we need to remove the default filter and install one
12162                  * specific to the MAC address.
12163                  */
12164                 i40e_rm_default_mac_filter(vsi, mac_addr);
12165                 spin_lock_bh(&vsi->mac_filter_hash_lock);
12166                 i40e_add_mac_filter(vsi, mac_addr);
12167                 spin_unlock_bh(&vsi->mac_filter_hash_lock);
12168         } else {
12169                 /* Relate the VSI_VMDQ name to the VSI_MAIN name. Note that we
12170                  * are still limited by IFNAMSIZ, but we're adding 'v%d\0' to
12171                  * the end, which is 4 bytes long, so force truncation of the
12172                  * original name by IFNAMSIZ - 4
12173                  */
12174                 snprintf(netdev->name, IFNAMSIZ, "%.*sv%%d",
12175                          IFNAMSIZ - 4,
12176                          pf->vsi[pf->lan_vsi]->netdev->name);
12177                 eth_random_addr(mac_addr);
12178
12179                 spin_lock_bh(&vsi->mac_filter_hash_lock);
12180                 i40e_add_mac_filter(vsi, mac_addr);
12181                 spin_unlock_bh(&vsi->mac_filter_hash_lock);
12182         }
12183
12184         /* Add the broadcast filter so that we initially will receive
12185          * broadcast packets. Note that when a new VLAN is first added the
12186          * driver will convert all filters marked I40E_VLAN_ANY into VLAN
12187          * specific filters as part of transitioning into "vlan" operation.
12188          * When more VLANs are added, the driver will copy each existing MAC
12189          * filter and add it for the new VLAN.
12190          *
12191          * Broadcast filters are handled specially by
12192          * i40e_sync_filters_subtask, as the driver must to set the broadcast
12193          * promiscuous bit instead of adding this directly as a MAC/VLAN
12194          * filter. The subtask will update the correct broadcast promiscuous
12195          * bits as VLANs become active or inactive.
12196          */
12197         eth_broadcast_addr(broadcast);
12198         spin_lock_bh(&vsi->mac_filter_hash_lock);
12199         i40e_add_mac_filter(vsi, broadcast);
12200         spin_unlock_bh(&vsi->mac_filter_hash_lock);
12201
12202         ether_addr_copy(netdev->dev_addr, mac_addr);
12203         ether_addr_copy(netdev->perm_addr, mac_addr);
12204
12205         /* i40iw_net_event() reads 16 bytes from neigh->primary_key */
12206         netdev->neigh_priv_len = sizeof(u32) * 4;
12207
12208         netdev->priv_flags |= IFF_UNICAST_FLT;
12209         netdev->priv_flags |= IFF_SUPP_NOFCS;
12210         /* Setup netdev TC information */
12211         i40e_vsi_config_netdev_tc(vsi, vsi->tc_config.enabled_tc);
12212
12213         netdev->netdev_ops = &i40e_netdev_ops;
12214         netdev->watchdog_timeo = 5 * HZ;
12215         i40e_set_ethtool_ops(netdev);
12216
12217         /* MTU range: 68 - 9706 */
12218         netdev->min_mtu = ETH_MIN_MTU;
12219         netdev->max_mtu = I40E_MAX_RXBUFFER - I40E_PACKET_HDR_PAD;
12220
12221         return 0;
12222 }
12223
12224 /**
12225  * i40e_vsi_delete - Delete a VSI from the switch
12226  * @vsi: the VSI being removed
12227  *
12228  * Returns 0 on success, negative value on failure
12229  **/
12230 static void i40e_vsi_delete(struct i40e_vsi *vsi)
12231 {
12232         /* remove default VSI is not allowed */
12233         if (vsi == vsi->back->vsi[vsi->back->lan_vsi])
12234                 return;
12235
12236         i40e_aq_delete_element(&vsi->back->hw, vsi->seid, NULL);
12237 }
12238
12239 /**
12240  * i40e_is_vsi_uplink_mode_veb - Check if the VSI's uplink bridge mode is VEB
12241  * @vsi: the VSI being queried
12242  *
12243  * Returns 1 if HW bridge mode is VEB and return 0 in case of VEPA mode
12244  **/
12245 int i40e_is_vsi_uplink_mode_veb(struct i40e_vsi *vsi)
12246 {
12247         struct i40e_veb *veb;
12248         struct i40e_pf *pf = vsi->back;
12249
12250         /* Uplink is not a bridge so default to VEB */
12251         if (vsi->veb_idx == I40E_NO_VEB)
12252                 return 1;
12253
12254         veb = pf->veb[vsi->veb_idx];
12255         if (!veb) {
12256                 dev_info(&pf->pdev->dev,
12257                          "There is no veb associated with the bridge\n");
12258                 return -ENOENT;
12259         }
12260
12261         /* Uplink is a bridge in VEPA mode */
12262         if (veb->bridge_mode & BRIDGE_MODE_VEPA) {
12263                 return 0;
12264         } else {
12265                 /* Uplink is a bridge in VEB mode */
12266                 return 1;
12267         }
12268
12269         /* VEPA is now default bridge, so return 0 */
12270         return 0;
12271 }
12272
12273 /**
12274  * i40e_add_vsi - Add a VSI to the switch
12275  * @vsi: the VSI being configured
12276  *
12277  * This initializes a VSI context depending on the VSI type to be added and
12278  * passes it down to the add_vsi aq command.
12279  **/
12280 static int i40e_add_vsi(struct i40e_vsi *vsi)
12281 {
12282         int ret = -ENODEV;
12283         struct i40e_pf *pf = vsi->back;
12284         struct i40e_hw *hw = &pf->hw;
12285         struct i40e_vsi_context ctxt;
12286         struct i40e_mac_filter *f;
12287         struct hlist_node *h;
12288         int bkt;
12289
12290         u8 enabled_tc = 0x1; /* TC0 enabled */
12291         int f_count = 0;
12292
12293         memset(&ctxt, 0, sizeof(ctxt));
12294         switch (vsi->type) {
12295         case I40E_VSI_MAIN:
12296                 /* The PF's main VSI is already setup as part of the
12297                  * device initialization, so we'll not bother with
12298                  * the add_vsi call, but we will retrieve the current
12299                  * VSI context.
12300                  */
12301                 ctxt.seid = pf->main_vsi_seid;
12302                 ctxt.pf_num = pf->hw.pf_id;
12303                 ctxt.vf_num = 0;
12304                 ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL);
12305                 ctxt.flags = I40E_AQ_VSI_TYPE_PF;
12306                 if (ret) {
12307                         dev_info(&pf->pdev->dev,
12308                                  "couldn't get PF vsi config, err %s aq_err %s\n",
12309                                  i40e_stat_str(&pf->hw, ret),
12310                                  i40e_aq_str(&pf->hw,
12311                                              pf->hw.aq.asq_last_status));
12312                         return -ENOENT;
12313                 }
12314                 vsi->info = ctxt.info;
12315                 vsi->info.valid_sections = 0;
12316
12317                 vsi->seid = ctxt.seid;
12318                 vsi->id = ctxt.vsi_number;
12319
12320                 enabled_tc = i40e_pf_get_tc_map(pf);
12321
12322                 /* Source pruning is enabled by default, so the flag is
12323                  * negative logic - if it's set, we need to fiddle with
12324                  * the VSI to disable source pruning.
12325                  */
12326                 if (pf->flags & I40E_FLAG_SOURCE_PRUNING_DISABLED) {
12327                         memset(&ctxt, 0, sizeof(ctxt));
12328                         ctxt.seid = pf->main_vsi_seid;
12329                         ctxt.pf_num = pf->hw.pf_id;
12330                         ctxt.vf_num = 0;
12331                         ctxt.info.valid_sections |=
12332                                      cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
12333                         ctxt.info.switch_id =
12334                                    cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_LOCAL_LB);
12335                         ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
12336                         if (ret) {
12337                                 dev_info(&pf->pdev->dev,
12338                                          "update vsi failed, err %s aq_err %s\n",
12339                                          i40e_stat_str(&pf->hw, ret),
12340                                          i40e_aq_str(&pf->hw,
12341                                                      pf->hw.aq.asq_last_status));
12342                                 ret = -ENOENT;
12343                                 goto err;
12344                         }
12345                 }
12346
12347                 /* MFP mode setup queue map and update VSI */
12348                 if ((pf->flags & I40E_FLAG_MFP_ENABLED) &&
12349                     !(pf->hw.func_caps.iscsi)) { /* NIC type PF */
12350                         memset(&ctxt, 0, sizeof(ctxt));
12351                         ctxt.seid = pf->main_vsi_seid;
12352                         ctxt.pf_num = pf->hw.pf_id;
12353                         ctxt.vf_num = 0;
12354                         i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false);
12355                         ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
12356                         if (ret) {
12357                                 dev_info(&pf->pdev->dev,
12358                                          "update vsi failed, err %s aq_err %s\n",
12359                                          i40e_stat_str(&pf->hw, ret),
12360                                          i40e_aq_str(&pf->hw,
12361                                                     pf->hw.aq.asq_last_status));
12362                                 ret = -ENOENT;
12363                                 goto err;
12364                         }
12365                         /* update the local VSI info queue map */
12366                         i40e_vsi_update_queue_map(vsi, &ctxt);
12367                         vsi->info.valid_sections = 0;
12368                 } else {
12369                         /* Default/Main VSI is only enabled for TC0
12370                          * reconfigure it to enable all TCs that are
12371                          * available on the port in SFP mode.
12372                          * For MFP case the iSCSI PF would use this
12373                          * flow to enable LAN+iSCSI TC.
12374                          */
12375                         ret = i40e_vsi_config_tc(vsi, enabled_tc);
12376                         if (ret) {
12377                                 /* Single TC condition is not fatal,
12378                                  * message and continue
12379                                  */
12380                                 dev_info(&pf->pdev->dev,
12381                                          "failed to configure TCs for main VSI tc_map 0x%08x, err %s aq_err %s\n",
12382                                          enabled_tc,
12383                                          i40e_stat_str(&pf->hw, ret),
12384                                          i40e_aq_str(&pf->hw,
12385                                                     pf->hw.aq.asq_last_status));
12386                         }
12387                 }
12388                 break;
12389
12390         case I40E_VSI_FDIR:
12391                 ctxt.pf_num = hw->pf_id;
12392                 ctxt.vf_num = 0;
12393                 ctxt.uplink_seid = vsi->uplink_seid;
12394                 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
12395                 ctxt.flags = I40E_AQ_VSI_TYPE_PF;
12396                 if ((pf->flags & I40E_FLAG_VEB_MODE_ENABLED) &&
12397                     (i40e_is_vsi_uplink_mode_veb(vsi))) {
12398                         ctxt.info.valid_sections |=
12399                              cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
12400                         ctxt.info.switch_id =
12401                            cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
12402                 }
12403                 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true);
12404                 break;
12405
12406         case I40E_VSI_VMDQ2:
12407                 ctxt.pf_num = hw->pf_id;
12408                 ctxt.vf_num = 0;
12409                 ctxt.uplink_seid = vsi->uplink_seid;
12410                 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
12411                 ctxt.flags = I40E_AQ_VSI_TYPE_VMDQ2;
12412
12413                 /* This VSI is connected to VEB so the switch_id
12414                  * should be set to zero by default.
12415                  */
12416                 if (i40e_is_vsi_uplink_mode_veb(vsi)) {
12417                         ctxt.info.valid_sections |=
12418                                 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
12419                         ctxt.info.switch_id =
12420                                 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
12421                 }
12422
12423                 /* Setup the VSI tx/rx queue map for TC0 only for now */
12424                 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true);
12425                 break;
12426
12427         case I40E_VSI_SRIOV:
12428                 ctxt.pf_num = hw->pf_id;
12429                 ctxt.vf_num = vsi->vf_id + hw->func_caps.vf_base_id;
12430                 ctxt.uplink_seid = vsi->uplink_seid;
12431                 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
12432                 ctxt.flags = I40E_AQ_VSI_TYPE_VF;
12433
12434                 /* This VSI is connected to VEB so the switch_id
12435                  * should be set to zero by default.
12436                  */
12437                 if (i40e_is_vsi_uplink_mode_veb(vsi)) {
12438                         ctxt.info.valid_sections |=
12439                                 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
12440                         ctxt.info.switch_id =
12441                                 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
12442                 }
12443
12444                 if (vsi->back->flags & I40E_FLAG_IWARP_ENABLED) {
12445                         ctxt.info.valid_sections |=
12446                                 cpu_to_le16(I40E_AQ_VSI_PROP_QUEUE_OPT_VALID);
12447                         ctxt.info.queueing_opt_flags |=
12448                                 (I40E_AQ_VSI_QUE_OPT_TCP_ENA |
12449                                  I40E_AQ_VSI_QUE_OPT_RSS_LUT_VSI);
12450                 }
12451
12452                 ctxt.info.valid_sections |= cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
12453                 ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_MODE_ALL;
12454                 if (pf->vf[vsi->vf_id].spoofchk) {
12455                         ctxt.info.valid_sections |=
12456                                 cpu_to_le16(I40E_AQ_VSI_PROP_SECURITY_VALID);
12457                         ctxt.info.sec_flags |=
12458                                 (I40E_AQ_VSI_SEC_FLAG_ENABLE_VLAN_CHK |
12459                                  I40E_AQ_VSI_SEC_FLAG_ENABLE_MAC_CHK);
12460                 }
12461                 /* Setup the VSI tx/rx queue map for TC0 only for now */
12462                 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true);
12463                 break;
12464
12465         case I40E_VSI_IWARP:
12466                 /* send down message to iWARP */
12467                 break;
12468
12469         default:
12470                 return -ENODEV;
12471         }
12472
12473         if (vsi->type != I40E_VSI_MAIN) {
12474                 ret = i40e_aq_add_vsi(hw, &ctxt, NULL);
12475                 if (ret) {
12476                         dev_info(&vsi->back->pdev->dev,
12477                                  "add vsi failed, err %s aq_err %s\n",
12478                                  i40e_stat_str(&pf->hw, ret),
12479                                  i40e_aq_str(&pf->hw,
12480                                              pf->hw.aq.asq_last_status));
12481                         ret = -ENOENT;
12482                         goto err;
12483                 }
12484                 vsi->info = ctxt.info;
12485                 vsi->info.valid_sections = 0;
12486                 vsi->seid = ctxt.seid;
12487                 vsi->id = ctxt.vsi_number;
12488         }
12489
12490         spin_lock_bh(&vsi->mac_filter_hash_lock);
12491         vsi->active_filters = 0;
12492         /* If macvlan filters already exist, force them to get loaded */
12493         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
12494                 f->state = I40E_FILTER_NEW;
12495                 f_count++;
12496         }
12497         spin_unlock_bh(&vsi->mac_filter_hash_lock);
12498         clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
12499
12500         if (f_count) {
12501                 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
12502                 set_bit(__I40E_MACVLAN_SYNC_PENDING, pf->state);
12503         }
12504
12505         /* Update VSI BW information */
12506         ret = i40e_vsi_get_bw_info(vsi);
12507         if (ret) {
12508                 dev_info(&pf->pdev->dev,
12509                          "couldn't get vsi bw info, err %s aq_err %s\n",
12510                          i40e_stat_str(&pf->hw, ret),
12511                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
12512                 /* VSI is already added so not tearing that up */
12513                 ret = 0;
12514         }
12515
12516 err:
12517         return ret;
12518 }
12519
12520 /**
12521  * i40e_vsi_release - Delete a VSI and free its resources
12522  * @vsi: the VSI being removed
12523  *
12524  * Returns 0 on success or < 0 on error
12525  **/
12526 int i40e_vsi_release(struct i40e_vsi *vsi)
12527 {
12528         struct i40e_mac_filter *f;
12529         struct hlist_node *h;
12530         struct i40e_veb *veb = NULL;
12531         struct i40e_pf *pf;
12532         u16 uplink_seid;
12533         int i, n, bkt;
12534
12535         pf = vsi->back;
12536
12537         /* release of a VEB-owner or last VSI is not allowed */
12538         if (vsi->flags & I40E_VSI_FLAG_VEB_OWNER) {
12539                 dev_info(&pf->pdev->dev, "VSI %d has existing VEB %d\n",
12540                          vsi->seid, vsi->uplink_seid);
12541                 return -ENODEV;
12542         }
12543         if (vsi == pf->vsi[pf->lan_vsi] &&
12544             !test_bit(__I40E_DOWN, pf->state)) {
12545                 dev_info(&pf->pdev->dev, "Can't remove PF VSI\n");
12546                 return -ENODEV;
12547         }
12548         set_bit(__I40E_VSI_RELEASING, vsi->state);
12549         uplink_seid = vsi->uplink_seid;
12550         if (vsi->type != I40E_VSI_SRIOV) {
12551                 if (vsi->netdev_registered) {
12552                         vsi->netdev_registered = false;
12553                         if (vsi->netdev) {
12554                                 /* results in a call to i40e_close() */
12555                                 unregister_netdev(vsi->netdev);
12556                         }
12557                 } else {
12558                         i40e_vsi_close(vsi);
12559                 }
12560                 i40e_vsi_disable_irq(vsi);
12561         }
12562
12563         spin_lock_bh(&vsi->mac_filter_hash_lock);
12564
12565         /* clear the sync flag on all filters */
12566         if (vsi->netdev) {
12567                 __dev_uc_unsync(vsi->netdev, NULL);
12568                 __dev_mc_unsync(vsi->netdev, NULL);
12569         }
12570
12571         /* make sure any remaining filters are marked for deletion */
12572         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist)
12573                 __i40e_del_filter(vsi, f);
12574
12575         spin_unlock_bh(&vsi->mac_filter_hash_lock);
12576
12577         i40e_sync_vsi_filters(vsi);
12578
12579         i40e_vsi_delete(vsi);
12580         i40e_vsi_free_q_vectors(vsi);
12581         if (vsi->netdev) {
12582                 free_netdev(vsi->netdev);
12583                 vsi->netdev = NULL;
12584         }
12585         i40e_vsi_clear_rings(vsi);
12586         i40e_vsi_clear(vsi);
12587
12588         /* If this was the last thing on the VEB, except for the
12589          * controlling VSI, remove the VEB, which puts the controlling
12590          * VSI onto the next level down in the switch.
12591          *
12592          * Well, okay, there's one more exception here: don't remove
12593          * the orphan VEBs yet.  We'll wait for an explicit remove request
12594          * from up the network stack.
12595          */
12596         for (n = 0, i = 0; i < pf->num_alloc_vsi; i++) {
12597                 if (pf->vsi[i] &&
12598                     pf->vsi[i]->uplink_seid == uplink_seid &&
12599                     (pf->vsi[i]->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) {
12600                         n++;      /* count the VSIs */
12601                 }
12602         }
12603         for (i = 0; i < I40E_MAX_VEB; i++) {
12604                 if (!pf->veb[i])
12605                         continue;
12606                 if (pf->veb[i]->uplink_seid == uplink_seid)
12607                         n++;     /* count the VEBs */
12608                 if (pf->veb[i]->seid == uplink_seid)
12609                         veb = pf->veb[i];
12610         }
12611         if (n == 0 && veb && veb->uplink_seid != 0)
12612                 i40e_veb_release(veb);
12613
12614         return 0;
12615 }
12616
12617 /**
12618  * i40e_vsi_setup_vectors - Set up the q_vectors for the given VSI
12619  * @vsi: ptr to the VSI
12620  *
12621  * This should only be called after i40e_vsi_mem_alloc() which allocates the
12622  * corresponding SW VSI structure and initializes num_queue_pairs for the
12623  * newly allocated VSI.
12624  *
12625  * Returns 0 on success or negative on failure
12626  **/
12627 static int i40e_vsi_setup_vectors(struct i40e_vsi *vsi)
12628 {
12629         int ret = -ENOENT;
12630         struct i40e_pf *pf = vsi->back;
12631
12632         if (vsi->q_vectors[0]) {
12633                 dev_info(&pf->pdev->dev, "VSI %d has existing q_vectors\n",
12634                          vsi->seid);
12635                 return -EEXIST;
12636         }
12637
12638         if (vsi->base_vector) {
12639                 dev_info(&pf->pdev->dev, "VSI %d has non-zero base vector %d\n",
12640                          vsi->seid, vsi->base_vector);
12641                 return -EEXIST;
12642         }
12643
12644         ret = i40e_vsi_alloc_q_vectors(vsi);
12645         if (ret) {
12646                 dev_info(&pf->pdev->dev,
12647                          "failed to allocate %d q_vector for VSI %d, ret=%d\n",
12648                          vsi->num_q_vectors, vsi->seid, ret);
12649                 vsi->num_q_vectors = 0;
12650                 goto vector_setup_out;
12651         }
12652
12653         /* In Legacy mode, we do not have to get any other vector since we
12654          * piggyback on the misc/ICR0 for queue interrupts.
12655         */
12656         if (!(pf->flags & I40E_FLAG_MSIX_ENABLED))
12657                 return ret;
12658         if (vsi->num_q_vectors)
12659                 vsi->base_vector = i40e_get_lump(pf, pf->irq_pile,
12660                                                  vsi->num_q_vectors, vsi->idx);
12661         if (vsi->base_vector < 0) {
12662                 dev_info(&pf->pdev->dev,
12663                          "failed to get tracking for %d vectors for VSI %d, err=%d\n",
12664                          vsi->num_q_vectors, vsi->seid, vsi->base_vector);
12665                 i40e_vsi_free_q_vectors(vsi);
12666                 ret = -ENOENT;
12667                 goto vector_setup_out;
12668         }
12669
12670 vector_setup_out:
12671         return ret;
12672 }
12673
12674 /**
12675  * i40e_vsi_reinit_setup - return and reallocate resources for a VSI
12676  * @vsi: pointer to the vsi.
12677  *
12678  * This re-allocates a vsi's queue resources.
12679  *
12680  * Returns pointer to the successfully allocated and configured VSI sw struct
12681  * on success, otherwise returns NULL on failure.
12682  **/
12683 static struct i40e_vsi *i40e_vsi_reinit_setup(struct i40e_vsi *vsi)
12684 {
12685         u16 alloc_queue_pairs;
12686         struct i40e_pf *pf;
12687         u8 enabled_tc;
12688         int ret;
12689
12690         if (!vsi)
12691                 return NULL;
12692
12693         pf = vsi->back;
12694
12695         i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx);
12696         i40e_vsi_clear_rings(vsi);
12697
12698         i40e_vsi_free_arrays(vsi, false);
12699         i40e_set_num_rings_in_vsi(vsi);
12700         ret = i40e_vsi_alloc_arrays(vsi, false);
12701         if (ret)
12702                 goto err_vsi;
12703
12704         alloc_queue_pairs = vsi->alloc_queue_pairs *
12705                             (i40e_enabled_xdp_vsi(vsi) ? 2 : 1);
12706
12707         ret = i40e_get_lump(pf, pf->qp_pile, alloc_queue_pairs, vsi->idx);
12708         if (ret < 0) {
12709                 dev_info(&pf->pdev->dev,
12710                          "failed to get tracking for %d queues for VSI %d err %d\n",
12711                          alloc_queue_pairs, vsi->seid, ret);
12712                 goto err_vsi;
12713         }
12714         vsi->base_queue = ret;
12715
12716         /* Update the FW view of the VSI. Force a reset of TC and queue
12717          * layout configurations.
12718          */
12719         enabled_tc = pf->vsi[pf->lan_vsi]->tc_config.enabled_tc;
12720         pf->vsi[pf->lan_vsi]->tc_config.enabled_tc = 0;
12721         pf->vsi[pf->lan_vsi]->seid = pf->main_vsi_seid;
12722         i40e_vsi_config_tc(pf->vsi[pf->lan_vsi], enabled_tc);
12723         if (vsi->type == I40E_VSI_MAIN)
12724                 i40e_rm_default_mac_filter(vsi, pf->hw.mac.perm_addr);
12725
12726         /* assign it some queues */
12727         ret = i40e_alloc_rings(vsi);
12728         if (ret)
12729                 goto err_rings;
12730
12731         /* map all of the rings to the q_vectors */
12732         i40e_vsi_map_rings_to_vectors(vsi);
12733         return vsi;
12734
12735 err_rings:
12736         i40e_vsi_free_q_vectors(vsi);
12737         if (vsi->netdev_registered) {
12738                 vsi->netdev_registered = false;
12739                 unregister_netdev(vsi->netdev);
12740                 free_netdev(vsi->netdev);
12741                 vsi->netdev = NULL;
12742         }
12743         i40e_aq_delete_element(&pf->hw, vsi->seid, NULL);
12744 err_vsi:
12745         i40e_vsi_clear(vsi);
12746         return NULL;
12747 }
12748
12749 /**
12750  * i40e_vsi_setup - Set up a VSI by a given type
12751  * @pf: board private structure
12752  * @type: VSI type
12753  * @uplink_seid: the switch element to link to
12754  * @param1: usage depends upon VSI type. For VF types, indicates VF id
12755  *
12756  * This allocates the sw VSI structure and its queue resources, then add a VSI
12757  * to the identified VEB.
12758  *
12759  * Returns pointer to the successfully allocated and configure VSI sw struct on
12760  * success, otherwise returns NULL on failure.
12761  **/
12762 struct i40e_vsi *i40e_vsi_setup(struct i40e_pf *pf, u8 type,
12763                                 u16 uplink_seid, u32 param1)
12764 {
12765         struct i40e_vsi *vsi = NULL;
12766         struct i40e_veb *veb = NULL;
12767         u16 alloc_queue_pairs;
12768         int ret, i;
12769         int v_idx;
12770
12771         /* The requested uplink_seid must be either
12772          *     - the PF's port seid
12773          *              no VEB is needed because this is the PF
12774          *              or this is a Flow Director special case VSI
12775          *     - seid of an existing VEB
12776          *     - seid of a VSI that owns an existing VEB
12777          *     - seid of a VSI that doesn't own a VEB
12778          *              a new VEB is created and the VSI becomes the owner
12779          *     - seid of the PF VSI, which is what creates the first VEB
12780          *              this is a special case of the previous
12781          *
12782          * Find which uplink_seid we were given and create a new VEB if needed
12783          */
12784         for (i = 0; i < I40E_MAX_VEB; i++) {
12785                 if (pf->veb[i] && pf->veb[i]->seid == uplink_seid) {
12786                         veb = pf->veb[i];
12787                         break;
12788                 }
12789         }
12790
12791         if (!veb && uplink_seid != pf->mac_seid) {
12792
12793                 for (i = 0; i < pf->num_alloc_vsi; i++) {
12794                         if (pf->vsi[i] && pf->vsi[i]->seid == uplink_seid) {
12795                                 vsi = pf->vsi[i];
12796                                 break;
12797                         }
12798                 }
12799                 if (!vsi) {
12800                         dev_info(&pf->pdev->dev, "no such uplink_seid %d\n",
12801                                  uplink_seid);
12802                         return NULL;
12803                 }
12804
12805                 if (vsi->uplink_seid == pf->mac_seid)
12806                         veb = i40e_veb_setup(pf, 0, pf->mac_seid, vsi->seid,
12807                                              vsi->tc_config.enabled_tc);
12808                 else if ((vsi->flags & I40E_VSI_FLAG_VEB_OWNER) == 0)
12809                         veb = i40e_veb_setup(pf, 0, vsi->uplink_seid, vsi->seid,
12810                                              vsi->tc_config.enabled_tc);
12811                 if (veb) {
12812                         if (vsi->seid != pf->vsi[pf->lan_vsi]->seid) {
12813                                 dev_info(&vsi->back->pdev->dev,
12814                                          "New VSI creation error, uplink seid of LAN VSI expected.\n");
12815                                 return NULL;
12816                         }
12817                         /* We come up by default in VEPA mode if SRIOV is not
12818                          * already enabled, in which case we can't force VEPA
12819                          * mode.
12820                          */
12821                         if (!(pf->flags & I40E_FLAG_VEB_MODE_ENABLED)) {
12822                                 veb->bridge_mode = BRIDGE_MODE_VEPA;
12823                                 pf->flags &= ~I40E_FLAG_VEB_MODE_ENABLED;
12824                         }
12825                         i40e_config_bridge_mode(veb);
12826                 }
12827                 for (i = 0; i < I40E_MAX_VEB && !veb; i++) {
12828                         if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid)
12829                                 veb = pf->veb[i];
12830                 }
12831                 if (!veb) {
12832                         dev_info(&pf->pdev->dev, "couldn't add VEB\n");
12833                         return NULL;
12834                 }
12835
12836                 vsi->flags |= I40E_VSI_FLAG_VEB_OWNER;
12837                 uplink_seid = veb->seid;
12838         }
12839
12840         /* get vsi sw struct */
12841         v_idx = i40e_vsi_mem_alloc(pf, type);
12842         if (v_idx < 0)
12843                 goto err_alloc;
12844         vsi = pf->vsi[v_idx];
12845         if (!vsi)
12846                 goto err_alloc;
12847         vsi->type = type;
12848         vsi->veb_idx = (veb ? veb->idx : I40E_NO_VEB);
12849
12850         if (type == I40E_VSI_MAIN)
12851                 pf->lan_vsi = v_idx;
12852         else if (type == I40E_VSI_SRIOV)
12853                 vsi->vf_id = param1;
12854         /* assign it some queues */
12855         alloc_queue_pairs = vsi->alloc_queue_pairs *
12856                             (i40e_enabled_xdp_vsi(vsi) ? 2 : 1);
12857
12858         ret = i40e_get_lump(pf, pf->qp_pile, alloc_queue_pairs, vsi->idx);
12859         if (ret < 0) {
12860                 dev_info(&pf->pdev->dev,
12861                          "failed to get tracking for %d queues for VSI %d err=%d\n",
12862                          alloc_queue_pairs, vsi->seid, ret);
12863                 goto err_vsi;
12864         }
12865         vsi->base_queue = ret;
12866
12867         /* get a VSI from the hardware */
12868         vsi->uplink_seid = uplink_seid;
12869         ret = i40e_add_vsi(vsi);
12870         if (ret)
12871                 goto err_vsi;
12872
12873         switch (vsi->type) {
12874         /* setup the netdev if needed */
12875         case I40E_VSI_MAIN:
12876         case I40E_VSI_VMDQ2:
12877                 ret = i40e_config_netdev(vsi);
12878                 if (ret)
12879                         goto err_netdev;
12880                 ret = i40e_netif_set_realnum_tx_rx_queues(vsi);
12881                 if (ret)
12882                         goto err_netdev;
12883                 ret = register_netdev(vsi->netdev);
12884                 if (ret)
12885                         goto err_netdev;
12886                 vsi->netdev_registered = true;
12887                 netif_carrier_off(vsi->netdev);
12888 #ifdef CONFIG_I40E_DCB
12889                 /* Setup DCB netlink interface */
12890                 i40e_dcbnl_setup(vsi);
12891 #endif /* CONFIG_I40E_DCB */
12892                 /* fall through */
12893
12894         case I40E_VSI_FDIR:
12895                 /* set up vectors and rings if needed */
12896                 ret = i40e_vsi_setup_vectors(vsi);
12897                 if (ret)
12898                         goto err_msix;
12899
12900                 ret = i40e_alloc_rings(vsi);
12901                 if (ret)
12902                         goto err_rings;
12903
12904                 /* map all of the rings to the q_vectors */
12905                 i40e_vsi_map_rings_to_vectors(vsi);
12906
12907                 i40e_vsi_reset_stats(vsi);
12908                 break;
12909
12910         default:
12911                 /* no netdev or rings for the other VSI types */
12912                 break;
12913         }
12914
12915         if ((pf->hw_features & I40E_HW_RSS_AQ_CAPABLE) &&
12916             (vsi->type == I40E_VSI_VMDQ2)) {
12917                 ret = i40e_vsi_config_rss(vsi);
12918         }
12919         return vsi;
12920
12921 err_rings:
12922         i40e_vsi_free_q_vectors(vsi);
12923 err_msix:
12924         if (vsi->netdev_registered) {
12925                 vsi->netdev_registered = false;
12926                 unregister_netdev(vsi->netdev);
12927                 free_netdev(vsi->netdev);
12928                 vsi->netdev = NULL;
12929         }
12930 err_netdev:
12931         i40e_aq_delete_element(&pf->hw, vsi->seid, NULL);
12932 err_vsi:
12933         i40e_vsi_clear(vsi);
12934 err_alloc:
12935         return NULL;
12936 }
12937
12938 /**
12939  * i40e_veb_get_bw_info - Query VEB BW information
12940  * @veb: the veb to query
12941  *
12942  * Query the Tx scheduler BW configuration data for given VEB
12943  **/
12944 static int i40e_veb_get_bw_info(struct i40e_veb *veb)
12945 {
12946         struct i40e_aqc_query_switching_comp_ets_config_resp ets_data;
12947         struct i40e_aqc_query_switching_comp_bw_config_resp bw_data;
12948         struct i40e_pf *pf = veb->pf;
12949         struct i40e_hw *hw = &pf->hw;
12950         u32 tc_bw_max;
12951         int ret = 0;
12952         int i;
12953
12954         ret = i40e_aq_query_switch_comp_bw_config(hw, veb->seid,
12955                                                   &bw_data, NULL);
12956         if (ret) {
12957                 dev_info(&pf->pdev->dev,
12958                          "query veb bw config failed, err %s aq_err %s\n",
12959                          i40e_stat_str(&pf->hw, ret),
12960                          i40e_aq_str(&pf->hw, hw->aq.asq_last_status));
12961                 goto out;
12962         }
12963
12964         ret = i40e_aq_query_switch_comp_ets_config(hw, veb->seid,
12965                                                    &ets_data, NULL);
12966         if (ret) {
12967                 dev_info(&pf->pdev->dev,
12968                          "query veb bw ets config failed, err %s aq_err %s\n",
12969                          i40e_stat_str(&pf->hw, ret),
12970                          i40e_aq_str(&pf->hw, hw->aq.asq_last_status));
12971                 goto out;
12972         }
12973
12974         veb->bw_limit = le16_to_cpu(ets_data.port_bw_limit);
12975         veb->bw_max_quanta = ets_data.tc_bw_max;
12976         veb->is_abs_credits = bw_data.absolute_credits_enable;
12977         veb->enabled_tc = ets_data.tc_valid_bits;
12978         tc_bw_max = le16_to_cpu(bw_data.tc_bw_max[0]) |
12979                     (le16_to_cpu(bw_data.tc_bw_max[1]) << 16);
12980         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
12981                 veb->bw_tc_share_credits[i] = bw_data.tc_bw_share_credits[i];
12982                 veb->bw_tc_limit_credits[i] =
12983                                         le16_to_cpu(bw_data.tc_bw_limits[i]);
12984                 veb->bw_tc_max_quanta[i] = ((tc_bw_max >> (i*4)) & 0x7);
12985         }
12986
12987 out:
12988         return ret;
12989 }
12990
12991 /**
12992  * i40e_veb_mem_alloc - Allocates the next available struct veb in the PF
12993  * @pf: board private structure
12994  *
12995  * On error: returns error code (negative)
12996  * On success: returns vsi index in PF (positive)
12997  **/
12998 static int i40e_veb_mem_alloc(struct i40e_pf *pf)
12999 {
13000         int ret = -ENOENT;
13001         struct i40e_veb *veb;
13002         int i;
13003
13004         /* Need to protect the allocation of switch elements at the PF level */
13005         mutex_lock(&pf->switch_mutex);
13006
13007         /* VEB list may be fragmented if VEB creation/destruction has
13008          * been happening.  We can afford to do a quick scan to look
13009          * for any free slots in the list.
13010          *
13011          * find next empty veb slot, looping back around if necessary
13012          */
13013         i = 0;
13014         while ((i < I40E_MAX_VEB) && (pf->veb[i] != NULL))
13015                 i++;
13016         if (i >= I40E_MAX_VEB) {
13017                 ret = -ENOMEM;
13018                 goto err_alloc_veb;  /* out of VEB slots! */
13019         }
13020
13021         veb = kzalloc(sizeof(*veb), GFP_KERNEL);
13022         if (!veb) {
13023                 ret = -ENOMEM;
13024                 goto err_alloc_veb;
13025         }
13026         veb->pf = pf;
13027         veb->idx = i;
13028         veb->enabled_tc = 1;
13029
13030         pf->veb[i] = veb;
13031         ret = i;
13032 err_alloc_veb:
13033         mutex_unlock(&pf->switch_mutex);
13034         return ret;
13035 }
13036
13037 /**
13038  * i40e_switch_branch_release - Delete a branch of the switch tree
13039  * @branch: where to start deleting
13040  *
13041  * This uses recursion to find the tips of the branch to be
13042  * removed, deleting until we get back to and can delete this VEB.
13043  **/
13044 static void i40e_switch_branch_release(struct i40e_veb *branch)
13045 {
13046         struct i40e_pf *pf = branch->pf;
13047         u16 branch_seid = branch->seid;
13048         u16 veb_idx = branch->idx;
13049         int i;
13050
13051         /* release any VEBs on this VEB - RECURSION */
13052         for (i = 0; i < I40E_MAX_VEB; i++) {
13053                 if (!pf->veb[i])
13054                         continue;
13055                 if (pf->veb[i]->uplink_seid == branch->seid)
13056                         i40e_switch_branch_release(pf->veb[i]);
13057         }
13058
13059         /* Release the VSIs on this VEB, but not the owner VSI.
13060          *
13061          * NOTE: Removing the last VSI on a VEB has the SIDE EFFECT of removing
13062          *       the VEB itself, so don't use (*branch) after this loop.
13063          */
13064         for (i = 0; i < pf->num_alloc_vsi; i++) {
13065                 if (!pf->vsi[i])
13066                         continue;
13067                 if (pf->vsi[i]->uplink_seid == branch_seid &&
13068                    (pf->vsi[i]->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) {
13069                         i40e_vsi_release(pf->vsi[i]);
13070                 }
13071         }
13072
13073         /* There's one corner case where the VEB might not have been
13074          * removed, so double check it here and remove it if needed.
13075          * This case happens if the veb was created from the debugfs
13076          * commands and no VSIs were added to it.
13077          */
13078         if (pf->veb[veb_idx])
13079                 i40e_veb_release(pf->veb[veb_idx]);
13080 }
13081
13082 /**
13083  * i40e_veb_clear - remove veb struct
13084  * @veb: the veb to remove
13085  **/
13086 static void i40e_veb_clear(struct i40e_veb *veb)
13087 {
13088         if (!veb)
13089                 return;
13090
13091         if (veb->pf) {
13092                 struct i40e_pf *pf = veb->pf;
13093
13094                 mutex_lock(&pf->switch_mutex);
13095                 if (pf->veb[veb->idx] == veb)
13096                         pf->veb[veb->idx] = NULL;
13097                 mutex_unlock(&pf->switch_mutex);
13098         }
13099
13100         kfree(veb);
13101 }
13102
13103 /**
13104  * i40e_veb_release - Delete a VEB and free its resources
13105  * @veb: the VEB being removed
13106  **/
13107 void i40e_veb_release(struct i40e_veb *veb)
13108 {
13109         struct i40e_vsi *vsi = NULL;
13110         struct i40e_pf *pf;
13111         int i, n = 0;
13112
13113         pf = veb->pf;
13114
13115         /* find the remaining VSI and check for extras */
13116         for (i = 0; i < pf->num_alloc_vsi; i++) {
13117                 if (pf->vsi[i] && pf->vsi[i]->uplink_seid == veb->seid) {
13118                         n++;
13119                         vsi = pf->vsi[i];
13120                 }
13121         }
13122         if (n != 1) {
13123                 dev_info(&pf->pdev->dev,
13124                          "can't remove VEB %d with %d VSIs left\n",
13125                          veb->seid, n);
13126                 return;
13127         }
13128
13129         /* move the remaining VSI to uplink veb */
13130         vsi->flags &= ~I40E_VSI_FLAG_VEB_OWNER;
13131         if (veb->uplink_seid) {
13132                 vsi->uplink_seid = veb->uplink_seid;
13133                 if (veb->uplink_seid == pf->mac_seid)
13134                         vsi->veb_idx = I40E_NO_VEB;
13135                 else
13136                         vsi->veb_idx = veb->veb_idx;
13137         } else {
13138                 /* floating VEB */
13139                 vsi->uplink_seid = pf->vsi[pf->lan_vsi]->uplink_seid;
13140                 vsi->veb_idx = pf->vsi[pf->lan_vsi]->veb_idx;
13141         }
13142
13143         i40e_aq_delete_element(&pf->hw, veb->seid, NULL);
13144         i40e_veb_clear(veb);
13145 }
13146
13147 /**
13148  * i40e_add_veb - create the VEB in the switch
13149  * @veb: the VEB to be instantiated
13150  * @vsi: the controlling VSI
13151  **/
13152 static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi)
13153 {
13154         struct i40e_pf *pf = veb->pf;
13155         bool enable_stats = !!(pf->flags & I40E_FLAG_VEB_STATS_ENABLED);
13156         int ret;
13157
13158         ret = i40e_aq_add_veb(&pf->hw, veb->uplink_seid, vsi->seid,
13159                               veb->enabled_tc, false,
13160                               &veb->seid, enable_stats, NULL);
13161
13162         /* get a VEB from the hardware */
13163         if (ret) {
13164                 dev_info(&pf->pdev->dev,
13165                          "couldn't add VEB, err %s aq_err %s\n",
13166                          i40e_stat_str(&pf->hw, ret),
13167                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
13168                 return -EPERM;
13169         }
13170
13171         /* get statistics counter */
13172         ret = i40e_aq_get_veb_parameters(&pf->hw, veb->seid, NULL, NULL,
13173                                          &veb->stats_idx, NULL, NULL, NULL);
13174         if (ret) {
13175                 dev_info(&pf->pdev->dev,
13176                          "couldn't get VEB statistics idx, err %s aq_err %s\n",
13177                          i40e_stat_str(&pf->hw, ret),
13178                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
13179                 return -EPERM;
13180         }
13181         ret = i40e_veb_get_bw_info(veb);
13182         if (ret) {
13183                 dev_info(&pf->pdev->dev,
13184                          "couldn't get VEB bw info, err %s aq_err %s\n",
13185                          i40e_stat_str(&pf->hw, ret),
13186                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
13187                 i40e_aq_delete_element(&pf->hw, veb->seid, NULL);
13188                 return -ENOENT;
13189         }
13190
13191         vsi->uplink_seid = veb->seid;
13192         vsi->veb_idx = veb->idx;
13193         vsi->flags |= I40E_VSI_FLAG_VEB_OWNER;
13194
13195         return 0;
13196 }
13197
13198 /**
13199  * i40e_veb_setup - Set up a VEB
13200  * @pf: board private structure
13201  * @flags: VEB setup flags
13202  * @uplink_seid: the switch element to link to
13203  * @vsi_seid: the initial VSI seid
13204  * @enabled_tc: Enabled TC bit-map
13205  *
13206  * This allocates the sw VEB structure and links it into the switch
13207  * It is possible and legal for this to be a duplicate of an already
13208  * existing VEB.  It is also possible for both uplink and vsi seids
13209  * to be zero, in order to create a floating VEB.
13210  *
13211  * Returns pointer to the successfully allocated VEB sw struct on
13212  * success, otherwise returns NULL on failure.
13213  **/
13214 struct i40e_veb *i40e_veb_setup(struct i40e_pf *pf, u16 flags,
13215                                 u16 uplink_seid, u16 vsi_seid,
13216                                 u8 enabled_tc)
13217 {
13218         struct i40e_veb *veb, *uplink_veb = NULL;
13219         int vsi_idx, veb_idx;
13220         int ret;
13221
13222         /* if one seid is 0, the other must be 0 to create a floating relay */
13223         if ((uplink_seid == 0 || vsi_seid == 0) &&
13224             (uplink_seid + vsi_seid != 0)) {
13225                 dev_info(&pf->pdev->dev,
13226                          "one, not both seid's are 0: uplink=%d vsi=%d\n",
13227                          uplink_seid, vsi_seid);
13228                 return NULL;
13229         }
13230
13231         /* make sure there is such a vsi and uplink */
13232         for (vsi_idx = 0; vsi_idx < pf->num_alloc_vsi; vsi_idx++)
13233                 if (pf->vsi[vsi_idx] && pf->vsi[vsi_idx]->seid == vsi_seid)
13234                         break;
13235         if (vsi_idx >= pf->num_alloc_vsi && vsi_seid != 0) {
13236                 dev_info(&pf->pdev->dev, "vsi seid %d not found\n",
13237                          vsi_seid);
13238                 return NULL;
13239         }
13240
13241         if (uplink_seid && uplink_seid != pf->mac_seid) {
13242                 for (veb_idx = 0; veb_idx < I40E_MAX_VEB; veb_idx++) {
13243                         if (pf->veb[veb_idx] &&
13244                             pf->veb[veb_idx]->seid == uplink_seid) {
13245                                 uplink_veb = pf->veb[veb_idx];
13246                                 break;
13247                         }
13248                 }
13249                 if (!uplink_veb) {
13250                         dev_info(&pf->pdev->dev,
13251                                  "uplink seid %d not found\n", uplink_seid);
13252                         return NULL;
13253                 }
13254         }
13255
13256         /* get veb sw struct */
13257         veb_idx = i40e_veb_mem_alloc(pf);
13258         if (veb_idx < 0)
13259                 goto err_alloc;
13260         veb = pf->veb[veb_idx];
13261         veb->flags = flags;
13262         veb->uplink_seid = uplink_seid;
13263         veb->veb_idx = (uplink_veb ? uplink_veb->idx : I40E_NO_VEB);
13264         veb->enabled_tc = (enabled_tc ? enabled_tc : 0x1);
13265
13266         /* create the VEB in the switch */
13267         ret = i40e_add_veb(veb, pf->vsi[vsi_idx]);
13268         if (ret)
13269                 goto err_veb;
13270         if (vsi_idx == pf->lan_vsi)
13271                 pf->lan_veb = veb->idx;
13272
13273         return veb;
13274
13275 err_veb:
13276         i40e_veb_clear(veb);
13277 err_alloc:
13278         return NULL;
13279 }
13280
13281 /**
13282  * i40e_setup_pf_switch_element - set PF vars based on switch type
13283  * @pf: board private structure
13284  * @ele: element we are building info from
13285  * @num_reported: total number of elements
13286  * @printconfig: should we print the contents
13287  *
13288  * helper function to assist in extracting a few useful SEID values.
13289  **/
13290 static void i40e_setup_pf_switch_element(struct i40e_pf *pf,
13291                                 struct i40e_aqc_switch_config_element_resp *ele,
13292                                 u16 num_reported, bool printconfig)
13293 {
13294         u16 downlink_seid = le16_to_cpu(ele->downlink_seid);
13295         u16 uplink_seid = le16_to_cpu(ele->uplink_seid);
13296         u8 element_type = ele->element_type;
13297         u16 seid = le16_to_cpu(ele->seid);
13298
13299         if (printconfig)
13300                 dev_info(&pf->pdev->dev,
13301                          "type=%d seid=%d uplink=%d downlink=%d\n",
13302                          element_type, seid, uplink_seid, downlink_seid);
13303
13304         switch (element_type) {
13305         case I40E_SWITCH_ELEMENT_TYPE_MAC:
13306                 pf->mac_seid = seid;
13307                 break;
13308         case I40E_SWITCH_ELEMENT_TYPE_VEB:
13309                 /* Main VEB? */
13310                 if (uplink_seid != pf->mac_seid)
13311                         break;
13312                 if (pf->lan_veb == I40E_NO_VEB) {
13313                         int v;
13314
13315                         /* find existing or else empty VEB */
13316                         for (v = 0; v < I40E_MAX_VEB; v++) {
13317                                 if (pf->veb[v] && (pf->veb[v]->seid == seid)) {
13318                                         pf->lan_veb = v;
13319                                         break;
13320                                 }
13321                         }
13322                         if (pf->lan_veb == I40E_NO_VEB) {
13323                                 v = i40e_veb_mem_alloc(pf);
13324                                 if (v < 0)
13325                                         break;
13326                                 pf->lan_veb = v;
13327                         }
13328                 }
13329
13330                 pf->veb[pf->lan_veb]->seid = seid;
13331                 pf->veb[pf->lan_veb]->uplink_seid = pf->mac_seid;
13332                 pf->veb[pf->lan_veb]->pf = pf;
13333                 pf->veb[pf->lan_veb]->veb_idx = I40E_NO_VEB;
13334                 break;
13335         case I40E_SWITCH_ELEMENT_TYPE_VSI:
13336                 if (num_reported != 1)
13337                         break;
13338                 /* This is immediately after a reset so we can assume this is
13339                  * the PF's VSI
13340                  */
13341                 pf->mac_seid = uplink_seid;
13342                 pf->pf_seid = downlink_seid;
13343                 pf->main_vsi_seid = seid;
13344                 if (printconfig)
13345                         dev_info(&pf->pdev->dev,
13346                                  "pf_seid=%d main_vsi_seid=%d\n",
13347                                  pf->pf_seid, pf->main_vsi_seid);
13348                 break;
13349         case I40E_SWITCH_ELEMENT_TYPE_PF:
13350         case I40E_SWITCH_ELEMENT_TYPE_VF:
13351         case I40E_SWITCH_ELEMENT_TYPE_EMP:
13352         case I40E_SWITCH_ELEMENT_TYPE_BMC:
13353         case I40E_SWITCH_ELEMENT_TYPE_PE:
13354         case I40E_SWITCH_ELEMENT_TYPE_PA:
13355                 /* ignore these for now */
13356                 break;
13357         default:
13358                 dev_info(&pf->pdev->dev, "unknown element type=%d seid=%d\n",
13359                          element_type, seid);
13360                 break;
13361         }
13362 }
13363
13364 /**
13365  * i40e_fetch_switch_configuration - Get switch config from firmware
13366  * @pf: board private structure
13367  * @printconfig: should we print the contents
13368  *
13369  * Get the current switch configuration from the device and
13370  * extract a few useful SEID values.
13371  **/
13372 int i40e_fetch_switch_configuration(struct i40e_pf *pf, bool printconfig)
13373 {
13374         struct i40e_aqc_get_switch_config_resp *sw_config;
13375         u16 next_seid = 0;
13376         int ret = 0;
13377         u8 *aq_buf;
13378         int i;
13379
13380         aq_buf = kzalloc(I40E_AQ_LARGE_BUF, GFP_KERNEL);
13381         if (!aq_buf)
13382                 return -ENOMEM;
13383
13384         sw_config = (struct i40e_aqc_get_switch_config_resp *)aq_buf;
13385         do {
13386                 u16 num_reported, num_total;
13387
13388                 ret = i40e_aq_get_switch_config(&pf->hw, sw_config,
13389                                                 I40E_AQ_LARGE_BUF,
13390                                                 &next_seid, NULL);
13391                 if (ret) {
13392                         dev_info(&pf->pdev->dev,
13393                                  "get switch config failed err %s aq_err %s\n",
13394                                  i40e_stat_str(&pf->hw, ret),
13395                                  i40e_aq_str(&pf->hw,
13396                                              pf->hw.aq.asq_last_status));
13397                         kfree(aq_buf);
13398                         return -ENOENT;
13399                 }
13400
13401                 num_reported = le16_to_cpu(sw_config->header.num_reported);
13402                 num_total = le16_to_cpu(sw_config->header.num_total);
13403
13404                 if (printconfig)
13405                         dev_info(&pf->pdev->dev,
13406                                  "header: %d reported %d total\n",
13407                                  num_reported, num_total);
13408
13409                 for (i = 0; i < num_reported; i++) {
13410                         struct i40e_aqc_switch_config_element_resp *ele =
13411                                 &sw_config->element[i];
13412
13413                         i40e_setup_pf_switch_element(pf, ele, num_reported,
13414                                                      printconfig);
13415                 }
13416         } while (next_seid != 0);
13417
13418         kfree(aq_buf);
13419         return ret;
13420 }
13421
13422 /**
13423  * i40e_setup_pf_switch - Setup the HW switch on startup or after reset
13424  * @pf: board private structure
13425  * @reinit: if the Main VSI needs to re-initialized.
13426  *
13427  * Returns 0 on success, negative value on failure
13428  **/
13429 static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit)
13430 {
13431         u16 flags = 0;
13432         int ret;
13433
13434         /* find out what's out there already */
13435         ret = i40e_fetch_switch_configuration(pf, false);
13436         if (ret) {
13437                 dev_info(&pf->pdev->dev,
13438                          "couldn't fetch switch config, err %s aq_err %s\n",
13439                          i40e_stat_str(&pf->hw, ret),
13440                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
13441                 return ret;
13442         }
13443         i40e_pf_reset_stats(pf);
13444
13445         /* set the switch config bit for the whole device to
13446          * support limited promisc or true promisc
13447          * when user requests promisc. The default is limited
13448          * promisc.
13449         */
13450
13451         if ((pf->hw.pf_id == 0) &&
13452             !(pf->flags & I40E_FLAG_TRUE_PROMISC_SUPPORT)) {
13453                 flags = I40E_AQ_SET_SWITCH_CFG_PROMISC;
13454                 pf->last_sw_conf_flags = flags;
13455         }
13456
13457         if (pf->hw.pf_id == 0) {
13458                 u16 valid_flags;
13459
13460                 valid_flags = I40E_AQ_SET_SWITCH_CFG_PROMISC;
13461                 ret = i40e_aq_set_switch_config(&pf->hw, flags, valid_flags, 0,
13462                                                 NULL);
13463                 if (ret && pf->hw.aq.asq_last_status != I40E_AQ_RC_ESRCH) {
13464                         dev_info(&pf->pdev->dev,
13465                                  "couldn't set switch config bits, err %s aq_err %s\n",
13466                                  i40e_stat_str(&pf->hw, ret),
13467                                  i40e_aq_str(&pf->hw,
13468                                              pf->hw.aq.asq_last_status));
13469                         /* not a fatal problem, just keep going */
13470                 }
13471                 pf->last_sw_conf_valid_flags = valid_flags;
13472         }
13473
13474         /* first time setup */
13475         if (pf->lan_vsi == I40E_NO_VSI || reinit) {
13476                 struct i40e_vsi *vsi = NULL;
13477                 u16 uplink_seid;
13478
13479                 /* Set up the PF VSI associated with the PF's main VSI
13480                  * that is already in the HW switch
13481                  */
13482                 if (pf->lan_veb != I40E_NO_VEB && pf->veb[pf->lan_veb])
13483                         uplink_seid = pf->veb[pf->lan_veb]->seid;
13484                 else
13485                         uplink_seid = pf->mac_seid;
13486                 if (pf->lan_vsi == I40E_NO_VSI)
13487                         vsi = i40e_vsi_setup(pf, I40E_VSI_MAIN, uplink_seid, 0);
13488                 else if (reinit)
13489                         vsi = i40e_vsi_reinit_setup(pf->vsi[pf->lan_vsi]);
13490                 if (!vsi) {
13491                         dev_info(&pf->pdev->dev, "setup of MAIN VSI failed\n");
13492                         i40e_cloud_filter_exit(pf);
13493                         i40e_fdir_teardown(pf);
13494                         return -EAGAIN;
13495                 }
13496         } else {
13497                 /* force a reset of TC and queue layout configurations */
13498                 u8 enabled_tc = pf->vsi[pf->lan_vsi]->tc_config.enabled_tc;
13499
13500                 pf->vsi[pf->lan_vsi]->tc_config.enabled_tc = 0;
13501                 pf->vsi[pf->lan_vsi]->seid = pf->main_vsi_seid;
13502                 i40e_vsi_config_tc(pf->vsi[pf->lan_vsi], enabled_tc);
13503         }
13504         i40e_vlan_stripping_disable(pf->vsi[pf->lan_vsi]);
13505
13506         i40e_fdir_sb_setup(pf);
13507
13508         /* Setup static PF queue filter control settings */
13509         ret = i40e_setup_pf_filter_control(pf);
13510         if (ret) {
13511                 dev_info(&pf->pdev->dev, "setup_pf_filter_control failed: %d\n",
13512                          ret);
13513                 /* Failure here should not stop continuing other steps */
13514         }
13515
13516         /* enable RSS in the HW, even for only one queue, as the stack can use
13517          * the hash
13518          */
13519         if ((pf->flags & I40E_FLAG_RSS_ENABLED))
13520                 i40e_pf_config_rss(pf);
13521
13522         /* fill in link information and enable LSE reporting */
13523         i40e_link_event(pf);
13524
13525         /* Initialize user-specific link properties */
13526         pf->fc_autoneg_status = ((pf->hw.phy.link_info.an_info &
13527                                   I40E_AQ_AN_COMPLETED) ? true : false);
13528
13529         i40e_ptp_init(pf);
13530
13531         /* repopulate tunnel port filters */
13532         i40e_sync_udp_filters(pf);
13533
13534         return ret;
13535 }
13536
13537 /**
13538  * i40e_determine_queue_usage - Work out queue distribution
13539  * @pf: board private structure
13540  **/
13541 static void i40e_determine_queue_usage(struct i40e_pf *pf)
13542 {
13543         int queues_left;
13544         int q_max;
13545
13546         pf->num_lan_qps = 0;
13547
13548         /* Find the max queues to be put into basic use.  We'll always be
13549          * using TC0, whether or not DCB is running, and TC0 will get the
13550          * big RSS set.
13551          */
13552         queues_left = pf->hw.func_caps.num_tx_qp;
13553
13554         if ((queues_left == 1) ||
13555             !(pf->flags & I40E_FLAG_MSIX_ENABLED)) {
13556                 /* one qp for PF, no queues for anything else */
13557                 queues_left = 0;
13558                 pf->alloc_rss_size = pf->num_lan_qps = 1;
13559
13560                 /* make sure all the fancies are disabled */
13561                 pf->flags &= ~(I40E_FLAG_RSS_ENABLED    |
13562                                I40E_FLAG_IWARP_ENABLED  |
13563                                I40E_FLAG_FD_SB_ENABLED  |
13564                                I40E_FLAG_FD_ATR_ENABLED |
13565                                I40E_FLAG_DCB_CAPABLE    |
13566                                I40E_FLAG_DCB_ENABLED    |
13567                                I40E_FLAG_SRIOV_ENABLED  |
13568                                I40E_FLAG_VMDQ_ENABLED);
13569                 pf->flags |= I40E_FLAG_FD_SB_INACTIVE;
13570         } else if (!(pf->flags & (I40E_FLAG_RSS_ENABLED |
13571                                   I40E_FLAG_FD_SB_ENABLED |
13572                                   I40E_FLAG_FD_ATR_ENABLED |
13573                                   I40E_FLAG_DCB_CAPABLE))) {
13574                 /* one qp for PF */
13575                 pf->alloc_rss_size = pf->num_lan_qps = 1;
13576                 queues_left -= pf->num_lan_qps;
13577
13578                 pf->flags &= ~(I40E_FLAG_RSS_ENABLED    |
13579                                I40E_FLAG_IWARP_ENABLED  |
13580                                I40E_FLAG_FD_SB_ENABLED  |
13581                                I40E_FLAG_FD_ATR_ENABLED |
13582                                I40E_FLAG_DCB_ENABLED    |
13583                                I40E_FLAG_VMDQ_ENABLED);
13584                 pf->flags |= I40E_FLAG_FD_SB_INACTIVE;
13585         } else {
13586                 /* Not enough queues for all TCs */
13587                 if ((pf->flags & I40E_FLAG_DCB_CAPABLE) &&
13588                     (queues_left < I40E_MAX_TRAFFIC_CLASS)) {
13589                         pf->flags &= ~(I40E_FLAG_DCB_CAPABLE |
13590                                         I40E_FLAG_DCB_ENABLED);
13591                         dev_info(&pf->pdev->dev, "not enough queues for DCB. DCB is disabled.\n");
13592                 }
13593
13594                 /* limit lan qps to the smaller of qps, cpus or msix */
13595                 q_max = max_t(int, pf->rss_size_max, num_online_cpus());
13596                 q_max = min_t(int, q_max, pf->hw.func_caps.num_tx_qp);
13597                 q_max = min_t(int, q_max, pf->hw.func_caps.num_msix_vectors);
13598                 pf->num_lan_qps = q_max;
13599
13600                 queues_left -= pf->num_lan_qps;
13601         }
13602
13603         if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
13604                 if (queues_left > 1) {
13605                         queues_left -= 1; /* save 1 queue for FD */
13606                 } else {
13607                         pf->flags &= ~I40E_FLAG_FD_SB_ENABLED;
13608                         pf->flags |= I40E_FLAG_FD_SB_INACTIVE;
13609                         dev_info(&pf->pdev->dev, "not enough queues for Flow Director. Flow Director feature is disabled\n");
13610                 }
13611         }
13612
13613         if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) &&
13614             pf->num_vf_qps && pf->num_req_vfs && queues_left) {
13615                 pf->num_req_vfs = min_t(int, pf->num_req_vfs,
13616                                         (queues_left / pf->num_vf_qps));
13617                 queues_left -= (pf->num_req_vfs * pf->num_vf_qps);
13618         }
13619
13620         if ((pf->flags & I40E_FLAG_VMDQ_ENABLED) &&
13621             pf->num_vmdq_vsis && pf->num_vmdq_qps && queues_left) {
13622                 pf->num_vmdq_vsis = min_t(int, pf->num_vmdq_vsis,
13623                                           (queues_left / pf->num_vmdq_qps));
13624                 queues_left -= (pf->num_vmdq_vsis * pf->num_vmdq_qps);
13625         }
13626
13627         pf->queues_left = queues_left;
13628         dev_dbg(&pf->pdev->dev,
13629                 "qs_avail=%d FD SB=%d lan_qs=%d lan_tc0=%d vf=%d*%d vmdq=%d*%d, remaining=%d\n",
13630                 pf->hw.func_caps.num_tx_qp,
13631                 !!(pf->flags & I40E_FLAG_FD_SB_ENABLED),
13632                 pf->num_lan_qps, pf->alloc_rss_size, pf->num_req_vfs,
13633                 pf->num_vf_qps, pf->num_vmdq_vsis, pf->num_vmdq_qps,
13634                 queues_left);
13635 }
13636
13637 /**
13638  * i40e_setup_pf_filter_control - Setup PF static filter control
13639  * @pf: PF to be setup
13640  *
13641  * i40e_setup_pf_filter_control sets up a PF's initial filter control
13642  * settings. If PE/FCoE are enabled then it will also set the per PF
13643  * based filter sizes required for them. It also enables Flow director,
13644  * ethertype and macvlan type filter settings for the pf.
13645  *
13646  * Returns 0 on success, negative on failure
13647  **/
13648 static int i40e_setup_pf_filter_control(struct i40e_pf *pf)
13649 {
13650         struct i40e_filter_control_settings *settings = &pf->filter_settings;
13651
13652         settings->hash_lut_size = I40E_HASH_LUT_SIZE_128;
13653
13654         /* Flow Director is enabled */
13655         if (pf->flags & (I40E_FLAG_FD_SB_ENABLED | I40E_FLAG_FD_ATR_ENABLED))
13656                 settings->enable_fdir = true;
13657
13658         /* Ethtype and MACVLAN filters enabled for PF */
13659         settings->enable_ethtype = true;
13660         settings->enable_macvlan = true;
13661
13662         if (i40e_set_filter_control(&pf->hw, settings))
13663                 return -ENOENT;
13664
13665         return 0;
13666 }
13667
13668 #define INFO_STRING_LEN 255
13669 #define REMAIN(__x) (INFO_STRING_LEN - (__x))
13670 static void i40e_print_features(struct i40e_pf *pf)
13671 {
13672         struct i40e_hw *hw = &pf->hw;
13673         char *buf;
13674         int i;
13675
13676         buf = kmalloc(INFO_STRING_LEN, GFP_KERNEL);
13677         if (!buf)
13678                 return;
13679
13680         i = snprintf(buf, INFO_STRING_LEN, "Features: PF-id[%d]", hw->pf_id);
13681 #ifdef CONFIG_PCI_IOV
13682         i += snprintf(&buf[i], REMAIN(i), " VFs: %d", pf->num_req_vfs);
13683 #endif
13684         i += snprintf(&buf[i], REMAIN(i), " VSIs: %d QP: %d",
13685                       pf->hw.func_caps.num_vsis,
13686                       pf->vsi[pf->lan_vsi]->num_queue_pairs);
13687         if (pf->flags & I40E_FLAG_RSS_ENABLED)
13688                 i += snprintf(&buf[i], REMAIN(i), " RSS");
13689         if (pf->flags & I40E_FLAG_FD_ATR_ENABLED)
13690                 i += snprintf(&buf[i], REMAIN(i), " FD_ATR");
13691         if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
13692                 i += snprintf(&buf[i], REMAIN(i), " FD_SB");
13693                 i += snprintf(&buf[i], REMAIN(i), " NTUPLE");
13694         }
13695         if (pf->flags & I40E_FLAG_DCB_CAPABLE)
13696                 i += snprintf(&buf[i], REMAIN(i), " DCB");
13697         i += snprintf(&buf[i], REMAIN(i), " VxLAN");
13698         i += snprintf(&buf[i], REMAIN(i), " Geneve");
13699         if (pf->flags & I40E_FLAG_PTP)
13700                 i += snprintf(&buf[i], REMAIN(i), " PTP");
13701         if (pf->flags & I40E_FLAG_VEB_MODE_ENABLED)
13702                 i += snprintf(&buf[i], REMAIN(i), " VEB");
13703         else
13704                 i += snprintf(&buf[i], REMAIN(i), " VEPA");
13705
13706         dev_info(&pf->pdev->dev, "%s\n", buf);
13707         kfree(buf);
13708         WARN_ON(i > INFO_STRING_LEN);
13709 }
13710
13711 /**
13712  * i40e_get_platform_mac_addr - get platform-specific MAC address
13713  * @pdev: PCI device information struct
13714  * @pf: board private structure
13715  *
13716  * Look up the MAC address for the device. First we'll try
13717  * eth_platform_get_mac_address, which will check Open Firmware, or arch
13718  * specific fallback. Otherwise, we'll default to the stored value in
13719  * firmware.
13720  **/
13721 static void i40e_get_platform_mac_addr(struct pci_dev *pdev, struct i40e_pf *pf)
13722 {
13723         if (eth_platform_get_mac_address(&pdev->dev, pf->hw.mac.addr))
13724                 i40e_get_mac_addr(&pf->hw, pf->hw.mac.addr);
13725 }
13726
13727 /**
13728  * i40e_probe - Device initialization routine
13729  * @pdev: PCI device information struct
13730  * @ent: entry in i40e_pci_tbl
13731  *
13732  * i40e_probe initializes a PF identified by a pci_dev structure.
13733  * The OS initialization, configuring of the PF private structure,
13734  * and a hardware reset occur.
13735  *
13736  * Returns 0 on success, negative on failure
13737  **/
13738 static int i40e_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
13739 {
13740         struct i40e_aq_get_phy_abilities_resp abilities;
13741         struct i40e_pf *pf;
13742         struct i40e_hw *hw;
13743         static u16 pfs_found;
13744         u16 wol_nvm_bits;
13745         u16 link_status;
13746         int err;
13747         u32 val;
13748         u32 i;
13749
13750         err = pci_enable_device_mem(pdev);
13751         if (err)
13752                 return err;
13753
13754         /* set up for high or low dma */
13755         err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
13756         if (err) {
13757                 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
13758                 if (err) {
13759                         dev_err(&pdev->dev,
13760                                 "DMA configuration failed: 0x%x\n", err);
13761                         goto err_dma;
13762                 }
13763         }
13764
13765         /* set up pci connections */
13766         err = pci_request_mem_regions(pdev, i40e_driver_name);
13767         if (err) {
13768                 dev_info(&pdev->dev,
13769                          "pci_request_selected_regions failed %d\n", err);
13770                 goto err_pci_reg;
13771         }
13772
13773         pci_enable_pcie_error_reporting(pdev);
13774         pci_set_master(pdev);
13775
13776         /* Now that we have a PCI connection, we need to do the
13777          * low level device setup.  This is primarily setting up
13778          * the Admin Queue structures and then querying for the
13779          * device's current profile information.
13780          */
13781         pf = kzalloc(sizeof(*pf), GFP_KERNEL);
13782         if (!pf) {
13783                 err = -ENOMEM;
13784                 goto err_pf_alloc;
13785         }
13786         pf->next_vsi = 0;
13787         pf->pdev = pdev;
13788         set_bit(__I40E_DOWN, pf->state);
13789
13790         hw = &pf->hw;
13791         hw->back = pf;
13792
13793         pf->ioremap_len = min_t(int, pci_resource_len(pdev, 0),
13794                                 I40E_MAX_CSR_SPACE);
13795
13796         hw->hw_addr = ioremap(pci_resource_start(pdev, 0), pf->ioremap_len);
13797         if (!hw->hw_addr) {
13798                 err = -EIO;
13799                 dev_info(&pdev->dev, "ioremap(0x%04x, 0x%04x) failed: 0x%x\n",
13800                          (unsigned int)pci_resource_start(pdev, 0),
13801                          pf->ioremap_len, err);
13802                 goto err_ioremap;
13803         }
13804         hw->vendor_id = pdev->vendor;
13805         hw->device_id = pdev->device;
13806         pci_read_config_byte(pdev, PCI_REVISION_ID, &hw->revision_id);
13807         hw->subsystem_vendor_id = pdev->subsystem_vendor;
13808         hw->subsystem_device_id = pdev->subsystem_device;
13809         hw->bus.device = PCI_SLOT(pdev->devfn);
13810         hw->bus.func = PCI_FUNC(pdev->devfn);
13811         hw->bus.bus_id = pdev->bus->number;
13812         pf->instance = pfs_found;
13813
13814         /* Select something other than the 802.1ad ethertype for the
13815          * switch to use internally and drop on ingress.
13816          */
13817         hw->switch_tag = 0xffff;
13818         hw->first_tag = ETH_P_8021AD;
13819         hw->second_tag = ETH_P_8021Q;
13820
13821         INIT_LIST_HEAD(&pf->l3_flex_pit_list);
13822         INIT_LIST_HEAD(&pf->l4_flex_pit_list);
13823
13824         /* set up the locks for the AQ, do this only once in probe
13825          * and destroy them only once in remove
13826          */
13827         mutex_init(&hw->aq.asq_mutex);
13828         mutex_init(&hw->aq.arq_mutex);
13829
13830         pf->msg_enable = netif_msg_init(debug,
13831                                         NETIF_MSG_DRV |
13832                                         NETIF_MSG_PROBE |
13833                                         NETIF_MSG_LINK);
13834         if (debug < -1)
13835                 pf->hw.debug_mask = debug;
13836
13837         /* do a special CORER for clearing PXE mode once at init */
13838         if (hw->revision_id == 0 &&
13839             (rd32(hw, I40E_GLLAN_RCTL_0) & I40E_GLLAN_RCTL_0_PXE_MODE_MASK)) {
13840                 wr32(hw, I40E_GLGEN_RTRIG, I40E_GLGEN_RTRIG_CORER_MASK);
13841                 i40e_flush(hw);
13842                 msleep(200);
13843                 pf->corer_count++;
13844
13845                 i40e_clear_pxe_mode(hw);
13846         }
13847
13848         /* Reset here to make sure all is clean and to define PF 'n' */
13849         i40e_clear_hw(hw);
13850         err = i40e_pf_reset(hw);
13851         if (err) {
13852                 dev_info(&pdev->dev, "Initial pf_reset failed: %d\n", err);
13853                 goto err_pf_reset;
13854         }
13855         pf->pfr_count++;
13856
13857         hw->aq.num_arq_entries = I40E_AQ_LEN;
13858         hw->aq.num_asq_entries = I40E_AQ_LEN;
13859         hw->aq.arq_buf_size = I40E_MAX_AQ_BUF_SIZE;
13860         hw->aq.asq_buf_size = I40E_MAX_AQ_BUF_SIZE;
13861         pf->adminq_work_limit = I40E_AQ_WORK_LIMIT;
13862
13863         snprintf(pf->int_name, sizeof(pf->int_name) - 1,
13864                  "%s-%s:misc",
13865                  dev_driver_string(&pf->pdev->dev), dev_name(&pdev->dev));
13866
13867         err = i40e_init_shared_code(hw);
13868         if (err) {
13869                 dev_warn(&pdev->dev, "unidentified MAC or BLANK NVM: %d\n",
13870                          err);
13871                 goto err_pf_reset;
13872         }
13873
13874         /* set up a default setting for link flow control */
13875         pf->hw.fc.requested_mode = I40E_FC_NONE;
13876
13877         err = i40e_init_adminq(hw);
13878         if (err) {
13879                 if (err == I40E_ERR_FIRMWARE_API_VERSION)
13880                         dev_info(&pdev->dev,
13881                                  "The driver for the device stopped because the NVM image is newer than expected. You must install the most recent version of the network driver.\n");
13882                 else
13883                         dev_info(&pdev->dev,
13884                                  "The driver for the device stopped because the device firmware failed to init. Try updating your NVM image.\n");
13885
13886                 goto err_pf_reset;
13887         }
13888         i40e_get_oem_version(hw);
13889
13890         /* provide nvm, fw, api versions */
13891         dev_info(&pdev->dev, "fw %d.%d.%05d api %d.%d nvm %s\n",
13892                  hw->aq.fw_maj_ver, hw->aq.fw_min_ver, hw->aq.fw_build,
13893                  hw->aq.api_maj_ver, hw->aq.api_min_ver,
13894                  i40e_nvm_version_str(hw));
13895
13896         if (hw->aq.api_maj_ver == I40E_FW_API_VERSION_MAJOR &&
13897             hw->aq.api_min_ver > I40E_FW_MINOR_VERSION(hw))
13898                 dev_info(&pdev->dev,
13899                          "The driver for the device detected a newer version of the NVM image than expected. Please install the most recent version of the network driver.\n");
13900         else if (hw->aq.api_maj_ver == 1 && hw->aq.api_min_ver < 4)
13901                 dev_info(&pdev->dev,
13902                          "The driver for the device detected an older version of the NVM image than expected. Please update the NVM image.\n");
13903
13904         i40e_verify_eeprom(pf);
13905
13906         /* Rev 0 hardware was never productized */
13907         if (hw->revision_id < 1)
13908                 dev_warn(&pdev->dev, "This device is a pre-production adapter/LOM. Please be aware there may be issues with your hardware. If you are experiencing problems please contact your Intel or hardware representative who provided you with this hardware.\n");
13909
13910         i40e_clear_pxe_mode(hw);
13911         err = i40e_get_capabilities(pf, i40e_aqc_opc_list_func_capabilities);
13912         if (err)
13913                 goto err_adminq_setup;
13914
13915         err = i40e_sw_init(pf);
13916         if (err) {
13917                 dev_info(&pdev->dev, "sw_init failed: %d\n", err);
13918                 goto err_sw_init;
13919         }
13920
13921         err = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp,
13922                                 hw->func_caps.num_rx_qp, 0, 0);
13923         if (err) {
13924                 dev_info(&pdev->dev, "init_lan_hmc failed: %d\n", err);
13925                 goto err_init_lan_hmc;
13926         }
13927
13928         err = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY);
13929         if (err) {
13930                 dev_info(&pdev->dev, "configure_lan_hmc failed: %d\n", err);
13931                 err = -ENOENT;
13932                 goto err_configure_lan_hmc;
13933         }
13934
13935         /* Disable LLDP for NICs that have firmware versions lower than v4.3.
13936          * Ignore error return codes because if it was already disabled via
13937          * hardware settings this will fail
13938          */
13939         if (pf->hw_features & I40E_HW_STOP_FW_LLDP) {
13940                 dev_info(&pdev->dev, "Stopping firmware LLDP agent.\n");
13941                 i40e_aq_stop_lldp(hw, true, NULL);
13942         }
13943
13944         /* allow a platform config to override the HW addr */
13945         i40e_get_platform_mac_addr(pdev, pf);
13946
13947         if (!is_valid_ether_addr(hw->mac.addr)) {
13948                 dev_info(&pdev->dev, "invalid MAC address %pM\n", hw->mac.addr);
13949                 err = -EIO;
13950                 goto err_mac_addr;
13951         }
13952         dev_info(&pdev->dev, "MAC address: %pM\n", hw->mac.addr);
13953         ether_addr_copy(hw->mac.perm_addr, hw->mac.addr);
13954         i40e_get_port_mac_addr(hw, hw->mac.port_addr);
13955         if (is_valid_ether_addr(hw->mac.port_addr))
13956                 pf->hw_features |= I40E_HW_PORT_ID_VALID;
13957
13958         pci_set_drvdata(pdev, pf);
13959         pci_save_state(pdev);
13960
13961         /* Enable FW to write default DCB config on link-up */
13962         i40e_aq_set_dcb_parameters(hw, true, NULL);
13963
13964 #ifdef CONFIG_I40E_DCB
13965         err = i40e_init_pf_dcb(pf);
13966         if (err) {
13967                 dev_info(&pdev->dev, "DCB init failed %d, disabled\n", err);
13968                 pf->flags &= ~(I40E_FLAG_DCB_CAPABLE | I40E_FLAG_DCB_ENABLED);
13969                 /* Continue without DCB enabled */
13970         }
13971 #endif /* CONFIG_I40E_DCB */
13972
13973         /* set up periodic task facility */
13974         timer_setup(&pf->service_timer, i40e_service_timer, 0);
13975         pf->service_timer_period = HZ;
13976
13977         INIT_WORK(&pf->service_task, i40e_service_task);
13978         clear_bit(__I40E_SERVICE_SCHED, pf->state);
13979
13980         /* NVM bit on means WoL disabled for the port */
13981         i40e_read_nvm_word(hw, I40E_SR_NVM_WAKE_ON_LAN, &wol_nvm_bits);
13982         if (BIT (hw->port) & wol_nvm_bits || hw->partition_id != 1)
13983                 pf->wol_en = false;
13984         else
13985                 pf->wol_en = true;
13986         device_set_wakeup_enable(&pf->pdev->dev, pf->wol_en);
13987
13988         /* set up the main switch operations */
13989         i40e_determine_queue_usage(pf);
13990         err = i40e_init_interrupt_scheme(pf);
13991         if (err)
13992                 goto err_switch_setup;
13993
13994         /* The number of VSIs reported by the FW is the minimum guaranteed
13995          * to us; HW supports far more and we share the remaining pool with
13996          * the other PFs. We allocate space for more than the guarantee with
13997          * the understanding that we might not get them all later.
13998          */
13999         if (pf->hw.func_caps.num_vsis < I40E_MIN_VSI_ALLOC)
14000                 pf->num_alloc_vsi = I40E_MIN_VSI_ALLOC;
14001         else
14002                 pf->num_alloc_vsi = pf->hw.func_caps.num_vsis;
14003
14004         /* Set up the *vsi struct and our local tracking of the MAIN PF vsi. */
14005         pf->vsi = kcalloc(pf->num_alloc_vsi, sizeof(struct i40e_vsi *),
14006                           GFP_KERNEL);
14007         if (!pf->vsi) {
14008                 err = -ENOMEM;
14009                 goto err_switch_setup;
14010         }
14011
14012 #ifdef CONFIG_PCI_IOV
14013         /* prep for VF support */
14014         if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) &&
14015             (pf->flags & I40E_FLAG_MSIX_ENABLED) &&
14016             !test_bit(__I40E_BAD_EEPROM, pf->state)) {
14017                 if (pci_num_vf(pdev))
14018                         pf->flags |= I40E_FLAG_VEB_MODE_ENABLED;
14019         }
14020 #endif
14021         err = i40e_setup_pf_switch(pf, false);
14022         if (err) {
14023                 dev_info(&pdev->dev, "setup_pf_switch failed: %d\n", err);
14024                 goto err_vsis;
14025         }
14026         INIT_LIST_HEAD(&pf->vsi[pf->lan_vsi]->ch_list);
14027
14028         /* if FDIR VSI was set up, start it now */
14029         for (i = 0; i < pf->num_alloc_vsi; i++) {
14030                 if (pf->vsi[i] && pf->vsi[i]->type == I40E_VSI_FDIR) {
14031                         i40e_vsi_open(pf->vsi[i]);
14032                         break;
14033                 }
14034         }
14035
14036         /* The driver only wants link up/down and module qualification
14037          * reports from firmware.  Note the negative logic.
14038          */
14039         err = i40e_aq_set_phy_int_mask(&pf->hw,
14040                                        ~(I40E_AQ_EVENT_LINK_UPDOWN |
14041                                          I40E_AQ_EVENT_MEDIA_NA |
14042                                          I40E_AQ_EVENT_MODULE_QUAL_FAIL), NULL);
14043         if (err)
14044                 dev_info(&pf->pdev->dev, "set phy mask fail, err %s aq_err %s\n",
14045                          i40e_stat_str(&pf->hw, err),
14046                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
14047
14048         /* Reconfigure hardware for allowing smaller MSS in the case
14049          * of TSO, so that we avoid the MDD being fired and causing
14050          * a reset in the case of small MSS+TSO.
14051          */
14052         val = rd32(hw, I40E_REG_MSS);
14053         if ((val & I40E_REG_MSS_MIN_MASK) > I40E_64BYTE_MSS) {
14054                 val &= ~I40E_REG_MSS_MIN_MASK;
14055                 val |= I40E_64BYTE_MSS;
14056                 wr32(hw, I40E_REG_MSS, val);
14057         }
14058
14059         if (pf->hw_features & I40E_HW_RESTART_AUTONEG) {
14060                 msleep(75);
14061                 err = i40e_aq_set_link_restart_an(&pf->hw, true, NULL);
14062                 if (err)
14063                         dev_info(&pf->pdev->dev, "link restart failed, err %s aq_err %s\n",
14064                                  i40e_stat_str(&pf->hw, err),
14065                                  i40e_aq_str(&pf->hw,
14066                                              pf->hw.aq.asq_last_status));
14067         }
14068         /* The main driver is (mostly) up and happy. We need to set this state
14069          * before setting up the misc vector or we get a race and the vector
14070          * ends up disabled forever.
14071          */
14072         clear_bit(__I40E_DOWN, pf->state);
14073
14074         /* In case of MSIX we are going to setup the misc vector right here
14075          * to handle admin queue events etc. In case of legacy and MSI
14076          * the misc functionality and queue processing is combined in
14077          * the same vector and that gets setup at open.
14078          */
14079         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
14080                 err = i40e_setup_misc_vector(pf);
14081                 if (err) {
14082                         dev_info(&pdev->dev,
14083                                  "setup of misc vector failed: %d\n", err);
14084                         i40e_cloud_filter_exit(pf);
14085                         i40e_fdir_teardown(pf);
14086                         goto err_vsis;
14087                 }
14088         }
14089
14090 #ifdef CONFIG_PCI_IOV
14091         /* prep for VF support */
14092         if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) &&
14093             (pf->flags & I40E_FLAG_MSIX_ENABLED) &&
14094             !test_bit(__I40E_BAD_EEPROM, pf->state)) {
14095                 /* disable link interrupts for VFs */
14096                 val = rd32(hw, I40E_PFGEN_PORTMDIO_NUM);
14097                 val &= ~I40E_PFGEN_PORTMDIO_NUM_VFLINK_STAT_ENA_MASK;
14098                 wr32(hw, I40E_PFGEN_PORTMDIO_NUM, val);
14099                 i40e_flush(hw);
14100
14101                 if (pci_num_vf(pdev)) {
14102                         dev_info(&pdev->dev,
14103                                  "Active VFs found, allocating resources.\n");
14104                         err = i40e_alloc_vfs(pf, pci_num_vf(pdev));
14105                         if (err)
14106                                 dev_info(&pdev->dev,
14107                                          "Error %d allocating resources for existing VFs\n",
14108                                          err);
14109                 }
14110         }
14111 #endif /* CONFIG_PCI_IOV */
14112
14113         if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
14114                 pf->iwarp_base_vector = i40e_get_lump(pf, pf->irq_pile,
14115                                                       pf->num_iwarp_msix,
14116                                                       I40E_IWARP_IRQ_PILE_ID);
14117                 if (pf->iwarp_base_vector < 0) {
14118                         dev_info(&pdev->dev,
14119                                  "failed to get tracking for %d vectors for IWARP err=%d\n",
14120                                  pf->num_iwarp_msix, pf->iwarp_base_vector);
14121                         pf->flags &= ~I40E_FLAG_IWARP_ENABLED;
14122                 }
14123         }
14124
14125         i40e_dbg_pf_init(pf);
14126
14127         /* tell the firmware that we're starting */
14128         i40e_send_version(pf);
14129
14130         /* since everything's happy, start the service_task timer */
14131         mod_timer(&pf->service_timer,
14132                   round_jiffies(jiffies + pf->service_timer_period));
14133
14134         /* add this PF to client device list and launch a client service task */
14135         if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
14136                 err = i40e_lan_add_device(pf);
14137                 if (err)
14138                         dev_info(&pdev->dev, "Failed to add PF to client API service list: %d\n",
14139                                  err);
14140         }
14141
14142 #define PCI_SPEED_SIZE 8
14143 #define PCI_WIDTH_SIZE 8
14144         /* Devices on the IOSF bus do not have this information
14145          * and will report PCI Gen 1 x 1 by default so don't bother
14146          * checking them.
14147          */
14148         if (!(pf->hw_features & I40E_HW_NO_PCI_LINK_CHECK)) {
14149                 char speed[PCI_SPEED_SIZE] = "Unknown";
14150                 char width[PCI_WIDTH_SIZE] = "Unknown";
14151
14152                 /* Get the negotiated link width and speed from PCI config
14153                  * space
14154                  */
14155                 pcie_capability_read_word(pf->pdev, PCI_EXP_LNKSTA,
14156                                           &link_status);
14157
14158                 i40e_set_pci_config_data(hw, link_status);
14159
14160                 switch (hw->bus.speed) {
14161                 case i40e_bus_speed_8000:
14162                         strncpy(speed, "8.0", PCI_SPEED_SIZE); break;
14163                 case i40e_bus_speed_5000:
14164                         strncpy(speed, "5.0", PCI_SPEED_SIZE); break;
14165                 case i40e_bus_speed_2500:
14166                         strncpy(speed, "2.5", PCI_SPEED_SIZE); break;
14167                 default:
14168                         break;
14169                 }
14170                 switch (hw->bus.width) {
14171                 case i40e_bus_width_pcie_x8:
14172                         strncpy(width, "8", PCI_WIDTH_SIZE); break;
14173                 case i40e_bus_width_pcie_x4:
14174                         strncpy(width, "4", PCI_WIDTH_SIZE); break;
14175                 case i40e_bus_width_pcie_x2:
14176                         strncpy(width, "2", PCI_WIDTH_SIZE); break;
14177                 case i40e_bus_width_pcie_x1:
14178                         strncpy(width, "1", PCI_WIDTH_SIZE); break;
14179                 default:
14180                         break;
14181                 }
14182
14183                 dev_info(&pdev->dev, "PCI-Express: Speed %sGT/s Width x%s\n",
14184                          speed, width);
14185
14186                 if (hw->bus.width < i40e_bus_width_pcie_x8 ||
14187                     hw->bus.speed < i40e_bus_speed_8000) {
14188                         dev_warn(&pdev->dev, "PCI-Express bandwidth available for this device may be insufficient for optimal performance.\n");
14189                         dev_warn(&pdev->dev, "Please move the device to a different PCI-e link with more lanes and/or higher transfer rate.\n");
14190                 }
14191         }
14192
14193         /* get the requested speeds from the fw */
14194         err = i40e_aq_get_phy_capabilities(hw, false, false, &abilities, NULL);
14195         if (err)
14196                 dev_dbg(&pf->pdev->dev, "get requested speeds ret =  %s last_status =  %s\n",
14197                         i40e_stat_str(&pf->hw, err),
14198                         i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
14199         pf->hw.phy.link_info.requested_speeds = abilities.link_speed;
14200
14201         /* get the supported phy types from the fw */
14202         err = i40e_aq_get_phy_capabilities(hw, false, true, &abilities, NULL);
14203         if (err)
14204                 dev_dbg(&pf->pdev->dev, "get supported phy types ret =  %s last_status =  %s\n",
14205                         i40e_stat_str(&pf->hw, err),
14206                         i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
14207
14208         /* Add a filter to drop all Flow control frames from any VSI from being
14209          * transmitted. By doing so we stop a malicious VF from sending out
14210          * PAUSE or PFC frames and potentially controlling traffic for other
14211          * PF/VF VSIs.
14212          * The FW can still send Flow control frames if enabled.
14213          */
14214         i40e_add_filter_to_drop_tx_flow_control_frames(&pf->hw,
14215                                                        pf->main_vsi_seid);
14216
14217         if ((pf->hw.device_id == I40E_DEV_ID_10G_BASE_T) ||
14218                 (pf->hw.device_id == I40E_DEV_ID_10G_BASE_T4))
14219                 pf->hw_features |= I40E_HW_PHY_CONTROLS_LEDS;
14220         if (pf->hw.device_id == I40E_DEV_ID_SFP_I_X722)
14221                 pf->hw_features |= I40E_HW_HAVE_CRT_RETIMER;
14222         /* print a string summarizing features */
14223         i40e_print_features(pf);
14224
14225         return 0;
14226
14227         /* Unwind what we've done if something failed in the setup */
14228 err_vsis:
14229         set_bit(__I40E_DOWN, pf->state);
14230         i40e_clear_interrupt_scheme(pf);
14231         kfree(pf->vsi);
14232 err_switch_setup:
14233         i40e_reset_interrupt_capability(pf);
14234         del_timer_sync(&pf->service_timer);
14235 err_mac_addr:
14236 err_configure_lan_hmc:
14237         (void)i40e_shutdown_lan_hmc(hw);
14238 err_init_lan_hmc:
14239         kfree(pf->qp_pile);
14240 err_sw_init:
14241 err_adminq_setup:
14242 err_pf_reset:
14243         iounmap(hw->hw_addr);
14244 err_ioremap:
14245         kfree(pf);
14246 err_pf_alloc:
14247         pci_disable_pcie_error_reporting(pdev);
14248         pci_release_mem_regions(pdev);
14249 err_pci_reg:
14250 err_dma:
14251         pci_disable_device(pdev);
14252         return err;
14253 }
14254
14255 /**
14256  * i40e_remove - Device removal routine
14257  * @pdev: PCI device information struct
14258  *
14259  * i40e_remove is called by the PCI subsystem to alert the driver
14260  * that is should release a PCI device.  This could be caused by a
14261  * Hot-Plug event, or because the driver is going to be removed from
14262  * memory.
14263  **/
14264 static void i40e_remove(struct pci_dev *pdev)
14265 {
14266         struct i40e_pf *pf = pci_get_drvdata(pdev);
14267         struct i40e_hw *hw = &pf->hw;
14268         i40e_status ret_code;
14269         int i;
14270
14271         i40e_dbg_pf_exit(pf);
14272
14273         i40e_ptp_stop(pf);
14274
14275         /* Disable RSS in hw */
14276         i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), 0);
14277         i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), 0);
14278
14279         while (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
14280                 usleep_range(1000, 2000);
14281
14282         if (pf->flags & I40E_FLAG_SRIOV_ENABLED) {
14283                 set_bit(__I40E_VF_RESETS_DISABLED, pf->state);
14284                 i40e_free_vfs(pf);
14285                 pf->flags &= ~I40E_FLAG_SRIOV_ENABLED;
14286         }
14287         /* no more scheduling of any task */
14288         set_bit(__I40E_SUSPENDED, pf->state);
14289         set_bit(__I40E_DOWN, pf->state);
14290         if (pf->service_timer.function)
14291                 del_timer_sync(&pf->service_timer);
14292         if (pf->service_task.func)
14293                 cancel_work_sync(&pf->service_task);
14294
14295         /* Client close must be called explicitly here because the timer
14296          * has been stopped.
14297          */
14298         i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], false);
14299
14300         i40e_fdir_teardown(pf);
14301
14302         /* If there is a switch structure or any orphans, remove them.
14303          * This will leave only the PF's VSI remaining.
14304          */
14305         for (i = 0; i < I40E_MAX_VEB; i++) {
14306                 if (!pf->veb[i])
14307                         continue;
14308
14309                 if (pf->veb[i]->uplink_seid == pf->mac_seid ||
14310                     pf->veb[i]->uplink_seid == 0)
14311                         i40e_switch_branch_release(pf->veb[i]);
14312         }
14313
14314         /* Now we can shutdown the PF's VSI, just before we kill
14315          * adminq and hmc.
14316          */
14317         if (pf->vsi[pf->lan_vsi])
14318                 i40e_vsi_release(pf->vsi[pf->lan_vsi]);
14319
14320         i40e_cloud_filter_exit(pf);
14321
14322         /* remove attached clients */
14323         if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
14324                 ret_code = i40e_lan_del_device(pf);
14325                 if (ret_code)
14326                         dev_warn(&pdev->dev, "Failed to delete client device: %d\n",
14327                                  ret_code);
14328         }
14329
14330         /* shutdown and destroy the HMC */
14331         if (hw->hmc.hmc_obj) {
14332                 ret_code = i40e_shutdown_lan_hmc(hw);
14333                 if (ret_code)
14334                         dev_warn(&pdev->dev,
14335                                  "Failed to destroy the HMC resources: %d\n",
14336                                  ret_code);
14337         }
14338
14339         /* shutdown the adminq */
14340         i40e_shutdown_adminq(hw);
14341
14342         /* destroy the locks only once, here */
14343         mutex_destroy(&hw->aq.arq_mutex);
14344         mutex_destroy(&hw->aq.asq_mutex);
14345
14346         /* Clear all dynamic memory lists of rings, q_vectors, and VSIs */
14347         rtnl_lock();
14348         i40e_clear_interrupt_scheme(pf);
14349         for (i = 0; i < pf->num_alloc_vsi; i++) {
14350                 if (pf->vsi[i]) {
14351                         i40e_vsi_clear_rings(pf->vsi[i]);
14352                         i40e_vsi_clear(pf->vsi[i]);
14353                         pf->vsi[i] = NULL;
14354                 }
14355         }
14356         rtnl_unlock();
14357
14358         for (i = 0; i < I40E_MAX_VEB; i++) {
14359                 kfree(pf->veb[i]);
14360                 pf->veb[i] = NULL;
14361         }
14362
14363         kfree(pf->qp_pile);
14364         kfree(pf->vsi);
14365
14366         iounmap(hw->hw_addr);
14367         kfree(pf);
14368         pci_release_mem_regions(pdev);
14369
14370         pci_disable_pcie_error_reporting(pdev);
14371         pci_disable_device(pdev);
14372 }
14373
14374 /**
14375  * i40e_pci_error_detected - warning that something funky happened in PCI land
14376  * @pdev: PCI device information struct
14377  * @error: the type of PCI error
14378  *
14379  * Called to warn that something happened and the error handling steps
14380  * are in progress.  Allows the driver to quiesce things, be ready for
14381  * remediation.
14382  **/
14383 static pci_ers_result_t i40e_pci_error_detected(struct pci_dev *pdev,
14384                                                 enum pci_channel_state error)
14385 {
14386         struct i40e_pf *pf = pci_get_drvdata(pdev);
14387
14388         dev_info(&pdev->dev, "%s: error %d\n", __func__, error);
14389
14390         if (!pf) {
14391                 dev_info(&pdev->dev,
14392                          "Cannot recover - error happened during device probe\n");
14393                 return PCI_ERS_RESULT_DISCONNECT;
14394         }
14395
14396         /* shutdown all operations */
14397         if (!test_bit(__I40E_SUSPENDED, pf->state))
14398                 i40e_prep_for_reset(pf, false);
14399
14400         /* Request a slot reset */
14401         return PCI_ERS_RESULT_NEED_RESET;
14402 }
14403
14404 /**
14405  * i40e_pci_error_slot_reset - a PCI slot reset just happened
14406  * @pdev: PCI device information struct
14407  *
14408  * Called to find if the driver can work with the device now that
14409  * the pci slot has been reset.  If a basic connection seems good
14410  * (registers are readable and have sane content) then return a
14411  * happy little PCI_ERS_RESULT_xxx.
14412  **/
14413 static pci_ers_result_t i40e_pci_error_slot_reset(struct pci_dev *pdev)
14414 {
14415         struct i40e_pf *pf = pci_get_drvdata(pdev);
14416         pci_ers_result_t result;
14417         int err;
14418         u32 reg;
14419
14420         dev_dbg(&pdev->dev, "%s\n", __func__);
14421         if (pci_enable_device_mem(pdev)) {
14422                 dev_info(&pdev->dev,
14423                          "Cannot re-enable PCI device after reset.\n");
14424                 result = PCI_ERS_RESULT_DISCONNECT;
14425         } else {
14426                 pci_set_master(pdev);
14427                 pci_restore_state(pdev);
14428                 pci_save_state(pdev);
14429                 pci_wake_from_d3(pdev, false);
14430
14431                 reg = rd32(&pf->hw, I40E_GLGEN_RTRIG);
14432                 if (reg == 0)
14433                         result = PCI_ERS_RESULT_RECOVERED;
14434                 else
14435                         result = PCI_ERS_RESULT_DISCONNECT;
14436         }
14437
14438         err = pci_cleanup_aer_uncorrect_error_status(pdev);
14439         if (err) {
14440                 dev_info(&pdev->dev,
14441                          "pci_cleanup_aer_uncorrect_error_status failed 0x%0x\n",
14442                          err);
14443                 /* non-fatal, continue */
14444         }
14445
14446         return result;
14447 }
14448
14449 /**
14450  * i40e_pci_error_reset_prepare - prepare device driver for pci reset
14451  * @pdev: PCI device information struct
14452  */
14453 static void i40e_pci_error_reset_prepare(struct pci_dev *pdev)
14454 {
14455         struct i40e_pf *pf = pci_get_drvdata(pdev);
14456
14457         i40e_prep_for_reset(pf, false);
14458 }
14459
14460 /**
14461  * i40e_pci_error_reset_done - pci reset done, device driver reset can begin
14462  * @pdev: PCI device information struct
14463  */
14464 static void i40e_pci_error_reset_done(struct pci_dev *pdev)
14465 {
14466         struct i40e_pf *pf = pci_get_drvdata(pdev);
14467
14468         i40e_reset_and_rebuild(pf, false, false);
14469 }
14470
14471 /**
14472  * i40e_pci_error_resume - restart operations after PCI error recovery
14473  * @pdev: PCI device information struct
14474  *
14475  * Called to allow the driver to bring things back up after PCI error
14476  * and/or reset recovery has finished.
14477  **/
14478 static void i40e_pci_error_resume(struct pci_dev *pdev)
14479 {
14480         struct i40e_pf *pf = pci_get_drvdata(pdev);
14481
14482         dev_dbg(&pdev->dev, "%s\n", __func__);
14483         if (test_bit(__I40E_SUSPENDED, pf->state))
14484                 return;
14485
14486         i40e_handle_reset_warning(pf, false);
14487 }
14488
14489 /**
14490  * i40e_enable_mc_magic_wake - enable multicast magic packet wake up
14491  * using the mac_address_write admin q function
14492  * @pf: pointer to i40e_pf struct
14493  **/
14494 static void i40e_enable_mc_magic_wake(struct i40e_pf *pf)
14495 {
14496         struct i40e_hw *hw = &pf->hw;
14497         i40e_status ret;
14498         u8 mac_addr[6];
14499         u16 flags = 0;
14500
14501         /* Get current MAC address in case it's an LAA */
14502         if (pf->vsi[pf->lan_vsi] && pf->vsi[pf->lan_vsi]->netdev) {
14503                 ether_addr_copy(mac_addr,
14504                                 pf->vsi[pf->lan_vsi]->netdev->dev_addr);
14505         } else {
14506                 dev_err(&pf->pdev->dev,
14507                         "Failed to retrieve MAC address; using default\n");
14508                 ether_addr_copy(mac_addr, hw->mac.addr);
14509         }
14510
14511         /* The FW expects the mac address write cmd to first be called with
14512          * one of these flags before calling it again with the multicast
14513          * enable flags.
14514          */
14515         flags = I40E_AQC_WRITE_TYPE_LAA_WOL;
14516
14517         if (hw->func_caps.flex10_enable && hw->partition_id != 1)
14518                 flags = I40E_AQC_WRITE_TYPE_LAA_ONLY;
14519
14520         ret = i40e_aq_mac_address_write(hw, flags, mac_addr, NULL);
14521         if (ret) {
14522                 dev_err(&pf->pdev->dev,
14523                         "Failed to update MAC address registers; cannot enable Multicast Magic packet wake up");
14524                 return;
14525         }
14526
14527         flags = I40E_AQC_MC_MAG_EN
14528                         | I40E_AQC_WOL_PRESERVE_ON_PFR
14529                         | I40E_AQC_WRITE_TYPE_UPDATE_MC_MAG;
14530         ret = i40e_aq_mac_address_write(hw, flags, mac_addr, NULL);
14531         if (ret)
14532                 dev_err(&pf->pdev->dev,
14533                         "Failed to enable Multicast Magic Packet wake up\n");
14534 }
14535
14536 /**
14537  * i40e_shutdown - PCI callback for shutting down
14538  * @pdev: PCI device information struct
14539  **/
14540 static void i40e_shutdown(struct pci_dev *pdev)
14541 {
14542         struct i40e_pf *pf = pci_get_drvdata(pdev);
14543         struct i40e_hw *hw = &pf->hw;
14544
14545         set_bit(__I40E_SUSPENDED, pf->state);
14546         set_bit(__I40E_DOWN, pf->state);
14547
14548         del_timer_sync(&pf->service_timer);
14549         cancel_work_sync(&pf->service_task);
14550         i40e_cloud_filter_exit(pf);
14551         i40e_fdir_teardown(pf);
14552
14553         /* Client close must be called explicitly here because the timer
14554          * has been stopped.
14555          */
14556         i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], false);
14557
14558         if (pf->wol_en && (pf->hw_features & I40E_HW_WOL_MC_MAGIC_PKT_WAKE))
14559                 i40e_enable_mc_magic_wake(pf);
14560
14561         i40e_prep_for_reset(pf, false);
14562
14563         wr32(hw, I40E_PFPM_APM,
14564              (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0));
14565         wr32(hw, I40E_PFPM_WUFC,
14566              (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0));
14567
14568         /* Since we're going to destroy queues during the
14569          * i40e_clear_interrupt_scheme() we should hold the RTNL lock for this
14570          * whole section
14571          */
14572         rtnl_lock();
14573         i40e_clear_interrupt_scheme(pf);
14574         rtnl_unlock();
14575
14576         if (system_state == SYSTEM_POWER_OFF) {
14577                 pci_wake_from_d3(pdev, pf->wol_en);
14578                 pci_set_power_state(pdev, PCI_D3hot);
14579         }
14580 }
14581
14582 /**
14583  * i40e_suspend - PM callback for moving to D3
14584  * @dev: generic device information structure
14585  **/
14586 static int __maybe_unused i40e_suspend(struct device *dev)
14587 {
14588         struct pci_dev *pdev = to_pci_dev(dev);
14589         struct i40e_pf *pf = pci_get_drvdata(pdev);
14590         struct i40e_hw *hw = &pf->hw;
14591
14592         /* If we're already suspended, then there is nothing to do */
14593         if (test_and_set_bit(__I40E_SUSPENDED, pf->state))
14594                 return 0;
14595
14596         set_bit(__I40E_DOWN, pf->state);
14597
14598         /* Ensure service task will not be running */
14599         del_timer_sync(&pf->service_timer);
14600         cancel_work_sync(&pf->service_task);
14601
14602         /* Client close must be called explicitly here because the timer
14603          * has been stopped.
14604          */
14605         i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], false);
14606
14607         if (pf->wol_en && (pf->hw_features & I40E_HW_WOL_MC_MAGIC_PKT_WAKE))
14608                 i40e_enable_mc_magic_wake(pf);
14609
14610         /* Since we're going to destroy queues during the
14611          * i40e_clear_interrupt_scheme() we should hold the RTNL lock for this
14612          * whole section
14613          */
14614         rtnl_lock();
14615
14616         i40e_prep_for_reset(pf, true);
14617
14618         wr32(hw, I40E_PFPM_APM, (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0));
14619         wr32(hw, I40E_PFPM_WUFC, (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0));
14620
14621         /* Clear the interrupt scheme and release our IRQs so that the system
14622          * can safely hibernate even when there are a large number of CPUs.
14623          * Otherwise hibernation might fail when mapping all the vectors back
14624          * to CPU0.
14625          */
14626         i40e_clear_interrupt_scheme(pf);
14627
14628         rtnl_unlock();
14629
14630         return 0;
14631 }
14632
14633 /**
14634  * i40e_resume - PM callback for waking up from D3
14635  * @dev: generic device information structure
14636  **/
14637 static int __maybe_unused i40e_resume(struct device *dev)
14638 {
14639         struct pci_dev *pdev = to_pci_dev(dev);
14640         struct i40e_pf *pf = pci_get_drvdata(pdev);
14641         int err;
14642
14643         /* If we're not suspended, then there is nothing to do */
14644         if (!test_bit(__I40E_SUSPENDED, pf->state))
14645                 return 0;
14646
14647         /* We need to hold the RTNL lock prior to restoring interrupt schemes,
14648          * since we're going to be restoring queues
14649          */
14650         rtnl_lock();
14651
14652         /* We cleared the interrupt scheme when we suspended, so we need to
14653          * restore it now to resume device functionality.
14654          */
14655         err = i40e_restore_interrupt_scheme(pf);
14656         if (err) {
14657                 dev_err(&pdev->dev, "Cannot restore interrupt scheme: %d\n",
14658                         err);
14659         }
14660
14661         clear_bit(__I40E_DOWN, pf->state);
14662         i40e_reset_and_rebuild(pf, false, true);
14663
14664         rtnl_unlock();
14665
14666         /* Clear suspended state last after everything is recovered */
14667         clear_bit(__I40E_SUSPENDED, pf->state);
14668
14669         /* Restart the service task */
14670         mod_timer(&pf->service_timer,
14671                   round_jiffies(jiffies + pf->service_timer_period));
14672
14673         return 0;
14674 }
14675
14676 static const struct pci_error_handlers i40e_err_handler = {
14677         .error_detected = i40e_pci_error_detected,
14678         .slot_reset = i40e_pci_error_slot_reset,
14679         .reset_prepare = i40e_pci_error_reset_prepare,
14680         .reset_done = i40e_pci_error_reset_done,
14681         .resume = i40e_pci_error_resume,
14682 };
14683
14684 static SIMPLE_DEV_PM_OPS(i40e_pm_ops, i40e_suspend, i40e_resume);
14685
14686 static struct pci_driver i40e_driver = {
14687         .name     = i40e_driver_name,
14688         .id_table = i40e_pci_tbl,
14689         .probe    = i40e_probe,
14690         .remove   = i40e_remove,
14691         .driver   = {
14692                 .pm = &i40e_pm_ops,
14693         },
14694         .shutdown = i40e_shutdown,
14695         .err_handler = &i40e_err_handler,
14696         .sriov_configure = i40e_pci_sriov_configure,
14697 };
14698
14699 /**
14700  * i40e_init_module - Driver registration routine
14701  *
14702  * i40e_init_module is the first routine called when the driver is
14703  * loaded. All it does is register with the PCI subsystem.
14704  **/
14705 static int __init i40e_init_module(void)
14706 {
14707         pr_info("%s: %s - version %s\n", i40e_driver_name,
14708                 i40e_driver_string, i40e_driver_version_str);
14709         pr_info("%s: %s\n", i40e_driver_name, i40e_copyright);
14710
14711         /* There is no need to throttle the number of active tasks because
14712          * each device limits its own task using a state bit for scheduling
14713          * the service task, and the device tasks do not interfere with each
14714          * other, so we don't set a max task limit. We must set WQ_MEM_RECLAIM
14715          * since we need to be able to guarantee forward progress even under
14716          * memory pressure.
14717          */
14718         i40e_wq = alloc_workqueue("%s", WQ_MEM_RECLAIM, 0, i40e_driver_name);
14719         if (!i40e_wq) {
14720                 pr_err("%s: Failed to create workqueue\n", i40e_driver_name);
14721                 return -ENOMEM;
14722         }
14723
14724         i40e_dbg_init();
14725         return pci_register_driver(&i40e_driver);
14726 }
14727 module_init(i40e_init_module);
14728
14729 /**
14730  * i40e_exit_module - Driver exit cleanup routine
14731  *
14732  * i40e_exit_module is called just before the driver is removed
14733  * from memory.
14734  **/
14735 static void __exit i40e_exit_module(void)
14736 {
14737         pci_unregister_driver(&i40e_driver);
14738         destroy_workqueue(i40e_wq);
14739         i40e_dbg_exit();
14740 }
14741 module_exit(i40e_exit_module);