drivers/net/ethernet/hisilicon/hns/hns_enet.c

   1 /*
   2  * Copyright (c) 2014-2015 Hisilicon Limited.
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License as published by
   6  * the Free Software Foundation; either version 2 of the License, or
   7  * (at your option) any later version.
   8  */
   9
  10 #include <linux/clk.h>
  11 #include <linux/cpumask.h>
  12 #include <linux/etherdevice.h>
  13 #include <linux/if_vlan.h>
  14 #include <linux/interrupt.h>
  15 #include <linux/io.h>
  16 #include <linux/ip.h>
  17 #include <linux/ipv6.h>
  18 #include <linux/module.h>
  19 #include <linux/phy.h>
  20 #include <linux/platform_device.h>
  21 #include <linux/skbuff.h>
  22
  23 #include "hnae.h"
  24 #include "hns_enet.h"
  25
  26 #define NIC_MAX_Q_PER_VF 16
  27 #define HNS_NIC_TX_TIMEOUT (5 * HZ)
  28
  29 #define SERVICE_TIMER_HZ (1 * HZ)
  30
  31 #define RCB_IRQ_NOT_INITED 0
  32 #define RCB_IRQ_INITED 1
  33
  34 static void fill_desc(struct hnae_ring *ring, void *priv,
  35                       int size, dma_addr_t dma, int frag_end,
  36                       int buf_num, enum hns_desc_type type)
  37 {
  38         struct hnae_desc *desc = &ring->desc[ring->next_to_use];
  39         struct hnae_desc_cb *desc_cb = &ring->desc_cb[ring->next_to_use];
  40         struct sk_buff *skb;
  41         __be16 protocol;
  42         u32 ip_offset;
  43         u32 asid_bufnum_pid = 0;
  44         u32 flag_ipoffset = 0;
  45
  46         desc_cb->priv = priv;
  47         desc_cb->length = size;
  48         desc_cb->dma = dma;
  49         desc_cb->type = type;
  50
  51         desc->addr = cpu_to_le64(dma);
  52         desc->tx.send_size = cpu_to_le16((u16)size);
  53
  54         /*config bd buffer end */
  55         flag_ipoffset |= 1 << HNS_TXD_VLD_B;
  56
  57         asid_bufnum_pid |= buf_num << HNS_TXD_BUFNUM_S;
  58
  59         if (type == DESC_TYPE_SKB) {
  60                 skb = (struct sk_buff *)priv;
  61
  62                 if (skb->ip_summed == CHECKSUM_PARTIAL) {
  63                         protocol = skb->protocol;
  64                         ip_offset = ETH_HLEN;
  65
  66                         /*if it is a SW VLAN check the next protocol*/
  67                         if (protocol == htons(ETH_P_8021Q)) {
  68                                 ip_offset += VLAN_HLEN;
  69                                 protocol = vlan_get_protocol(skb);
  70                                 skb->protocol = protocol;
  71                         }
  72
  73                         if (skb->protocol == htons(ETH_P_IP)) {
  74                                 flag_ipoffset |= 1 << HNS_TXD_L3CS_B;
  75                                 /* check for tcp/udp header */
  76                                 flag_ipoffset |= 1 << HNS_TXD_L4CS_B;
  77
  78                         } else if (skb->protocol == htons(ETH_P_IPV6)) {
  79                                 /* ipv6 has not l3 cs, check for L4 header */
  80                                 flag_ipoffset |= 1 << HNS_TXD_L4CS_B;
  81                         }
  82
  83                         flag_ipoffset |= ip_offset << HNS_TXD_IPOFFSET_S;
  84                 }
  85         }
  86
  87         flag_ipoffset |= frag_end << HNS_TXD_FE_B;
  88
  89         desc->tx.asid_bufnum_pid = cpu_to_le16(asid_bufnum_pid);
  90         desc->tx.flag_ipoffset = cpu_to_le32(flag_ipoffset);
  91
  92         ring_ptr_move_fw(ring, next_to_use);
  93 }
  94
  95 static void unfill_desc(struct hnae_ring *ring)
  96 {
  97         ring_ptr_move_bw(ring, next_to_use);
  98 }
  99
 100 int hns_nic_net_xmit_hw(struct net_device *ndev,
 101                         struct sk_buff *skb,
 102                         struct hns_nic_ring_data *ring_data)
 103 {
 104         struct hns_nic_priv *priv = netdev_priv(ndev);
 105         struct hnae_ring *ring = ring_data->ring;
 106         struct device *dev = ring_to_dev(ring);
 107         struct netdev_queue *dev_queue;
 108         struct skb_frag_struct *frag;
 109         int buf_num;
 110         dma_addr_t dma;
 111         int size, next_to_use;
 112         int i, j;
 113         struct sk_buff *new_skb;
 114
 115         assert(ring->max_desc_num_per_pkt <= ring->desc_num);
 116
 117         /* no. of segments (plus a header) */
 118         buf_num = skb_shinfo(skb)->nr_frags + 1;
 119
 120         if (unlikely(buf_num > ring->max_desc_num_per_pkt)) {
 121                 if (ring_space(ring) < 1) {
 122                         ring->stats.tx_busy++;
 123                         goto out_net_tx_busy;
 124                 }
 125
 126                 new_skb = skb_copy(skb, GFP_ATOMIC);
 127                 if (!new_skb) {
 128                         ring->stats.sw_err_cnt++;
 129                         netdev_err(ndev, "no memory to xmit!\n");
 130                         goto out_err_tx_ok;
 131                 }
 132
 133                 dev_kfree_skb_any(skb);
 134                 skb = new_skb;
 135                 buf_num = 1;
 136                 assert(skb_shinfo(skb)->nr_frags == 1);
 137         } else if (buf_num > ring_space(ring)) {
 138                 ring->stats.tx_busy++;
 139                 goto out_net_tx_busy;
 140         }
 141         next_to_use = ring->next_to_use;
 142
 143         /* fill the first part */
 144         size = skb_headlen(skb);
 145         dma = dma_map_single(dev, skb->data, size, DMA_TO_DEVICE);
 146         if (dma_mapping_error(dev, dma)) {
 147                 netdev_err(ndev, "TX head DMA map failed\n");
 148                 ring->stats.sw_err_cnt++;
 149                 goto out_err_tx_ok;
 150         }
 151         fill_desc(ring, skb, size, dma, buf_num == 1 ? 1 : 0, buf_num,
 152                   DESC_TYPE_SKB);
 153
 154         /* fill the fragments */
 155         for (i = 1; i < buf_num; i++) {
 156                 frag = &skb_shinfo(skb)->frags[i - 1];
 157                 size = skb_frag_size(frag);
 158                 dma = skb_frag_dma_map(dev, frag, 0, size, DMA_TO_DEVICE);
 159                 if (dma_mapping_error(dev, dma)) {
 160                         netdev_err(ndev, "TX frag(%d) DMA map failed\n", i);
 161                         ring->stats.sw_err_cnt++;
 162                         goto out_map_frag_fail;
 163                 }
 164                 fill_desc(ring, skb_frag_page(frag), size, dma,
 165                           buf_num - 1 == i ? 1 : 0, buf_num, DESC_TYPE_PAGE);
 166         }
 167
 168         /*complete translate all packets*/
 169         dev_queue = netdev_get_tx_queue(ndev, skb->queue_mapping);
 170         netdev_tx_sent_queue(dev_queue, skb->len);
 171
 172         wmb(); /* commit all data before submit */
 173         assert(skb->queue_mapping < priv->ae_handle->q_num);
 174         hnae_queue_xmit(priv->ae_handle->qs[skb->queue_mapping], buf_num);
 175         ring->stats.tx_pkts++;
 176         ring->stats.tx_bytes += skb->len;
 177
 178         return NETDEV_TX_OK;
 179
 180 out_map_frag_fail:
 181
 182         for (j = i - 1; j > 0; j--) {
 183                 unfill_desc(ring);
 184                 next_to_use = ring->next_to_use;
 185                 dma_unmap_page(dev, ring->desc_cb[next_to_use].dma,
 186                                ring->desc_cb[next_to_use].length,
 187                                DMA_TO_DEVICE);
 188         }
 189
 190         unfill_desc(ring);
 191         next_to_use = ring->next_to_use;
 192         dma_unmap_single(dev, ring->desc_cb[next_to_use].dma,
 193                          ring->desc_cb[next_to_use].length, DMA_TO_DEVICE);
 194
 195 out_err_tx_ok:
 196
 197         dev_kfree_skb_any(skb);
 198         return NETDEV_TX_OK;
 199
 200 out_net_tx_busy:
 201
 202         netif_stop_subqueue(ndev, skb->queue_mapping);
 203
 204         /* Herbert's original patch had:
 205          *  smp_mb__after_netif_stop_queue();
 206          * but since that doesn't exist yet, just open code it.
 207          */
 208         smp_mb();
 209         return NETDEV_TX_BUSY;
 210 }
 211
 212 /**
 213  * hns_nic_get_headlen - determine size of header for RSC/LRO/GRO/FCOE
 214  * @data: pointer to the start of the headers
 215  * @max: total length of section to find headers in
 216  *
 217  * This function is meant to determine the length of headers that will
 218  * be recognized by hardware for LRO, GRO, and RSC offloads.  The main
 219  * motivation of doing this is to only perform one pull for IPv4 TCP
 220  * packets so that we can do basic things like calculating the gso_size
 221  * based on the average data per packet.
 222  **/
 223 static unsigned int hns_nic_get_headlen(unsigned char *data, u32 flag,
 224                                         unsigned int max_size)
 225 {
 226         unsigned char *network;
 227         u8 hlen;
 228
 229         /* this should never happen, but better safe than sorry */
 230         if (max_size < ETH_HLEN)
 231                 return max_size;
 232
 233         /* initialize network frame pointer */
 234         network = data;
 235
 236         /* set first protocol and move network header forward */
 237         network += ETH_HLEN;
 238
 239         /* handle any vlan tag if present */
 240         if (hnae_get_field(flag, HNS_RXD_VLAN_M, HNS_RXD_VLAN_S)
 241                 == HNS_RX_FLAG_VLAN_PRESENT) {
 242                 if ((typeof(max_size))(network - data) > (max_size - VLAN_HLEN))
 243                         return max_size;
 244
 245                 network += VLAN_HLEN;
 246         }
 247
 248         /* handle L3 protocols */
 249         if (hnae_get_field(flag, HNS_RXD_L3ID_M, HNS_RXD_L3ID_S)
 250                 == HNS_RX_FLAG_L3ID_IPV4) {
 251                 if ((typeof(max_size))(network - data) >
 252                     (max_size - sizeof(struct iphdr)))
 253                         return max_size;
 254
 255                 /* access ihl as a u8 to avoid unaligned access on ia64 */
 256                 hlen = (network[0] & 0x0F) << 2;
 257
 258                 /* verify hlen meets minimum size requirements */
 259                 if (hlen < sizeof(struct iphdr))
 260                         return network - data;
 261
 262                 /* record next protocol if header is present */
 263         } else if (hnae_get_field(flag, HNS_RXD_L3ID_M, HNS_RXD_L3ID_S)
 264                 == HNS_RX_FLAG_L3ID_IPV6) {
 265                 if ((typeof(max_size))(network - data) >
 266                     (max_size - sizeof(struct ipv6hdr)))
 267                         return max_size;
 268
 269                 /* record next protocol */
 270                 hlen = sizeof(struct ipv6hdr);
 271         } else {
 272                 return network - data;
 273         }
 274
 275         /* relocate pointer to start of L4 header */
 276         network += hlen;
 277
 278         /* finally sort out TCP/UDP */
 279         if (hnae_get_field(flag, HNS_RXD_L4ID_M, HNS_RXD_L4ID_S)
 280                 == HNS_RX_FLAG_L4ID_TCP) {
 281                 if ((typeof(max_size))(network - data) >
 282                     (max_size - sizeof(struct tcphdr)))
 283                         return max_size;
 284
 285                 /* access doff as a u8 to avoid unaligned access on ia64 */
 286                 hlen = (network[12] & 0xF0) >> 2;
 287
 288                 /* verify hlen meets minimum size requirements */
 289                 if (hlen < sizeof(struct tcphdr))
 290                         return network - data;
 291
 292                 network += hlen;
 293         } else if (hnae_get_field(flag, HNS_RXD_L4ID_M, HNS_RXD_L4ID_S)
 294                 == HNS_RX_FLAG_L4ID_UDP) {
 295                 if ((typeof(max_size))(network - data) >
 296                     (max_size - sizeof(struct udphdr)))
 297                         return max_size;
 298
 299                 network += sizeof(struct udphdr);
 300         }
 301
 302         /* If everything has gone correctly network should be the
 303          * data section of the packet and will be the end of the header.
 304          * If not then it probably represents the end of the last recognized
 305          * header.
 306          */
 307         if ((typeof(max_size))(network - data) < max_size)
 308                 return network - data;
 309         else
 310                 return max_size;
 311 }
 312
 313 static void
 314 hns_nic_reuse_page(struct hnae_desc_cb *desc_cb, int tsize, int last_offset)
 315 {
 316          /* avoid re-using remote pages,flag default unreuse */
 317         if (likely(page_to_nid(desc_cb->priv) == numa_node_id())) {
 318                 /* move offset up to the next cache line */
 319                 desc_cb->page_offset += tsize;
 320
 321                 if (desc_cb->page_offset <= last_offset) {
 322                         desc_cb->reuse_flag = 1;
 323                         /* bump ref count on page before it is given*/
 324                         get_page(desc_cb->priv);
 325                 }
 326         }
 327 }
 328
 329 static int hns_nic_poll_rx_skb(struct hns_nic_ring_data *ring_data,
 330                                struct sk_buff **out_skb, int *out_bnum)
 331 {
 332         struct hnae_ring *ring = ring_data->ring;
 333         struct net_device *ndev = ring_data->napi.dev;
 334         struct sk_buff *skb;
 335         struct hnae_desc *desc;
 336         struct hnae_desc_cb *desc_cb;
 337         unsigned char *va;
 338         int bnum, length, size, i, truesize, last_offset;
 339         int pull_len;
 340         u32 bnum_flag;
 341
 342         last_offset = hnae_page_size(ring) - hnae_buf_size(ring);
 343         desc = &ring->desc[ring->next_to_clean];
 344         desc_cb = &ring->desc_cb[ring->next_to_clean];
 345         length = le16_to_cpu(desc->rx.pkt_len);
 346         bnum_flag = le32_to_cpu(desc->rx.ipoff_bnum_pid_flag);
 347         bnum = hnae_get_field(bnum_flag, HNS_RXD_BUFNUM_M, HNS_RXD_BUFNUM_S);
 348         *out_bnum = bnum;
 349         va = (unsigned char *)desc_cb->buf + desc_cb->page_offset;
 350
 351         skb = *out_skb = napi_alloc_skb(&ring_data->napi, HNS_RX_HEAD_SIZE);
 352         if (unlikely(!skb)) {
 353                 netdev_err(ndev, "alloc rx skb fail\n");
 354                 ring->stats.sw_err_cnt++;
 355                 return -ENOMEM;
 356         }
 357
 358         if (length <= HNS_RX_HEAD_SIZE) {
 359                 memcpy(__skb_put(skb, length), va, ALIGN(length, sizeof(long)));
 360
 361                 /* we can reuse buffer as-is, just make sure it is local */
 362                 if (likely(page_to_nid(desc_cb->priv) == numa_node_id()))
 363                         desc_cb->reuse_flag = 1;
 364                 else /* this page cannot be reused so discard it */
 365                         put_page(desc_cb->priv);
 366
 367                 ring_ptr_move_fw(ring, next_to_clean);
 368
 369                 if (unlikely(bnum != 1)) { /* check err*/
 370                         *out_bnum = 1;
 371                         goto out_bnum_err;
 372                 }
 373         } else {
 374                 ring->stats.seg_pkt_cnt++;
 375
 376                 pull_len = hns_nic_get_headlen(va, bnum_flag, HNS_RX_HEAD_SIZE);
 377                 memcpy(__skb_put(skb, pull_len), va,
 378                        ALIGN(pull_len, sizeof(long)));
 379
 380                 size = le16_to_cpu(desc->rx.size);
 381                 truesize = ALIGN(size, L1_CACHE_BYTES);
 382                 skb_add_rx_frag(skb, 0, desc_cb->priv,
 383                                 desc_cb->page_offset + pull_len,
 384                                 size - pull_len, truesize - pull_len);
 385
 386                 hns_nic_reuse_page(desc_cb, truesize, last_offset);
 387                 ring_ptr_move_fw(ring, next_to_clean);
 388
 389                 if (unlikely(bnum >= (int)MAX_SKB_FRAGS)) { /* check err*/
 390                         *out_bnum = 1;
 391                         goto out_bnum_err;
 392                 }
 393                 for (i = 1; i < bnum; i++) {
 394                         desc = &ring->desc[ring->next_to_clean];
 395                         desc_cb = &ring->desc_cb[ring->next_to_clean];
 396                         size = le16_to_cpu(desc->rx.size);
 397                         truesize = ALIGN(size, L1_CACHE_BYTES);
 398                         skb_add_rx_frag(skb, i, desc_cb->priv,
 399                                         desc_cb->page_offset,
 400                                         size, truesize);
 401
 402                         hns_nic_reuse_page(desc_cb, truesize, last_offset);
 403                         ring_ptr_move_fw(ring, next_to_clean);
 404                 }
 405         }
 406
 407         /* check except process, free skb and jump the desc */
 408         if (unlikely((!bnum) || (bnum > ring->max_desc_num_per_pkt))) {
 409 out_bnum_err:
 410                 *out_bnum = *out_bnum ? *out_bnum : 1; /* ntc moved,cannot 0*/
 411                 netdev_err(ndev, "invalid bnum(%d,%d,%d,%d),%016llx,%016llx\n",
 412                            bnum, ring->max_desc_num_per_pkt,
 413                            length, (int)MAX_SKB_FRAGS,
 414                            ((u64 *)desc)[0], ((u64 *)desc)[1]);
 415                 ring->stats.err_bd_num++;
 416                 dev_kfree_skb_any(skb);
 417                 return -EDOM;
 418         }
 419
 420         bnum_flag = le32_to_cpu(desc->rx.ipoff_bnum_pid_flag);
 421
 422         if (unlikely(!hnae_get_bit(bnum_flag, HNS_RXD_VLD_B))) {
 423                 netdev_err(ndev, "no valid bd,%016llx,%016llx\n",
 424                            ((u64 *)desc)[0], ((u64 *)desc)[1]);
 425                 ring->stats.non_vld_descs++;
 426                 dev_kfree_skb_any(skb);
 427                 return -EINVAL;
 428         }
 429
 430         if (unlikely((!desc->rx.pkt_len) ||
 431                      hnae_get_bit(bnum_flag, HNS_RXD_DROP_B))) {
 432                 ring->stats.err_pkt_len++;
 433                 dev_kfree_skb_any(skb);
 434                 return -EFAULT;
 435         }
 436
 437         if (unlikely(hnae_get_bit(bnum_flag, HNS_RXD_L2E_B))) {
 438                 ring->stats.l2_err++;
 439                 dev_kfree_skb_any(skb);
 440                 return -EFAULT;
 441         }
 442
 443         ring->stats.rx_pkts++;
 444         ring->stats.rx_bytes += skb->len;
 445
 446         if (unlikely(hnae_get_bit(bnum_flag, HNS_RXD_L3E_B) ||
 447                      hnae_get_bit(bnum_flag, HNS_RXD_L4E_B))) {
 448                 ring->stats.l3l4_csum_err++;
 449                 return 0;
 450         }
 451
 452         skb->ip_summed = CHECKSUM_UNNECESSARY;
 453
 454         return 0;
 455 }
 456
 457 static void
 458 hns_nic_alloc_rx_buffers(struct hns_nic_ring_data *ring_data, int cleand_count)
 459 {
 460         int i, ret;
 461         struct hnae_desc_cb res_cbs;
 462         struct hnae_desc_cb *desc_cb;
 463         struct hnae_ring *ring = ring_data->ring;
 464         struct net_device *ndev = ring_data->napi.dev;
 465
 466         for (i = 0; i < cleand_count; i++) {
 467                 desc_cb = &ring->desc_cb[ring->next_to_use];
 468                 if (desc_cb->reuse_flag) {
 469                         ring->stats.reuse_pg_cnt++;
 470                         hnae_reuse_buffer(ring, ring->next_to_use);
 471                 } else {
 472                         ret = hnae_reserve_buffer_map(ring, &res_cbs);
 473                         if (ret) {
 474                                 ring->stats.sw_err_cnt++;
 475                                 netdev_err(ndev, "hnae reserve buffer map failed.\n");
 476                                 break;
 477                         }
 478                         hnae_replace_buffer(ring, ring->next_to_use, &res_cbs);
 479                 }
 480
 481                 ring_ptr_move_fw(ring, next_to_use);
 482         }
 483
 484         wmb(); /* make all data has been write before submit */
 485         writel_relaxed(i, ring->io_base + RCB_REG_HEAD);
 486 }
 487
 488 /* return error number for error or number of desc left to take
 489  */
 490 static void hns_nic_rx_up_pro(struct hns_nic_ring_data *ring_data,
 491                               struct sk_buff *skb)
 492 {
 493         struct net_device *ndev = ring_data->napi.dev;
 494
 495         skb->protocol = eth_type_trans(skb, ndev);
 496         (void)napi_gro_receive(&ring_data->napi, skb);
 497         ndev->last_rx = jiffies;
 498 }
 499
 500 static int hns_nic_rx_poll_one(struct hns_nic_ring_data *ring_data,
 501                                int budget, void *v)
 502 {
 503         struct hnae_ring *ring = ring_data->ring;
 504         struct sk_buff *skb;
 505         int num, bnum, ex_num;
 506 #define RCB_NOF_ALLOC_RX_BUFF_ONCE 16
 507         int recv_pkts, recv_bds, clean_count, err;
 508
 509         num = readl_relaxed(ring->io_base + RCB_REG_FBDNUM);
 510         rmb(); /* make sure num taken effect before the other data is touched */
 511
 512         recv_pkts = 0, recv_bds = 0, clean_count = 0;
 513 recv:
 514         while (recv_pkts < budget && recv_bds < num) {
 515                 /* reuse or realloc buffers*/
 516                 if (clean_count >= RCB_NOF_ALLOC_RX_BUFF_ONCE) {
 517                         hns_nic_alloc_rx_buffers(ring_data, clean_count);
 518                         clean_count = 0;
 519                 }
 520
 521                 /* poll one pkg*/
 522                 err = hns_nic_poll_rx_skb(ring_data, &skb, &bnum);
 523                 if (unlikely(!skb)) /* this fault cannot be repaired */
 524                         break;
 525
 526                 recv_bds += bnum;
 527                 clean_count += bnum;
 528                 if (unlikely(err)) {  /* do jump the err */
 529                         recv_pkts++;
 530                         continue;
 531                 }
 532
 533                 /* do update ip stack process*/
 534                 ((void (*)(struct hns_nic_ring_data *, struct sk_buff *))v)(
 535                                                         ring_data, skb);
 536                 recv_pkts++;
 537         }
 538
 539         /* make all data has been write before submit */
 540         if (clean_count > 0) {
 541                 hns_nic_alloc_rx_buffers(ring_data, clean_count);
 542                 clean_count = 0;
 543         }
 544
 545         if (recv_pkts < budget) {
 546                 ex_num = readl_relaxed(ring->io_base + RCB_REG_FBDNUM);
 547                 rmb(); /*complete read rx ring bd number*/
 548                 if (ex_num > 0) {
 549                         num += ex_num;
 550                         goto recv;
 551                 }
 552         }
 553
 554         return recv_pkts;
 555 }
 556
 557 static void hns_nic_rx_fini_pro(struct hns_nic_ring_data *ring_data)
 558 {
 559         struct hnae_ring *ring = ring_data->ring;
 560         int num = 0;
 561
 562         /* for hardware bug fixed */
 563         num = readl_relaxed(ring->io_base + RCB_REG_FBDNUM);
 564
 565         if (num > 0) {
 566                 ring_data->ring->q->handle->dev->ops->toggle_ring_irq(
 567                         ring_data->ring, 1);
 568
 569                 napi_schedule(&ring_data->napi);
 570         }
 571 }
 572
 573 static inline void hns_nic_reclaim_one_desc(struct hnae_ring *ring,
 574                                             int *bytes, int *pkts)
 575 {
 576         struct hnae_desc_cb *desc_cb = &ring->desc_cb[ring->next_to_clean];
 577
 578         (*pkts) += (desc_cb->type == DESC_TYPE_SKB);
 579         (*bytes) += desc_cb->length;
 580         /* desc_cb will be cleaned, after hnae_free_buffer_detach*/
 581         hnae_free_buffer_detach(ring, ring->next_to_clean);
 582
 583         ring_ptr_move_fw(ring, next_to_clean);
 584 }
 585
 586 static int is_valid_clean_head(struct hnae_ring *ring, int h)
 587 {
 588         int u = ring->next_to_use;
 589         int c = ring->next_to_clean;
 590
 591         if (unlikely(h > ring->desc_num))
 592                 return 0;
 593
 594         assert(u > 0 && u < ring->desc_num);
 595         assert(c > 0 && c < ring->desc_num);
 596         assert(u != c && h != c); /* must be checked before call this func */
 597
 598         return u > c ? (h > c && h <= u) : (h > c || h <= u);
 599 }
 600
 601 /* netif_tx_lock will turn down the performance, set only when necessary */
 602 #ifdef CONFIG_NET_POLL_CONTROLLER
 603 #define NETIF_TX_LOCK(ndev) netif_tx_lock(ndev)
 604 #define NETIF_TX_UNLOCK(ndev) netif_tx_unlock(ndev)
 605 #else
 606 #define NETIF_TX_LOCK(ndev)
 607 #define NETIF_TX_UNLOCK(ndev)
 608 #endif
 609 /* reclaim all desc in one budget
 610  * return error or number of desc left
 611  */
 612 static int hns_nic_tx_poll_one(struct hns_nic_ring_data *ring_data,
 613                                int budget, void *v)
 614 {
 615         struct hnae_ring *ring = ring_data->ring;
 616         struct net_device *ndev = ring_data->napi.dev;
 617         struct netdev_queue *dev_queue;
 618         struct hns_nic_priv *priv = netdev_priv(ndev);
 619         int head;
 620         int bytes, pkts;
 621
 622         NETIF_TX_LOCK(ndev);
 623
 624         head = readl_relaxed(ring->io_base + RCB_REG_HEAD);
 625         rmb(); /* make sure head is ready before touch any data */
 626
 627         if (is_ring_empty(ring) || head == ring->next_to_clean) {
 628                 NETIF_TX_UNLOCK(ndev);
 629                 return 0; /* no data to poll */
 630         }
 631
 632         if (!is_valid_clean_head(ring, head)) {
 633                 netdev_err(ndev, "wrong head (%d, %d-%d)\n", head,
 634                            ring->next_to_use, ring->next_to_clean);
 635                 ring->stats.io_err_cnt++;
 636                 NETIF_TX_UNLOCK(ndev);
 637                 return -EIO;
 638         }
 639
 640         bytes = 0;
 641         pkts = 0;
 642         while (head != ring->next_to_clean)
 643                 hns_nic_reclaim_one_desc(ring, &bytes, &pkts);
 644
 645         NETIF_TX_UNLOCK(ndev);
 646
 647         dev_queue = netdev_get_tx_queue(ndev, ring_data->queue_index);
 648         netdev_tx_completed_queue(dev_queue, pkts, bytes);
 649
 650         if (unlikely(pkts && netif_carrier_ok(ndev) &&
 651                      (ring_space(ring) >= ring->max_desc_num_per_pkt * 2))) {
 652                 /* Make sure that anybody stopping the queue after this
 653                  * sees the new next_to_clean.
 654                  */
 655                 smp_mb();
 656                 if (netif_tx_queue_stopped(dev_queue) &&
 657                     !test_bit(NIC_STATE_DOWN, &priv->state)) {
 658                         netif_tx_wake_queue(dev_queue);
 659                         ring->stats.restart_queue++;
 660                 }
 661         }
 662         return 0;
 663 }
 664
 665 static void hns_nic_tx_fini_pro(struct hns_nic_ring_data *ring_data)
 666 {
 667         struct hnae_ring *ring = ring_data->ring;
 668         int head = ring->next_to_clean;
 669
 670         /* for hardware bug fixed */
 671         head = readl_relaxed(ring->io_base + RCB_REG_HEAD);
 672
 673         if (head != ring->next_to_clean) {
 674                 ring_data->ring->q->handle->dev->ops->toggle_ring_irq(
 675                         ring_data->ring, 1);
 676
 677                 napi_schedule(&ring_data->napi);
 678         }
 679 }
 680
 681 static void hns_nic_tx_clr_all_bufs(struct hns_nic_ring_data *ring_data)
 682 {
 683         struct hnae_ring *ring = ring_data->ring;
 684         struct net_device *ndev = ring_data->napi.dev;
 685         struct netdev_queue *dev_queue;
 686         int head;
 687         int bytes, pkts;
 688
 689         NETIF_TX_LOCK(ndev);
 690
 691         head = ring->next_to_use; /* ntu :soft setted ring position*/
 692         bytes = 0;
 693         pkts = 0;
 694         while (head != ring->next_to_clean)
 695                 hns_nic_reclaim_one_desc(ring, &bytes, &pkts);
 696
 697         NETIF_TX_UNLOCK(ndev);
 698
 699         dev_queue = netdev_get_tx_queue(ndev, ring_data->queue_index);
 700         netdev_tx_reset_queue(dev_queue);
 701 }
 702
 703 static int hns_nic_common_poll(struct napi_struct *napi, int budget)
 704 {
 705         struct hns_nic_ring_data *ring_data =
 706                 container_of(napi, struct hns_nic_ring_data, napi);
 707         int clean_complete = ring_data->poll_one(
 708                                 ring_data, budget, ring_data->ex_process);
 709
 710         if (clean_complete >= 0 && clean_complete < budget) {
 711                 napi_complete(napi);
 712                 ring_data->ring->q->handle->dev->ops->toggle_ring_irq(
 713                         ring_data->ring, 0);
 714
 715                 ring_data->fini_process(ring_data);
 716         }
 717
 718         return clean_complete;
 719 }
 720
 721 static irqreturn_t hns_irq_handle(int irq, void *dev)
 722 {
 723         struct hns_nic_ring_data *ring_data = (struct hns_nic_ring_data *)dev;
 724
 725         ring_data->ring->q->handle->dev->ops->toggle_ring_irq(
 726                 ring_data->ring, 1);
 727         napi_schedule(&ring_data->napi);
 728
 729         return IRQ_HANDLED;
 730 }
 731
 732 /**
 733  *hns_nic_adjust_link - adjust net work mode by the phy stat or new param
 734  *@ndev: net device
 735  */
 736 static void hns_nic_adjust_link(struct net_device *ndev)
 737 {
 738         struct hns_nic_priv *priv = netdev_priv(ndev);
 739         struct hnae_handle *h = priv->ae_handle;
 740
 741         h->dev->ops->adjust_link(h, ndev->phydev->speed, ndev->phydev->duplex);
 742 }
 743
 744 /**
 745  *hns_nic_init_phy - init phy
 746  *@ndev: net device
 747  *@h: ae handle
 748  * Return 0 on success, negative on failure
 749  */
 750 int hns_nic_init_phy(struct net_device *ndev, struct hnae_handle *h)
 751 {
 752         struct hns_nic_priv *priv = netdev_priv(ndev);
 753         struct phy_device *phy_dev = NULL;
 754
 755         if (!h->phy_node)
 756                 return 0;
 757
 758         if (h->phy_if != PHY_INTERFACE_MODE_XGMII)
 759                 phy_dev = of_phy_connect(ndev, h->phy_node,
 760                                          hns_nic_adjust_link, 0, h->phy_if);
 761         else
 762                 phy_dev = of_phy_attach(ndev, h->phy_node, 0, h->phy_if);
 763
 764         if (unlikely(!phy_dev) || IS_ERR(phy_dev))
 765                 return !phy_dev ? -ENODEV : PTR_ERR(phy_dev);
 766
 767         phy_dev->supported &= h->if_support;
 768         phy_dev->advertising = phy_dev->supported;
 769
 770         if (h->phy_if == PHY_INTERFACE_MODE_XGMII)
 771                 phy_dev->autoneg = false;
 772
 773         priv->phy = phy_dev;
 774
 775         return 0;
 776 }
 777
 778 static int hns_nic_ring_open(struct net_device *netdev, int idx)
 779 {
 780         struct hns_nic_priv *priv = netdev_priv(netdev);
 781         struct hnae_handle *h = priv->ae_handle;
 782
 783         napi_enable(&priv->ring_data[idx].napi);
 784
 785         enable_irq(priv->ring_data[idx].ring->irq);
 786         h->dev->ops->toggle_ring_irq(priv->ring_data[idx].ring, 0);
 787
 788         return 0;
 789 }
 790
 791 static int hns_nic_net_set_mac_address(struct net_device *ndev, void *p)
 792 {
 793         struct hns_nic_priv *priv = netdev_priv(ndev);
 794         struct hnae_handle *h = priv->ae_handle;
 795         struct sockaddr *mac_addr = p;
 796         int ret;
 797
 798         if (!mac_addr || !is_valid_ether_addr((const u8 *)mac_addr->sa_data))
 799                 return -EADDRNOTAVAIL;
 800
 801         ret = h->dev->ops->set_mac_addr(h, mac_addr->sa_data);
 802         if (ret) {
 803                 netdev_err(ndev, "set_mac_address fail, ret=%d!\n", ret);
 804                 return ret;
 805         }
 806
 807         memcpy(ndev->dev_addr, mac_addr->sa_data, ndev->addr_len);
 808
 809         return 0;
 810 }
 811
 812 void hns_nic_update_stats(struct net_device *netdev)
 813 {
 814         struct hns_nic_priv *priv = netdev_priv(netdev);
 815         struct hnae_handle *h = priv->ae_handle;
 816
 817         h->dev->ops->update_stats(h, &netdev->stats);
 818 }
 819
 820 /* set mac addr if it is configed. or leave it to the AE driver */
 821 static void hns_init_mac_addr(struct net_device *ndev)
 822 {
 823         struct hns_nic_priv *priv = netdev_priv(ndev);
 824         struct device_node *node = priv->dev->of_node;
 825         const void *mac_addr_temp;
 826
 827         mac_addr_temp = of_get_mac_address(node);
 828         if (mac_addr_temp && is_valid_ether_addr(mac_addr_temp)) {
 829                 memcpy(ndev->dev_addr, mac_addr_temp, ndev->addr_len);
 830         } else {
 831                 eth_hw_addr_random(ndev);
 832                 dev_warn(priv->dev, "No valid mac, use random mac %pM",
 833                          ndev->dev_addr);
 834         }
 835 }
 836
 837 static void hns_nic_ring_close(struct net_device *netdev, int idx)
 838 {
 839         struct hns_nic_priv *priv = netdev_priv(netdev);
 840         struct hnae_handle *h = priv->ae_handle;
 841
 842         h->dev->ops->toggle_ring_irq(priv->ring_data[idx].ring, 1);
 843         disable_irq(priv->ring_data[idx].ring->irq);
 844
 845         napi_disable(&priv->ring_data[idx].napi);
 846 }
 847
 848 static int hns_nic_init_irq(struct hns_nic_priv *priv)
 849 {
 850         struct hnae_handle *h = priv->ae_handle;
 851         struct hns_nic_ring_data *rd;
 852         int i;
 853         int ret;
 854         int cpu;
 855         cpumask_t mask;
 856
 857         for (i = 0; i < h->q_num * 2; i++) {
 858                 rd = &priv->ring_data[i];
 859
 860                 if (rd->ring->irq_init_flag == RCB_IRQ_INITED)
 861                         break;
 862
 863                 snprintf(rd->ring->ring_name, RCB_RING_NAME_LEN,
 864                          "%s-%s%d", priv->netdev->name,
 865                          (i < h->q_num ? "tx" : "rx"), rd->queue_index);
 866
 867                 rd->ring->ring_name[RCB_RING_NAME_LEN - 1] = '\0';
 868
 869                 ret = request_irq(rd->ring->irq,
 870                                   hns_irq_handle, 0, rd->ring->ring_name, rd);
 871                 if (ret) {
 872                         netdev_err(priv->netdev, "request irq(%d) fail\n",
 873                                    rd->ring->irq);
 874                         return ret;
 875                 }
 876                 disable_irq(rd->ring->irq);
 877                 rd->ring->irq_init_flag = RCB_IRQ_INITED;
 878
 879                 /*set cpu affinity*/
 880                 if (cpu_online(rd->queue_index)) {
 881                         cpumask_clear(&mask);
 882                         cpu = rd->queue_index;
 883                         cpumask_set_cpu(cpu, &mask);
 884                         irq_set_affinity_hint(rd->ring->irq, &mask);
 885                 }
 886         }
 887
 888         return 0;
 889 }
 890
 891 static int hns_nic_net_up(struct net_device *ndev)
 892 {
 893         struct hns_nic_priv *priv = netdev_priv(ndev);
 894         struct hnae_handle *h = priv->ae_handle;
 895         int i, j, k;
 896         int ret;
 897
 898         ret = hns_nic_init_irq(priv);
 899         if (ret != 0) {
 900                 netdev_err(ndev, "hns init irq failed! ret=%d\n", ret);
 901                 return ret;
 902         }
 903
 904         for (i = 0; i < h->q_num * 2; i++) {
 905                 ret = hns_nic_ring_open(ndev, i);
 906                 if (ret)
 907                         goto out_has_some_queues;
 908         }
 909
 910         for (k = 0; k < h->q_num; k++)
 911                 h->dev->ops->toggle_queue_status(h->qs[k], 1);
 912
 913         ret = h->dev->ops->set_mac_addr(h, ndev->dev_addr);
 914         if (ret)
 915                 goto out_set_mac_addr_err;
 916
 917         ret = h->dev->ops->start ? h->dev->ops->start(h) : 0;
 918         if (ret)
 919                 goto out_start_err;
 920
 921         if (priv->phy)
 922                 phy_start(priv->phy);
 923
 924         clear_bit(NIC_STATE_DOWN, &priv->state);
 925         (void)mod_timer(&priv->service_timer, jiffies + SERVICE_TIMER_HZ);
 926
 927         return 0;
 928
 929 out_start_err:
 930         netif_stop_queue(ndev);
 931 out_set_mac_addr_err:
 932         for (k = 0; k < h->q_num; k++)
 933                 h->dev->ops->toggle_queue_status(h->qs[k], 0);
 934 out_has_some_queues:
 935         for (j = i - 1; j >= 0; j--)
 936                 hns_nic_ring_close(ndev, j);
 937
 938         set_bit(NIC_STATE_DOWN, &priv->state);
 939
 940         return ret;
 941 }
 942
 943 static void hns_nic_net_down(struct net_device *ndev)
 944 {
 945         int i;
 946         struct hnae_ae_ops *ops;
 947         struct hns_nic_priv *priv = netdev_priv(ndev);
 948
 949         if (test_and_set_bit(NIC_STATE_DOWN, &priv->state))
 950                 return;
 951
 952         (void)del_timer_sync(&priv->service_timer);
 953         netif_tx_stop_all_queues(ndev);
 954         netif_carrier_off(ndev);
 955         netif_tx_disable(ndev);
 956         priv->link = 0;
 957
 958         if (priv->phy)
 959                 phy_stop(priv->phy);
 960
 961         ops = priv->ae_handle->dev->ops;
 962
 963         if (ops->stop)
 964                 ops->stop(priv->ae_handle);
 965
 966         netif_tx_stop_all_queues(ndev);
 967
 968         for (i = priv->ae_handle->q_num - 1; i >= 0; i--) {
 969                 hns_nic_ring_close(ndev, i);
 970                 hns_nic_ring_close(ndev, i + priv->ae_handle->q_num);
 971
 972                 /* clean tx buffers*/
 973                 hns_nic_tx_clr_all_bufs(priv->ring_data + i);
 974         }
 975 }
 976
 977 void hns_nic_net_reset(struct net_device *ndev)
 978 {
 979         struct hns_nic_priv *priv = netdev_priv(ndev);
 980         struct hnae_handle *handle = priv->ae_handle;
 981
 982         while (test_and_set_bit(NIC_STATE_RESETTING, &priv->state))
 983                 usleep_range(1000, 2000);
 984
 985         (void)hnae_reinit_handle(handle);
 986
 987         clear_bit(NIC_STATE_RESETTING, &priv->state);
 988 }
 989
 990 void hns_nic_net_reinit(struct net_device *netdev)
 991 {
 992         struct hns_nic_priv *priv = netdev_priv(netdev);
 993
 994         priv->netdev->trans_start = jiffies;
 995         while (test_and_set_bit(NIC_STATE_REINITING, &priv->state))
 996                 usleep_range(1000, 2000);
 997
 998         hns_nic_net_down(netdev);
 999         hns_nic_net_reset(netdev);
1000         (void)hns_nic_net_up(netdev);
1001         clear_bit(NIC_STATE_REINITING, &priv->state);
1002 }
1003
1004 static int hns_nic_net_open(struct net_device *ndev)
1005 {
1006         struct hns_nic_priv *priv = netdev_priv(ndev);
1007         struct hnae_handle *h = priv->ae_handle;
1008         int ret;
1009
1010         if (test_bit(NIC_STATE_TESTING, &priv->state))
1011                 return -EBUSY;
1012
1013         priv->link = 0;
1014         netif_carrier_off(ndev);
1015
1016         ret = netif_set_real_num_tx_queues(ndev, h->q_num);
1017         if (ret < 0) {
1018                 netdev_err(ndev, "netif_set_real_num_tx_queues fail, ret=%d!\n",
1019                            ret);
1020                 return ret;
1021         }
1022
1023         ret = netif_set_real_num_rx_queues(ndev, h->q_num);
1024         if (ret < 0) {
1025                 netdev_err(ndev,
1026                            "netif_set_real_num_rx_queues fail, ret=%d!\n", ret);
1027                 return ret;
1028         }
1029
1030         ret = hns_nic_net_up(ndev);
1031         if (ret) {
1032                 netdev_err(ndev,
1033                            "hns net up fail, ret=%d!\n", ret);
1034                 return ret;
1035         }
1036
1037         return 0;
1038 }
1039
1040 static int hns_nic_net_stop(struct net_device *ndev)
1041 {
1042         hns_nic_net_down(ndev);
1043
1044         return 0;
1045 }
1046
1047 static void hns_tx_timeout_reset(struct hns_nic_priv *priv);
1048 static void hns_nic_net_timeout(struct net_device *ndev)
1049 {
1050         struct hns_nic_priv *priv = netdev_priv(ndev);
1051
1052         hns_tx_timeout_reset(priv);
1053 }
1054
1055 static int hns_nic_do_ioctl(struct net_device *netdev, struct ifreq *ifr,
1056                             int cmd)
1057 {
1058         struct hns_nic_priv *priv = netdev_priv(netdev);
1059         struct phy_device *phy_dev = priv->phy;
1060
1061         if (!netif_running(netdev))
1062                 return -EINVAL;
1063
1064         if (!phy_dev)
1065                 return -ENOTSUPP;
1066
1067         return phy_mii_ioctl(phy_dev, ifr, cmd);
1068 }
1069
1070 /* use only for netconsole to poll with the device without interrupt */
1071 #ifdef CONFIG_NET_POLL_CONTROLLER
1072 void hns_nic_poll_controller(struct net_device *ndev)
1073 {
1074         struct hns_nic_priv *priv = netdev_priv(ndev);
1075         unsigned long flags;
1076         int i;
1077
1078         local_irq_save(flags);
1079         for (i = 0; i < priv->ae_handle->q_num * 2; i++)
1080                 napi_schedule(&priv->ring_data[i].napi);
1081         local_irq_restore(flags);
1082 }
1083 #endif
1084
1085 static netdev_tx_t hns_nic_net_xmit(struct sk_buff *skb,
1086                                     struct net_device *ndev)
1087 {
1088         struct hns_nic_priv *priv = netdev_priv(ndev);
1089         int ret;
1090
1091         assert(skb->queue_mapping < ndev->ae_handle->q_num);
1092         ret = hns_nic_net_xmit_hw(ndev, skb,
1093                                   &tx_ring_data(priv, skb->queue_mapping));
1094         if (ret == NETDEV_TX_OK) {
1095                 ndev->trans_start = jiffies;
1096                 ndev->stats.tx_bytes += skb->len;
1097                 ndev->stats.tx_packets++;
1098         }
1099         return (netdev_tx_t)ret;
1100 }
1101
1102 static int hns_nic_change_mtu(struct net_device *ndev, int new_mtu)
1103 {
1104         struct hns_nic_priv *priv = netdev_priv(ndev);
1105         struct hnae_handle *h = priv->ae_handle;
1106         int ret;
1107
1108         /* MTU < 68 is an error and causes problems on some kernels */
1109         if (new_mtu < 68)
1110                 return -EINVAL;
1111
1112         if (!h->dev->ops->set_mtu)
1113                 return -ENOTSUPP;
1114
1115         if (netif_running(ndev)) {
1116                 (void)hns_nic_net_stop(ndev);
1117                 msleep(100);
1118
1119                 ret = h->dev->ops->set_mtu(h, new_mtu);
1120                 if (ret)
1121                         netdev_err(ndev, "set mtu fail, return value %d\n",
1122                                    ret);
1123
1124                 if (hns_nic_net_open(ndev))
1125                         netdev_err(ndev, "hns net open fail\n");
1126         } else {
1127                 ret = h->dev->ops->set_mtu(h, new_mtu);
1128         }
1129
1130         if (!ret)
1131                 ndev->mtu = new_mtu;
1132
1133         return ret;
1134 }
1135
1136 /**
1137  * nic_set_multicast_list - set mutl mac address
1138  * @netdev: net device
1139  * @p: mac address
1140  *
1141  * return void
1142  */
1143 void hns_set_multicast_list(struct net_device *ndev)
1144 {
1145         struct hns_nic_priv *priv = netdev_priv(ndev);
1146         struct hnae_handle *h = priv->ae_handle;
1147         struct netdev_hw_addr *ha = NULL;
1148
1149         if (!h) {
1150                 netdev_err(ndev, "hnae handle is null\n");
1151                 return;
1152         }
1153
1154         if (h->dev->ops->set_mc_addr) {
1155                 netdev_for_each_mc_addr(ha, ndev)
1156                         if (h->dev->ops->set_mc_addr(h, ha->addr))
1157                                 netdev_err(ndev, "set multicast fail\n");
1158         }
1159 }
1160
1161 void hns_nic_set_rx_mode(struct net_device *ndev)
1162 {
1163         struct hns_nic_priv *priv = netdev_priv(ndev);
1164         struct hnae_handle *h = priv->ae_handle;
1165
1166         if (h->dev->ops->set_promisc_mode) {
1167                 if (ndev->flags & IFF_PROMISC)
1168                         h->dev->ops->set_promisc_mode(h, 1);
1169                 else
1170                         h->dev->ops->set_promisc_mode(h, 0);
1171         }
1172
1173         hns_set_multicast_list(ndev);
1174 }
1175
1176 struct rtnl_link_stats64 *hns_nic_get_stats64(struct net_device *ndev,
1177                                               struct rtnl_link_stats64 *stats)
1178 {
1179         int idx = 0;
1180         u64 tx_bytes = 0;
1181         u64 rx_bytes = 0;
1182         u64 tx_pkts = 0;
1183         u64 rx_pkts = 0;
1184         struct hns_nic_priv *priv = netdev_priv(ndev);
1185         struct hnae_handle *h = priv->ae_handle;
1186
1187         for (idx = 0; idx < h->q_num; idx++) {
1188                 tx_bytes += h->qs[idx]->tx_ring.stats.tx_bytes;
1189                 tx_pkts += h->qs[idx]->tx_ring.stats.tx_pkts;
1190                 rx_bytes += h->qs[idx]->rx_ring.stats.rx_bytes;
1191                 rx_pkts += h->qs[idx]->rx_ring.stats.rx_pkts;
1192         }
1193
1194         stats->tx_bytes = tx_bytes;
1195         stats->tx_packets = tx_pkts;
1196         stats->rx_bytes = rx_bytes;
1197         stats->rx_packets = rx_pkts;
1198
1199         stats->rx_errors = ndev->stats.rx_errors;
1200         stats->multicast = ndev->stats.multicast;
1201         stats->rx_length_errors = ndev->stats.rx_length_errors;
1202         stats->rx_crc_errors = ndev->stats.rx_crc_errors;
1203         stats->rx_missed_errors = ndev->stats.rx_missed_errors;
1204
1205         stats->tx_errors = ndev->stats.tx_errors;
1206         stats->rx_dropped = ndev->stats.rx_dropped;
1207         stats->tx_dropped = ndev->stats.tx_dropped;
1208         stats->collisions = ndev->stats.collisions;
1209         stats->rx_over_errors = ndev->stats.rx_over_errors;
1210         stats->rx_frame_errors = ndev->stats.rx_frame_errors;
1211         stats->rx_fifo_errors = ndev->stats.rx_fifo_errors;
1212         stats->tx_aborted_errors = ndev->stats.tx_aborted_errors;
1213         stats->tx_carrier_errors = ndev->stats.tx_carrier_errors;
1214         stats->tx_fifo_errors = ndev->stats.tx_fifo_errors;
1215         stats->tx_heartbeat_errors = ndev->stats.tx_heartbeat_errors;
1216         stats->tx_window_errors = ndev->stats.tx_window_errors;
1217         stats->rx_compressed = ndev->stats.rx_compressed;
1218         stats->tx_compressed = ndev->stats.tx_compressed;
1219
1220         return stats;
1221 }
1222
1223 static const struct net_device_ops hns_nic_netdev_ops = {
1224         .ndo_open = hns_nic_net_open,
1225         .ndo_stop = hns_nic_net_stop,
1226         .ndo_start_xmit = hns_nic_net_xmit,
1227         .ndo_tx_timeout = hns_nic_net_timeout,
1228         .ndo_set_mac_address = hns_nic_net_set_mac_address,
1229         .ndo_change_mtu = hns_nic_change_mtu,
1230         .ndo_do_ioctl = hns_nic_do_ioctl,
1231         .ndo_get_stats64 = hns_nic_get_stats64,
1232 #ifdef CONFIG_NET_POLL_CONTROLLER
1233         .ndo_poll_controller = hns_nic_poll_controller,
1234 #endif
1235         .ndo_set_rx_mode = hns_nic_set_rx_mode,
1236 };
1237
1238 static void hns_nic_update_link_status(struct net_device *netdev)
1239 {
1240         struct hns_nic_priv *priv = netdev_priv(netdev);
1241
1242         struct hnae_handle *h = priv->ae_handle;
1243         int state = 1;
1244
1245         if (priv->phy) {
1246                 if (!genphy_update_link(priv->phy))
1247                         state = priv->phy->link;
1248                 else
1249                         state = 0;
1250         }
1251         state = state && h->dev->ops->get_status(h);
1252
1253         if (state != priv->link) {
1254                 if (state) {
1255                         netif_carrier_on(netdev);
1256                         netif_tx_wake_all_queues(netdev);
1257                         netdev_info(netdev, "link up\n");
1258                 } else {
1259                         netif_carrier_off(netdev);
1260                         netdev_info(netdev, "link down\n");
1261                 }
1262                 priv->link = state;
1263         }
1264 }
1265
1266 /* for dumping key regs*/
1267 static void hns_nic_dump(struct hns_nic_priv *priv)
1268 {
1269         struct hnae_handle *h = priv->ae_handle;
1270         struct hnae_ae_ops *ops = h->dev->ops;
1271         u32 *data, reg_num, i;
1272
1273         if (ops->get_regs_len && ops->get_regs) {
1274                 reg_num = ops->get_regs_len(priv->ae_handle);
1275                 reg_num = (reg_num + 3ul) & ~3ul;
1276                 data = kcalloc(reg_num, sizeof(u32), GFP_KERNEL);
1277                 if (data) {
1278                         ops->get_regs(priv->ae_handle, data);
1279                         for (i = 0; i < reg_num; i += 4)
1280                                 pr_info("0x%08x: 0x%08x 0x%08x 0x%08x 0x%08x\n",
1281                                         i, data[i], data[i + 1],
1282                                         data[i + 2], data[i + 3]);
1283                         kfree(data);
1284                 }
1285         }
1286
1287         for (i = 0; i < h->q_num; i++) {
1288                 pr_info("tx_queue%d_next_to_clean:%d\n",
1289                         i, h->qs[i]->tx_ring.next_to_clean);
1290                 pr_info("tx_queue%d_next_to_use:%d\n",
1291                         i, h->qs[i]->tx_ring.next_to_use);
1292                 pr_info("rx_queue%d_next_to_clean:%d\n",
1293                         i, h->qs[i]->rx_ring.next_to_clean);
1294                 pr_info("rx_queue%d_next_to_use:%d\n",
1295                         i, h->qs[i]->rx_ring.next_to_use);
1296         }
1297 }
1298
1299 /* for resetting suntask*/
1300 static void hns_nic_reset_subtask(struct hns_nic_priv *priv)
1301 {
1302         enum hnae_port_type type = priv->ae_handle->port_type;
1303
1304         if (!test_bit(NIC_STATE2_RESET_REQUESTED, &priv->state))
1305                 return;
1306         clear_bit(NIC_STATE2_RESET_REQUESTED, &priv->state);
1307
1308         /* If we're already down, removing or resetting, just bail */
1309         if (test_bit(NIC_STATE_DOWN, &priv->state) ||
1310             test_bit(NIC_STATE_REMOVING, &priv->state) ||
1311             test_bit(NIC_STATE_RESETTING, &priv->state))
1312                 return;
1313
1314         hns_nic_dump(priv);
1315         netdev_info(priv->netdev, "Reset %s port\n",
1316                     (type == HNAE_PORT_DEBUG ? "debug" : "business"));
1317
1318         rtnl_lock();
1319         /* put off any impending NetWatchDogTimeout */
1320         priv->netdev->trans_start = jiffies;
1321
1322         if (type == HNAE_PORT_DEBUG)
1323                 hns_nic_net_reinit(priv->netdev);
1324         rtnl_unlock();
1325 }
1326
1327 /* for doing service complete*/
1328 static void hns_nic_service_event_complete(struct hns_nic_priv *priv)
1329 {
1330         assert(!test_bit(NIC_STATE_SERVICE_SCHED, &priv->state));
1331
1332         smp_mb__before_atomic();
1333         clear_bit(NIC_STATE_SERVICE_SCHED, &priv->state);
1334 }
1335
1336 static void hns_nic_service_task(struct work_struct *work)
1337 {
1338         struct hns_nic_priv *priv
1339                 = container_of(work, struct hns_nic_priv, service_task);
1340         struct hnae_handle *h = priv->ae_handle;
1341
1342         hns_nic_update_link_status(priv->netdev);
1343         h->dev->ops->update_led_status(h);
1344         hns_nic_update_stats(priv->netdev);
1345
1346         hns_nic_reset_subtask(priv);
1347         hns_nic_service_event_complete(priv);
1348 }
1349
1350 static void hns_nic_task_schedule(struct hns_nic_priv *priv)
1351 {
1352         if (!test_bit(NIC_STATE_DOWN, &priv->state) &&
1353             !test_bit(NIC_STATE_REMOVING, &priv->state) &&
1354             !test_and_set_bit(NIC_STATE_SERVICE_SCHED, &priv->state))
1355                 (void)schedule_work(&priv->service_task);
1356 }
1357
1358 static void hns_nic_service_timer(unsigned long data)
1359 {
1360         struct hns_nic_priv *priv = (struct hns_nic_priv *)data;
1361
1362         (void)mod_timer(&priv->service_timer, jiffies + SERVICE_TIMER_HZ);
1363
1364         hns_nic_task_schedule(priv);
1365 }
1366
1367 /**
1368  * hns_tx_timeout_reset - initiate reset due to Tx timeout
1369  * @priv: driver private struct
1370  **/
1371 static void hns_tx_timeout_reset(struct hns_nic_priv *priv)
1372 {
1373         /* Do the reset outside of interrupt context */
1374         if (!test_bit(NIC_STATE_DOWN, &priv->state)) {
1375                 set_bit(NIC_STATE2_RESET_REQUESTED, &priv->state);
1376                 netdev_warn(priv->netdev,
1377                             "initiating reset due to tx timeout(%llu,0x%lx)\n",
1378                             priv->tx_timeout_count, priv->state);
1379                 priv->tx_timeout_count++;
1380                 hns_nic_task_schedule(priv);
1381         }
1382 }
1383
1384 static int hns_nic_init_ring_data(struct hns_nic_priv *priv)
1385 {
1386         struct hnae_handle *h = priv->ae_handle;
1387         struct hns_nic_ring_data *rd;
1388         int i;
1389
1390         if (h->q_num > NIC_MAX_Q_PER_VF) {
1391                 netdev_err(priv->netdev, "too much queue (%d)\n", h->q_num);
1392                 return -EINVAL;
1393         }
1394
1395         priv->ring_data = kzalloc(h->q_num * sizeof(*priv->ring_data) * 2,
1396                                   GFP_KERNEL);
1397         if (!priv->ring_data)
1398                 return -ENOMEM;
1399
1400         for (i = 0; i < h->q_num; i++) {
1401                 rd = &priv->ring_data[i];
1402                 rd->queue_index = i;
1403                 rd->ring = &h->qs[i]->tx_ring;
1404                 rd->poll_one = hns_nic_tx_poll_one;
1405                 rd->fini_process = hns_nic_tx_fini_pro;
1406
1407                 netif_napi_add(priv->netdev, &rd->napi,
1408                                hns_nic_common_poll, NAPI_POLL_WEIGHT);
1409                 rd->ring->irq_init_flag = RCB_IRQ_NOT_INITED;
1410         }
1411         for (i = h->q_num; i < h->q_num * 2; i++) {
1412                 rd = &priv->ring_data[i];
1413                 rd->queue_index = i - h->q_num;
1414                 rd->ring = &h->qs[i - h->q_num]->rx_ring;
1415                 rd->poll_one = hns_nic_rx_poll_one;
1416                 rd->ex_process = hns_nic_rx_up_pro;
1417                 rd->fini_process = hns_nic_rx_fini_pro;
1418
1419                 netif_napi_add(priv->netdev, &rd->napi,
1420                                hns_nic_common_poll, NAPI_POLL_WEIGHT);
1421                 rd->ring->irq_init_flag = RCB_IRQ_NOT_INITED;
1422         }
1423
1424         return 0;
1425 }
1426
1427 static void hns_nic_uninit_ring_data(struct hns_nic_priv *priv)
1428 {
1429         struct hnae_handle *h = priv->ae_handle;
1430         int i;
1431
1432         for (i = 0; i < h->q_num * 2; i++) {
1433                 netif_napi_del(&priv->ring_data[i].napi);
1434                 if (priv->ring_data[i].ring->irq_init_flag == RCB_IRQ_INITED) {
1435                         irq_set_affinity_hint(priv->ring_data[i].ring->irq,
1436                                               NULL);
1437                         free_irq(priv->ring_data[i].ring->irq,
1438                                  &priv->ring_data[i]);
1439                 }
1440
1441                 priv->ring_data[i].ring->irq_init_flag = RCB_IRQ_NOT_INITED;
1442         }
1443         kfree(priv->ring_data);
1444 }
1445
1446 static int hns_nic_try_get_ae(struct net_device *ndev)
1447 {
1448         struct hns_nic_priv *priv = netdev_priv(ndev);
1449         struct hnae_handle *h;
1450         int ret;
1451
1452         h = hnae_get_handle(&priv->netdev->dev,
1453                             priv->ae_name, priv->port_id, NULL);
1454         if (IS_ERR_OR_NULL(h)) {
1455                 ret = PTR_ERR(h);
1456                 dev_dbg(priv->dev, "has not handle, register notifier!\n");
1457                 goto out;
1458         }
1459         priv->ae_handle = h;
1460
1461         ret = hns_nic_init_phy(ndev, h);
1462         if (ret) {
1463                 dev_err(priv->dev, "probe phy device fail!\n");
1464                 goto out_init_phy;
1465         }
1466
1467         ret = hns_nic_init_ring_data(priv);
1468         if (ret) {
1469                 ret = -ENOMEM;
1470                 goto out_init_ring_data;
1471         }
1472
1473         ret = register_netdev(ndev);
1474         if (ret) {
1475                 dev_err(priv->dev, "probe register netdev fail!\n");
1476                 goto out_reg_ndev_fail;
1477         }
1478         return 0;
1479
1480 out_reg_ndev_fail:
1481         hns_nic_uninit_ring_data(priv);
1482         priv->ring_data = NULL;
1483 out_init_phy:
1484 out_init_ring_data:
1485         hnae_put_handle(priv->ae_handle);
1486         priv->ae_handle = NULL;
1487 out:
1488         return ret;
1489 }
1490
1491 static int hns_nic_notifier_action(struct notifier_block *nb,
1492                                    unsigned long action, void *data)
1493 {
1494         struct hns_nic_priv *priv =
1495                 container_of(nb, struct hns_nic_priv, notifier_block);
1496
1497         assert(action == HNAE_AE_REGISTER);
1498
1499         if (!hns_nic_try_get_ae(priv->netdev)) {
1500                 hnae_unregister_notifier(&priv->notifier_block);
1501                 priv->notifier_block.notifier_call = NULL;
1502         }
1503         return 0;
1504 }
1505
1506 static int hns_nic_dev_probe(struct platform_device *pdev)
1507 {
1508         struct device *dev = &pdev->dev;
1509         struct net_device *ndev;
1510         struct hns_nic_priv *priv;
1511         struct device_node *node = dev->of_node;
1512         int ret;
1513
1514         ndev = alloc_etherdev_mq(sizeof(struct hns_nic_priv), NIC_MAX_Q_PER_VF);
1515         if (!ndev)
1516                 return -ENOMEM;
1517
1518         platform_set_drvdata(pdev, ndev);
1519
1520         priv = netdev_priv(ndev);
1521         priv->dev = dev;
1522         priv->netdev = ndev;
1523
1524         if (of_device_is_compatible(node, "hisilicon,hns-nic-v2"))
1525                 priv->enet_ver = AE_VERSION_2;
1526         else
1527                 priv->enet_ver = AE_VERSION_1;
1528
1529         ret = of_property_read_string(node, "ae-name", &priv->ae_name);
1530         if (ret)
1531                 goto out_read_string_fail;
1532
1533         ret = of_property_read_u32(node, "port-id", &priv->port_id);
1534         if (ret)
1535                 goto out_read_string_fail;
1536
1537         hns_init_mac_addr(ndev);
1538
1539         ndev->watchdog_timeo = HNS_NIC_TX_TIMEOUT;
1540         ndev->priv_flags |= IFF_UNICAST_FLT;
1541         ndev->netdev_ops = &hns_nic_netdev_ops;
1542         hns_ethtool_set_ops(ndev);
1543         ndev->features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
1544                 NETIF_F_RXCSUM | NETIF_F_SG | NETIF_F_GSO |
1545                 NETIF_F_GRO;
1546         ndev->vlan_features |=
1547                 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_RXCSUM;
1548         ndev->vlan_features |= NETIF_F_SG | NETIF_F_GSO | NETIF_F_GRO;
1549
1550         SET_NETDEV_DEV(ndev, dev);
1551
1552         if (!dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64)))
1553                 dev_dbg(dev, "set mask to 64bit\n");
1554         else
1555                 dev_err(dev, "set mask to 32bit fail!\n");
1556
1557         /* carrier off reporting is important to ethtool even BEFORE open */
1558         netif_carrier_off(ndev);
1559
1560         setup_timer(&priv->service_timer, hns_nic_service_timer,
1561                     (unsigned long)priv);
1562         INIT_WORK(&priv->service_task, hns_nic_service_task);
1563
1564         set_bit(NIC_STATE_SERVICE_INITED, &priv->state);
1565         clear_bit(NIC_STATE_SERVICE_SCHED, &priv->state);
1566         set_bit(NIC_STATE_DOWN, &priv->state);
1567
1568         if (hns_nic_try_get_ae(priv->netdev)) {
1569                 priv->notifier_block.notifier_call = hns_nic_notifier_action;
1570                 ret = hnae_register_notifier(&priv->notifier_block);
1571                 if (ret) {
1572                         dev_err(dev, "register notifier fail!\n");
1573                         goto out_notify_fail;
1574                 }
1575                 dev_dbg(dev, "has not handle, register notifier!\n");
1576         }
1577
1578         return 0;
1579
1580 out_notify_fail:
1581         (void)cancel_work_sync(&priv->service_task);
1582 out_read_string_fail:
1583         free_netdev(ndev);
1584         return ret;
1585 }
1586
1587 static int hns_nic_dev_remove(struct platform_device *pdev)
1588 {
1589         struct net_device *ndev = platform_get_drvdata(pdev);
1590         struct hns_nic_priv *priv = netdev_priv(ndev);
1591
1592         if (ndev->reg_state != NETREG_UNINITIALIZED)
1593                 unregister_netdev(ndev);
1594
1595         if (priv->ring_data)
1596                 hns_nic_uninit_ring_data(priv);
1597         priv->ring_data = NULL;
1598
1599         if (priv->phy)
1600                 phy_disconnect(priv->phy);
1601         priv->phy = NULL;
1602
1603         if (!IS_ERR_OR_NULL(priv->ae_handle))
1604                 hnae_put_handle(priv->ae_handle);
1605         priv->ae_handle = NULL;
1606         if (priv->notifier_block.notifier_call)
1607                 hnae_unregister_notifier(&priv->notifier_block);
1608         priv->notifier_block.notifier_call = NULL;
1609
1610         set_bit(NIC_STATE_REMOVING, &priv->state);
1611         (void)cancel_work_sync(&priv->service_task);
1612
1613         free_netdev(ndev);
1614         return 0;
1615 }
1616
1617 static const struct of_device_id hns_enet_of_match[] = {
1618         {.compatible = "hisilicon,hns-nic-v1",},
1619         {.compatible = "hisilicon,hns-nic-v2",},
1620         {},
1621 };
1622
1623 MODULE_DEVICE_TABLE(of, hns_enet_of_match);
1624
1625 static struct platform_driver hns_nic_dev_driver = {
1626         .driver = {
1627                 .name = "hns-nic",
1628                 .of_match_table = hns_enet_of_match,
1629         },
1630         .probe = hns_nic_dev_probe,
1631         .remove = hns_nic_dev_remove,
1632 };
1633
1634 module_platform_driver(hns_nic_dev_driver);
1635
1636 MODULE_DESCRIPTION("HISILICON HNS Ethernet driver");
1637 MODULE_AUTHOR("Hisilicon, Inc.");
1638 MODULE_LICENSE("GPL");
1639 MODULE_ALIAS("platform:hns-nic");