drivers/net/ethernet/qualcomm/emac/emac-mac.c

   1 /* Copyright (c) 2013-2016, The Linux Foundation. All rights reserved.
   2  *
   3  * This program is free software; you can redistribute it and/or modify
   4  * it under the terms of the GNU General Public License version 2 and
   5  * only version 2 as published by the Free Software Foundation.
   6  *
   7  * This program is distributed in the hope that it will be useful,
   8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
   9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  10  * GNU General Public License for more details.
  11  */
  12
  13 /* Qualcomm Technologies, Inc. EMAC Ethernet Controller MAC layer support
  14  */
  15
  16 #include <linux/tcp.h>
  17 #include <linux/ip.h>
  18 #include <linux/ipv6.h>
  19 #include <linux/crc32.h>
  20 #include <linux/if_vlan.h>
  21 #include <linux/jiffies.h>
  22 #include <linux/phy.h>
  23 #include <linux/of.h>
  24 #include <net/ip6_checksum.h>
  25 #include "emac.h"
  26 #include "emac-sgmii.h"
  27
  28 /* EMAC base register offsets */
  29 #define EMAC_MAC_CTRL                   0x001480
  30 #define EMAC_WOL_CTRL0                  0x0014a0
  31 #define EMAC_RSS_KEY0                   0x0014b0
  32 #define EMAC_H1TPD_BASE_ADDR_LO         0x0014e0
  33 #define EMAC_H2TPD_BASE_ADDR_LO         0x0014e4
  34 #define EMAC_H3TPD_BASE_ADDR_LO         0x0014e8
  35 #define EMAC_INTER_SRAM_PART9           0x001534
  36 #define EMAC_DESC_CTRL_0                0x001540
  37 #define EMAC_DESC_CTRL_1                0x001544
  38 #define EMAC_DESC_CTRL_2                0x001550
  39 #define EMAC_DESC_CTRL_10               0x001554
  40 #define EMAC_DESC_CTRL_12               0x001558
  41 #define EMAC_DESC_CTRL_13               0x00155c
  42 #define EMAC_DESC_CTRL_3                0x001560
  43 #define EMAC_DESC_CTRL_4                0x001564
  44 #define EMAC_DESC_CTRL_5                0x001568
  45 #define EMAC_DESC_CTRL_14               0x00156c
  46 #define EMAC_DESC_CTRL_15               0x001570
  47 #define EMAC_DESC_CTRL_16               0x001574
  48 #define EMAC_DESC_CTRL_6                0x001578
  49 #define EMAC_DESC_CTRL_8                0x001580
  50 #define EMAC_DESC_CTRL_9                0x001584
  51 #define EMAC_DESC_CTRL_11               0x001588
  52 #define EMAC_TXQ_CTRL_0                 0x001590
  53 #define EMAC_TXQ_CTRL_1                 0x001594
  54 #define EMAC_TXQ_CTRL_2                 0x001598
  55 #define EMAC_RXQ_CTRL_0                 0x0015a0
  56 #define EMAC_RXQ_CTRL_1                 0x0015a4
  57 #define EMAC_RXQ_CTRL_2                 0x0015a8
  58 #define EMAC_RXQ_CTRL_3                 0x0015ac
  59 #define EMAC_BASE_CPU_NUMBER            0x0015b8
  60 #define EMAC_DMA_CTRL                   0x0015c0
  61 #define EMAC_MAILBOX_0                  0x0015e0
  62 #define EMAC_MAILBOX_5                  0x0015e4
  63 #define EMAC_MAILBOX_6                  0x0015e8
  64 #define EMAC_MAILBOX_13                 0x0015ec
  65 #define EMAC_MAILBOX_2                  0x0015f4
  66 #define EMAC_MAILBOX_3                  0x0015f8
  67 #define EMAC_MAILBOX_11                 0x00160c
  68 #define EMAC_AXI_MAST_CTRL              0x001610
  69 #define EMAC_MAILBOX_12                 0x001614
  70 #define EMAC_MAILBOX_9                  0x001618
  71 #define EMAC_MAILBOX_10                 0x00161c
  72 #define EMAC_ATHR_HEADER_CTRL           0x001620
  73 #define EMAC_CLK_GATE_CTRL              0x001814
  74 #define EMAC_MISC_CTRL                  0x001990
  75 #define EMAC_MAILBOX_7                  0x0019e0
  76 #define EMAC_MAILBOX_8                  0x0019e4
  77 #define EMAC_MAILBOX_15                 0x001bd4
  78 #define EMAC_MAILBOX_16                 0x001bd8
  79
  80 /* EMAC_MAC_CTRL */
  81 #define SINGLE_PAUSE_MODE               0x10000000
  82 #define DEBUG_MODE                      0x08000000
  83 #define BROAD_EN                        0x04000000
  84 #define MULTI_ALL                       0x02000000
  85 #define RX_CHKSUM_EN                    0x01000000
  86 #define HUGE                            0x00800000
  87 #define SPEED(x)                        (((x) & 0x3) << 20)
  88 #define SPEED_MASK                      SPEED(0x3)
  89 #define SIMR                            0x00080000
  90 #define TPAUSE                          0x00010000
  91 #define PROM_MODE                       0x00008000
  92 #define VLAN_STRIP                      0x00004000
  93 #define PRLEN_BMSK                      0x00003c00
  94 #define PRLEN_SHFT                      10
  95 #define HUGEN                           0x00000200
  96 #define FLCHK                           0x00000100
  97 #define PCRCE                           0x00000080
  98 #define CRCE                            0x00000040
  99 #define FULLD                           0x00000020
 100 #define MAC_LP_EN                       0x00000010
 101 #define RXFC                            0x00000008
 102 #define TXFC                            0x00000004
 103 #define RXEN                            0x00000002
 104 #define TXEN                            0x00000001
 105
 106
 107 /* EMAC_WOL_CTRL0 */
 108 #define LK_CHG_PME                      0x20
 109 #define LK_CHG_EN                       0x10
 110 #define MG_FRAME_PME                    0x8
 111 #define MG_FRAME_EN                     0x4
 112 #define WK_FRAME_EN                     0x1
 113
 114 /* EMAC_DESC_CTRL_3 */
 115 #define RFD_RING_SIZE_BMSK                                       0xfff
 116
 117 /* EMAC_DESC_CTRL_4 */
 118 #define RX_BUFFER_SIZE_BMSK                                     0xffff
 119
 120 /* EMAC_DESC_CTRL_6 */
 121 #define RRD_RING_SIZE_BMSK                                       0xfff
 122
 123 /* EMAC_DESC_CTRL_9 */
 124 #define TPD_RING_SIZE_BMSK                                      0xffff
 125
 126 /* EMAC_TXQ_CTRL_0 */
 127 #define NUM_TXF_BURST_PREF_BMSK                             0xffff0000
 128 #define NUM_TXF_BURST_PREF_SHFT                                     16
 129 #define LS_8023_SP                                                0x80
 130 #define TXQ_MODE                                                  0x40
 131 #define TXQ_EN                                                    0x20
 132 #define IP_OP_SP                                                  0x10
 133 #define NUM_TPD_BURST_PREF_BMSK                                    0xf
 134 #define NUM_TPD_BURST_PREF_SHFT                                      0
 135
 136 /* EMAC_TXQ_CTRL_1 */
 137 #define JUMBO_TASK_OFFLOAD_THRESHOLD_BMSK                        0x7ff
 138
 139 /* EMAC_TXQ_CTRL_2 */
 140 #define TXF_HWM_BMSK                                         0xfff0000
 141 #define TXF_LWM_BMSK                                             0xfff
 142
 143 /* EMAC_RXQ_CTRL_0 */
 144 #define RXQ_EN                                                 BIT(31)
 145 #define CUT_THRU_EN                                            BIT(30)
 146 #define RSS_HASH_EN                                            BIT(29)
 147 #define NUM_RFD_BURST_PREF_BMSK                              0x3f00000
 148 #define NUM_RFD_BURST_PREF_SHFT                                     20
 149 #define IDT_TABLE_SIZE_BMSK                                    0x1ff00
 150 #define IDT_TABLE_SIZE_SHFT                                          8
 151 #define SP_IPV6                                                   0x80
 152
 153 /* EMAC_RXQ_CTRL_1 */
 154 #define JUMBO_1KAH_BMSK                                         0xf000
 155 #define JUMBO_1KAH_SHFT                                             12
 156 #define RFD_PREF_LOW_TH                                           0x10
 157 #define RFD_PREF_LOW_THRESHOLD_BMSK                              0xfc0
 158 #define RFD_PREF_LOW_THRESHOLD_SHFT                                  6
 159 #define RFD_PREF_UP_TH                                            0x10
 160 #define RFD_PREF_UP_THRESHOLD_BMSK                                0x3f
 161 #define RFD_PREF_UP_THRESHOLD_SHFT                                   0
 162
 163 /* EMAC_RXQ_CTRL_2 */
 164 #define RXF_DOF_THRESFHOLD                                       0x1a0
 165 #define RXF_DOF_THRESHOLD_BMSK                               0xfff0000
 166 #define RXF_DOF_THRESHOLD_SHFT                                      16
 167 #define RXF_UOF_THRESFHOLD                                        0xbe
 168 #define RXF_UOF_THRESHOLD_BMSK                                   0xfff
 169 #define RXF_UOF_THRESHOLD_SHFT                                       0
 170
 171 /* EMAC_RXQ_CTRL_3 */
 172 #define RXD_TIMER_BMSK                                      0xffff0000
 173 #define RXD_THRESHOLD_BMSK                                       0xfff
 174 #define RXD_THRESHOLD_SHFT                                           0
 175
 176 /* EMAC_DMA_CTRL */
 177 #define DMAW_DLY_CNT_BMSK                                      0xf0000
 178 #define DMAW_DLY_CNT_SHFT                                           16
 179 #define DMAR_DLY_CNT_BMSK                                       0xf800
 180 #define DMAR_DLY_CNT_SHFT                                           11
 181 #define DMAR_REQ_PRI                                             0x400
 182 #define REGWRBLEN_BMSK                                           0x380
 183 #define REGWRBLEN_SHFT                                               7
 184 #define REGRDBLEN_BMSK                                            0x70
 185 #define REGRDBLEN_SHFT                                               4
 186 #define OUT_ORDER_MODE                                             0x4
 187 #define ENH_ORDER_MODE                                             0x2
 188 #define IN_ORDER_MODE                                              0x1
 189
 190 /* EMAC_MAILBOX_13 */
 191 #define RFD3_PROC_IDX_BMSK                                   0xfff0000
 192 #define RFD3_PROC_IDX_SHFT                                          16
 193 #define RFD3_PROD_IDX_BMSK                                       0xfff
 194 #define RFD3_PROD_IDX_SHFT                                           0
 195
 196 /* EMAC_MAILBOX_2 */
 197 #define NTPD_CONS_IDX_BMSK                                  0xffff0000
 198 #define NTPD_CONS_IDX_SHFT                                          16
 199
 200 /* EMAC_MAILBOX_3 */
 201 #define RFD0_CONS_IDX_BMSK                                       0xfff
 202 #define RFD0_CONS_IDX_SHFT                                           0
 203
 204 /* EMAC_MAILBOX_11 */
 205 #define H3TPD_PROD_IDX_BMSK                                 0xffff0000
 206 #define H3TPD_PROD_IDX_SHFT                                         16
 207
 208 /* EMAC_AXI_MAST_CTRL */
 209 #define DATA_BYTE_SWAP                                             0x8
 210 #define MAX_BOUND                                                  0x2
 211 #define MAX_BTYPE                                                  0x1
 212
 213 /* EMAC_MAILBOX_12 */
 214 #define H3TPD_CONS_IDX_BMSK                                 0xffff0000
 215 #define H3TPD_CONS_IDX_SHFT                                         16
 216
 217 /* EMAC_MAILBOX_9 */
 218 #define H2TPD_PROD_IDX_BMSK                                     0xffff
 219 #define H2TPD_PROD_IDX_SHFT                                          0
 220
 221 /* EMAC_MAILBOX_10 */
 222 #define H1TPD_CONS_IDX_BMSK                                 0xffff0000
 223 #define H1TPD_CONS_IDX_SHFT                                         16
 224 #define H2TPD_CONS_IDX_BMSK                                     0xffff
 225 #define H2TPD_CONS_IDX_SHFT                                          0
 226
 227 /* EMAC_ATHR_HEADER_CTRL */
 228 #define HEADER_CNT_EN                                              0x2
 229 #define HEADER_ENABLE                                              0x1
 230
 231 /* EMAC_MAILBOX_0 */
 232 #define RFD0_PROC_IDX_BMSK                                   0xfff0000
 233 #define RFD0_PROC_IDX_SHFT                                          16
 234 #define RFD0_PROD_IDX_BMSK                                       0xfff
 235 #define RFD0_PROD_IDX_SHFT                                           0
 236
 237 /* EMAC_MAILBOX_5 */
 238 #define RFD1_PROC_IDX_BMSK                                   0xfff0000
 239 #define RFD1_PROC_IDX_SHFT                                          16
 240 #define RFD1_PROD_IDX_BMSK                                       0xfff
 241 #define RFD1_PROD_IDX_SHFT                                           0
 242
 243 /* EMAC_MISC_CTRL */
 244 #define RX_UNCPL_INT_EN                                            0x1
 245
 246 /* EMAC_MAILBOX_7 */
 247 #define RFD2_CONS_IDX_BMSK                                   0xfff0000
 248 #define RFD2_CONS_IDX_SHFT                                          16
 249 #define RFD1_CONS_IDX_BMSK                                       0xfff
 250 #define RFD1_CONS_IDX_SHFT                                           0
 251
 252 /* EMAC_MAILBOX_8 */
 253 #define RFD3_CONS_IDX_BMSK                                       0xfff
 254 #define RFD3_CONS_IDX_SHFT                                           0
 255
 256 /* EMAC_MAILBOX_15 */
 257 #define NTPD_PROD_IDX_BMSK                                      0xffff
 258 #define NTPD_PROD_IDX_SHFT                                           0
 259
 260 /* EMAC_MAILBOX_16 */
 261 #define H1TPD_PROD_IDX_BMSK                                     0xffff
 262 #define H1TPD_PROD_IDX_SHFT                                          0
 263
 264 #define RXQ0_RSS_HSTYP_IPV6_TCP_EN                                0x20
 265 #define RXQ0_RSS_HSTYP_IPV6_EN                                    0x10
 266 #define RXQ0_RSS_HSTYP_IPV4_TCP_EN                                 0x8
 267 #define RXQ0_RSS_HSTYP_IPV4_EN                                     0x4
 268
 269 /* EMAC_EMAC_WRAPPER_TX_TS_INX */
 270 #define EMAC_WRAPPER_TX_TS_EMPTY                               BIT(31)
 271 #define EMAC_WRAPPER_TX_TS_INX_BMSK                             0xffff
 272
 273 struct emac_skb_cb {
 274         u32           tpd_idx;
 275         unsigned long jiffies;
 276 };
 277
 278 #define EMAC_SKB_CB(skb)        ((struct emac_skb_cb *)(skb)->cb)
 279 #define EMAC_RSS_IDT_SIZE       256
 280 #define JUMBO_1KAH              0x4
 281 #define RXD_TH                  0x100
 282 #define EMAC_TPD_LAST_FRAGMENT  0x80000000
 283 #define EMAC_TPD_TSTAMP_SAVE    0x80000000
 284
 285 /* EMAC Errors in emac_rrd.word[3] */
 286 #define EMAC_RRD_L4F            BIT(14)
 287 #define EMAC_RRD_IPF            BIT(15)
 288 #define EMAC_RRD_CRC            BIT(21)
 289 #define EMAC_RRD_FAE            BIT(22)
 290 #define EMAC_RRD_TRN            BIT(23)
 291 #define EMAC_RRD_RNT            BIT(24)
 292 #define EMAC_RRD_INC            BIT(25)
 293 #define EMAC_RRD_FOV            BIT(29)
 294 #define EMAC_RRD_LEN            BIT(30)
 295
 296 /* Error bits that will result in a received frame being discarded */
 297 #define EMAC_RRD_ERROR (EMAC_RRD_IPF | EMAC_RRD_CRC | EMAC_RRD_FAE | \
 298                         EMAC_RRD_TRN | EMAC_RRD_RNT | EMAC_RRD_INC | \
 299                         EMAC_RRD_FOV | EMAC_RRD_LEN)
 300 #define EMAC_RRD_STATS_DW_IDX 3
 301
 302 #define EMAC_RRD(RXQ, SIZE, IDX)        ((RXQ)->rrd.v_addr + (SIZE * (IDX)))
 303 #define EMAC_RFD(RXQ, SIZE, IDX)        ((RXQ)->rfd.v_addr + (SIZE * (IDX)))
 304 #define EMAC_TPD(TXQ, SIZE, IDX)        ((TXQ)->tpd.v_addr + (SIZE * (IDX)))
 305
 306 #define GET_RFD_BUFFER(RXQ, IDX)        (&((RXQ)->rfd.rfbuff[(IDX)]))
 307 #define GET_TPD_BUFFER(RTQ, IDX)        (&((RTQ)->tpd.tpbuff[(IDX)]))
 308
 309 #define EMAC_TX_POLL_HWTXTSTAMP_THRESHOLD       8
 310
 311 #define ISR_RX_PKT      (\
 312         RX_PKT_INT0     |\
 313         RX_PKT_INT1     |\
 314         RX_PKT_INT2     |\
 315         RX_PKT_INT3)
 316
 317 #define EMAC_MAC_IRQ_RES                                        "core0"
 318
 319 void emac_mac_multicast_addr_set(struct emac_adapter *adpt, u8 *addr)
 320 {
 321         u32 crc32, bit, reg, mta;
 322
 323         /* Calculate the CRC of the MAC address */
 324         crc32 = ether_crc(ETH_ALEN, addr);
 325
 326         /* The HASH Table is an array of 2 32-bit registers. It is
 327          * treated like an array of 64 bits (BitArray[hash_value]).
 328          * Use the upper 6 bits of the above CRC as the hash value.
 329          */
 330         reg = (crc32 >> 31) & 0x1;
 331         bit = (crc32 >> 26) & 0x1F;
 332
 333         mta = readl(adpt->base + EMAC_HASH_TAB_REG0 + (reg << 2));
 334         mta |= BIT(bit);
 335         writel(mta, adpt->base + EMAC_HASH_TAB_REG0 + (reg << 2));
 336 }
 337
 338 void emac_mac_multicast_addr_clear(struct emac_adapter *adpt)
 339 {
 340         writel(0, adpt->base + EMAC_HASH_TAB_REG0);
 341         writel(0, adpt->base + EMAC_HASH_TAB_REG1);
 342 }
 343
 344 /* definitions for RSS */
 345 #define EMAC_RSS_KEY(_i, _type) \
 346                 (EMAC_RSS_KEY0 + ((_i) * sizeof(_type)))
 347 #define EMAC_RSS_TBL(_i, _type) \
 348                 (EMAC_IDT_TABLE0 + ((_i) * sizeof(_type)))
 349
 350 /* Config MAC modes */
 351 void emac_mac_mode_config(struct emac_adapter *adpt)
 352 {
 353         struct net_device *netdev = adpt->netdev;
 354         u32 mac;
 355
 356         mac = readl(adpt->base + EMAC_MAC_CTRL);
 357         mac &= ~(VLAN_STRIP | PROM_MODE | MULTI_ALL | MAC_LP_EN);
 358
 359         if (netdev->features & NETIF_F_HW_VLAN_CTAG_RX)
 360                 mac |= VLAN_STRIP;
 361
 362         if (netdev->flags & IFF_PROMISC)
 363                 mac |= PROM_MODE;
 364
 365         if (netdev->flags & IFF_ALLMULTI)
 366                 mac |= MULTI_ALL;
 367
 368         writel(mac, adpt->base + EMAC_MAC_CTRL);
 369 }
 370
 371 /* Config descriptor rings */
 372 static void emac_mac_dma_rings_config(struct emac_adapter *adpt)
 373 {
 374         static const unsigned short tpd_q_offset[] = {
 375                 EMAC_DESC_CTRL_8,        EMAC_H1TPD_BASE_ADDR_LO,
 376                 EMAC_H2TPD_BASE_ADDR_LO, EMAC_H3TPD_BASE_ADDR_LO};
 377         static const unsigned short rfd_q_offset[] = {
 378                 EMAC_DESC_CTRL_2,        EMAC_DESC_CTRL_10,
 379                 EMAC_DESC_CTRL_12,       EMAC_DESC_CTRL_13};
 380         static const unsigned short rrd_q_offset[] = {
 381                 EMAC_DESC_CTRL_5,        EMAC_DESC_CTRL_14,
 382                 EMAC_DESC_CTRL_15,       EMAC_DESC_CTRL_16};
 383
 384         /* TPD (Transmit Packet Descriptor) */
 385         writel(upper_32_bits(adpt->tx_q.tpd.dma_addr),
 386                adpt->base + EMAC_DESC_CTRL_1);
 387
 388         writel(lower_32_bits(adpt->tx_q.tpd.dma_addr),
 389                adpt->base + tpd_q_offset[0]);
 390
 391         writel(adpt->tx_q.tpd.count & TPD_RING_SIZE_BMSK,
 392                adpt->base + EMAC_DESC_CTRL_9);
 393
 394         /* RFD (Receive Free Descriptor) & RRD (Receive Return Descriptor) */
 395         writel(upper_32_bits(adpt->rx_q.rfd.dma_addr),
 396                adpt->base + EMAC_DESC_CTRL_0);
 397
 398         writel(lower_32_bits(adpt->rx_q.rfd.dma_addr),
 399                adpt->base + rfd_q_offset[0]);
 400         writel(lower_32_bits(adpt->rx_q.rrd.dma_addr),
 401                adpt->base + rrd_q_offset[0]);
 402
 403         writel(adpt->rx_q.rfd.count & RFD_RING_SIZE_BMSK,
 404                adpt->base + EMAC_DESC_CTRL_3);
 405         writel(adpt->rx_q.rrd.count & RRD_RING_SIZE_BMSK,
 406                adpt->base + EMAC_DESC_CTRL_6);
 407
 408         writel(adpt->rxbuf_size & RX_BUFFER_SIZE_BMSK,
 409                adpt->base + EMAC_DESC_CTRL_4);
 410
 411         writel(0, adpt->base + EMAC_DESC_CTRL_11);
 412
 413         /* Load all of the base addresses above and ensure that triggering HW to
 414          * read ring pointers is flushed
 415          */
 416         writel(1, adpt->base + EMAC_INTER_SRAM_PART9);
 417 }
 418
 419 /* Config transmit parameters */
 420 static void emac_mac_tx_config(struct emac_adapter *adpt)
 421 {
 422         u32 val;
 423
 424         writel((EMAC_MAX_TX_OFFLOAD_THRESH >> 3) &
 425                JUMBO_TASK_OFFLOAD_THRESHOLD_BMSK, adpt->base + EMAC_TXQ_CTRL_1);
 426
 427         val = (adpt->tpd_burst << NUM_TPD_BURST_PREF_SHFT) &
 428                NUM_TPD_BURST_PREF_BMSK;
 429
 430         val |= TXQ_MODE | LS_8023_SP;
 431         val |= (0x0100 << NUM_TXF_BURST_PREF_SHFT) &
 432                 NUM_TXF_BURST_PREF_BMSK;
 433
 434         writel(val, adpt->base + EMAC_TXQ_CTRL_0);
 435         emac_reg_update32(adpt->base + EMAC_TXQ_CTRL_2,
 436                           (TXF_HWM_BMSK | TXF_LWM_BMSK), 0);
 437 }
 438
 439 /* Config receive parameters */
 440 static void emac_mac_rx_config(struct emac_adapter *adpt)
 441 {
 442         u32 val;
 443
 444         val = (adpt->rfd_burst << NUM_RFD_BURST_PREF_SHFT) &
 445                NUM_RFD_BURST_PREF_BMSK;
 446         val |= (SP_IPV6 | CUT_THRU_EN);
 447
 448         writel(val, adpt->base + EMAC_RXQ_CTRL_0);
 449
 450         val = readl(adpt->base + EMAC_RXQ_CTRL_1);
 451         val &= ~(JUMBO_1KAH_BMSK | RFD_PREF_LOW_THRESHOLD_BMSK |
 452                  RFD_PREF_UP_THRESHOLD_BMSK);
 453         val |= (JUMBO_1KAH << JUMBO_1KAH_SHFT) |
 454                 (RFD_PREF_LOW_TH << RFD_PREF_LOW_THRESHOLD_SHFT) |
 455                 (RFD_PREF_UP_TH  << RFD_PREF_UP_THRESHOLD_SHFT);
 456         writel(val, adpt->base + EMAC_RXQ_CTRL_1);
 457
 458         val = readl(adpt->base + EMAC_RXQ_CTRL_2);
 459         val &= ~(RXF_DOF_THRESHOLD_BMSK | RXF_UOF_THRESHOLD_BMSK);
 460         val |= (RXF_DOF_THRESFHOLD  << RXF_DOF_THRESHOLD_SHFT) |
 461                 (RXF_UOF_THRESFHOLD << RXF_UOF_THRESHOLD_SHFT);
 462         writel(val, adpt->base + EMAC_RXQ_CTRL_2);
 463
 464         val = readl(adpt->base + EMAC_RXQ_CTRL_3);
 465         val &= ~(RXD_TIMER_BMSK | RXD_THRESHOLD_BMSK);
 466         val |= RXD_TH << RXD_THRESHOLD_SHFT;
 467         writel(val, adpt->base + EMAC_RXQ_CTRL_3);
 468 }
 469
 470 /* Config dma */
 471 static void emac_mac_dma_config(struct emac_adapter *adpt)
 472 {
 473         u32 dma_ctrl = DMAR_REQ_PRI;
 474
 475         switch (adpt->dma_order) {
 476         case emac_dma_ord_in:
 477                 dma_ctrl |= IN_ORDER_MODE;
 478                 break;
 479         case emac_dma_ord_enh:
 480                 dma_ctrl |= ENH_ORDER_MODE;
 481                 break;
 482         case emac_dma_ord_out:
 483                 dma_ctrl |= OUT_ORDER_MODE;
 484                 break;
 485         default:
 486                 break;
 487         }
 488
 489         dma_ctrl |= (((u32)adpt->dmar_block) << REGRDBLEN_SHFT) &
 490                                                 REGRDBLEN_BMSK;
 491         dma_ctrl |= (((u32)adpt->dmaw_block) << REGWRBLEN_SHFT) &
 492                                                 REGWRBLEN_BMSK;
 493         dma_ctrl |= (((u32)adpt->dmar_dly_cnt) << DMAR_DLY_CNT_SHFT) &
 494                                                 DMAR_DLY_CNT_BMSK;
 495         dma_ctrl |= (((u32)adpt->dmaw_dly_cnt) << DMAW_DLY_CNT_SHFT) &
 496                                                 DMAW_DLY_CNT_BMSK;
 497
 498         /* config DMA and ensure that configuration is flushed to HW */
 499         writel(dma_ctrl, adpt->base + EMAC_DMA_CTRL);
 500 }
 501
 502 /* set MAC address */
 503 static void emac_set_mac_address(struct emac_adapter *adpt, u8 *addr)
 504 {
 505         u32 sta;
 506
 507         /* for example: 00-A0-C6-11-22-33
 508          * 0<-->C6112233, 1<-->00A0.
 509          */
 510
 511         /* low 32bit word */
 512         sta = (((u32)addr[2]) << 24) | (((u32)addr[3]) << 16) |
 513               (((u32)addr[4]) << 8)  | (((u32)addr[5]));
 514         writel(sta, adpt->base + EMAC_MAC_STA_ADDR0);
 515
 516         /* hight 32bit word */
 517         sta = (((u32)addr[0]) << 8) | (u32)addr[1];
 518         writel(sta, adpt->base + EMAC_MAC_STA_ADDR1);
 519 }
 520
 521 static void emac_mac_config(struct emac_adapter *adpt)
 522 {
 523         struct net_device *netdev = adpt->netdev;
 524         unsigned int max_frame;
 525         u32 val;
 526
 527         emac_set_mac_address(adpt, netdev->dev_addr);
 528
 529         max_frame = netdev->mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;
 530         adpt->rxbuf_size = netdev->mtu > EMAC_DEF_RX_BUF_SIZE ?
 531                 ALIGN(max_frame, 8) : EMAC_DEF_RX_BUF_SIZE;
 532
 533         emac_mac_dma_rings_config(adpt);
 534
 535         writel(netdev->mtu + ETH_HLEN + VLAN_HLEN + ETH_FCS_LEN,
 536                adpt->base + EMAC_MAX_FRAM_LEN_CTRL);
 537
 538         emac_mac_tx_config(adpt);
 539         emac_mac_rx_config(adpt);
 540         emac_mac_dma_config(adpt);
 541
 542         val = readl(adpt->base + EMAC_AXI_MAST_CTRL);
 543         val &= ~(DATA_BYTE_SWAP | MAX_BOUND);
 544         val |= MAX_BTYPE;
 545         writel(val, adpt->base + EMAC_AXI_MAST_CTRL);
 546         writel(0, adpt->base + EMAC_CLK_GATE_CTRL);
 547         writel(RX_UNCPL_INT_EN, adpt->base + EMAC_MISC_CTRL);
 548 }
 549
 550 void emac_mac_reset(struct emac_adapter *adpt)
 551 {
 552         emac_mac_stop(adpt);
 553
 554         emac_reg_update32(adpt->base + EMAC_DMA_MAS_CTRL, 0, SOFT_RST);
 555         usleep_range(100, 150); /* reset may take up to 100usec */
 556
 557         /* interrupt clear-on-read */
 558         emac_reg_update32(adpt->base + EMAC_DMA_MAS_CTRL, 0, INT_RD_CLR_EN);
 559 }
 560
 561 void emac_mac_start(struct emac_adapter *adpt)
 562 {
 563         struct phy_device *phydev = adpt->phydev;
 564         u32 mac, csr1;
 565
 566         /* enable tx queue */
 567         emac_reg_update32(adpt->base + EMAC_TXQ_CTRL_0, 0, TXQ_EN);
 568
 569         /* enable rx queue */
 570         emac_reg_update32(adpt->base + EMAC_RXQ_CTRL_0, 0, RXQ_EN);
 571
 572         /* enable mac control */
 573         mac = readl(adpt->base + EMAC_MAC_CTRL);
 574         csr1 = readl(adpt->csr + EMAC_EMAC_WRAPPER_CSR1);
 575
 576         mac |= TXEN | RXEN;     /* enable RX/TX */
 577
 578         /* Configure MAC flow control to match the PHY's settings. */
 579         if (phydev->pause)
 580                 mac |= RXFC;
 581         if (phydev->pause != phydev->asym_pause)
 582                 mac |= TXFC;
 583
 584         /* setup link speed */
 585         mac &= ~SPEED_MASK;
 586         if (phydev->speed == SPEED_1000) {
 587                 mac |= SPEED(2);
 588                 csr1 |= FREQ_MODE;
 589         } else {
 590                 mac |= SPEED(1);
 591                 csr1 &= ~FREQ_MODE;
 592         }
 593
 594         if (phydev->duplex == DUPLEX_FULL)
 595                 mac |= FULLD;
 596         else
 597                 mac &= ~FULLD;
 598
 599         /* other parameters */
 600         mac |= (CRCE | PCRCE);
 601         mac |= ((adpt->preamble << PRLEN_SHFT) & PRLEN_BMSK);
 602         mac |= BROAD_EN;
 603         mac |= FLCHK;
 604         mac &= ~RX_CHKSUM_EN;
 605         mac &= ~(HUGEN | VLAN_STRIP | TPAUSE | SIMR | HUGE | MULTI_ALL |
 606                  DEBUG_MODE | SINGLE_PAUSE_MODE);
 607
 608         writel_relaxed(csr1, adpt->csr + EMAC_EMAC_WRAPPER_CSR1);
 609
 610         writel_relaxed(mac, adpt->base + EMAC_MAC_CTRL);
 611
 612         /* enable interrupt read clear, low power sleep mode and
 613          * the irq moderators
 614          */
 615
 616         writel_relaxed(adpt->irq_mod, adpt->base + EMAC_IRQ_MOD_TIM_INIT);
 617         writel_relaxed(INT_RD_CLR_EN | LPW_MODE | IRQ_MODERATOR_EN |
 618                         IRQ_MODERATOR2_EN, adpt->base + EMAC_DMA_MAS_CTRL);
 619
 620         emac_mac_mode_config(adpt);
 621
 622         emac_reg_update32(adpt->base + EMAC_ATHR_HEADER_CTRL,
 623                           (HEADER_ENABLE | HEADER_CNT_EN), 0);
 624
 625         emac_reg_update32(adpt->csr + EMAC_EMAC_WRAPPER_CSR2, 0, WOL_EN);
 626 }
 627
 628 void emac_mac_stop(struct emac_adapter *adpt)
 629 {
 630         emac_reg_update32(adpt->base + EMAC_RXQ_CTRL_0, RXQ_EN, 0);
 631         emac_reg_update32(adpt->base + EMAC_TXQ_CTRL_0, TXQ_EN, 0);
 632         emac_reg_update32(adpt->base + EMAC_MAC_CTRL, TXEN | RXEN, 0);
 633         usleep_range(1000, 1050); /* stopping mac may take upto 1msec */
 634 }
 635
 636 /* Free all descriptors of given transmit queue */
 637 static void emac_tx_q_descs_free(struct emac_adapter *adpt)
 638 {
 639         struct emac_tx_queue *tx_q = &adpt->tx_q;
 640         unsigned int i;
 641         size_t size;
 642
 643         /* ring already cleared, nothing to do */
 644         if (!tx_q->tpd.tpbuff)
 645                 return;
 646
 647         for (i = 0; i < tx_q->tpd.count; i++) {
 648                 struct emac_buffer *tpbuf = GET_TPD_BUFFER(tx_q, i);
 649
 650                 if (tpbuf->dma_addr) {
 651                         dma_unmap_single(adpt->netdev->dev.parent,
 652                                          tpbuf->dma_addr, tpbuf->length,
 653                                          DMA_TO_DEVICE);
 654                         tpbuf->dma_addr = 0;
 655                 }
 656                 if (tpbuf->skb) {
 657                         dev_kfree_skb_any(tpbuf->skb);
 658                         tpbuf->skb = NULL;
 659                 }
 660         }
 661
 662         size = sizeof(struct emac_buffer) * tx_q->tpd.count;
 663         memset(tx_q->tpd.tpbuff, 0, size);
 664
 665         /* clear the descriptor ring */
 666         memset(tx_q->tpd.v_addr, 0, tx_q->tpd.size);
 667
 668         tx_q->tpd.consume_idx = 0;
 669         tx_q->tpd.produce_idx = 0;
 670 }
 671
 672 /* Free all descriptors of given receive queue */
 673 static void emac_rx_q_free_descs(struct emac_adapter *adpt)
 674 {
 675         struct device *dev = adpt->netdev->dev.parent;
 676         struct emac_rx_queue *rx_q = &adpt->rx_q;
 677         unsigned int i;
 678         size_t size;
 679
 680         /* ring already cleared, nothing to do */
 681         if (!rx_q->rfd.rfbuff)
 682                 return;
 683
 684         for (i = 0; i < rx_q->rfd.count; i++) {
 685                 struct emac_buffer *rfbuf = GET_RFD_BUFFER(rx_q, i);
 686
 687                 if (rfbuf->dma_addr) {
 688                         dma_unmap_single(dev, rfbuf->dma_addr, rfbuf->length,
 689                                          DMA_FROM_DEVICE);
 690                         rfbuf->dma_addr = 0;
 691                 }
 692                 if (rfbuf->skb) {
 693                         dev_kfree_skb(rfbuf->skb);
 694                         rfbuf->skb = NULL;
 695                 }
 696         }
 697
 698         size =  sizeof(struct emac_buffer) * rx_q->rfd.count;
 699         memset(rx_q->rfd.rfbuff, 0, size);
 700
 701         /* clear the descriptor rings */
 702         memset(rx_q->rrd.v_addr, 0, rx_q->rrd.size);
 703         rx_q->rrd.produce_idx = 0;
 704         rx_q->rrd.consume_idx = 0;
 705
 706         memset(rx_q->rfd.v_addr, 0, rx_q->rfd.size);
 707         rx_q->rfd.produce_idx = 0;
 708         rx_q->rfd.consume_idx = 0;
 709 }
 710
 711 /* Free all buffers associated with given transmit queue */
 712 static void emac_tx_q_bufs_free(struct emac_adapter *adpt)
 713 {
 714         struct emac_tx_queue *tx_q = &adpt->tx_q;
 715
 716         emac_tx_q_descs_free(adpt);
 717
 718         kfree(tx_q->tpd.tpbuff);
 719         tx_q->tpd.tpbuff = NULL;
 720         tx_q->tpd.v_addr = NULL;
 721         tx_q->tpd.dma_addr = 0;
 722         tx_q->tpd.size = 0;
 723 }
 724
 725 /* Allocate TX descriptor ring for the given transmit queue */
 726 static int emac_tx_q_desc_alloc(struct emac_adapter *adpt,
 727                                 struct emac_tx_queue *tx_q)
 728 {
 729         struct emac_ring_header *ring_header = &adpt->ring_header;
 730         size_t size;
 731
 732         size = sizeof(struct emac_buffer) * tx_q->tpd.count;
 733         tx_q->tpd.tpbuff = kzalloc(size, GFP_KERNEL);
 734         if (!tx_q->tpd.tpbuff)
 735                 return -ENOMEM;
 736
 737         tx_q->tpd.size = tx_q->tpd.count * (adpt->tpd_size * 4);
 738         tx_q->tpd.dma_addr = ring_header->dma_addr + ring_header->used;
 739         tx_q->tpd.v_addr = ring_header->v_addr + ring_header->used;
 740         ring_header->used += ALIGN(tx_q->tpd.size, 8);
 741         tx_q->tpd.produce_idx = 0;
 742         tx_q->tpd.consume_idx = 0;
 743
 744         return 0;
 745 }
 746
 747 /* Free all buffers associated with given transmit queue */
 748 static void emac_rx_q_bufs_free(struct emac_adapter *adpt)
 749 {
 750         struct emac_rx_queue *rx_q = &adpt->rx_q;
 751
 752         emac_rx_q_free_descs(adpt);
 753
 754         kfree(rx_q->rfd.rfbuff);
 755         rx_q->rfd.rfbuff   = NULL;
 756
 757         rx_q->rfd.v_addr   = NULL;
 758         rx_q->rfd.dma_addr = 0;
 759         rx_q->rfd.size     = 0;
 760
 761         rx_q->rrd.v_addr   = NULL;
 762         rx_q->rrd.dma_addr = 0;
 763         rx_q->rrd.size     = 0;
 764 }
 765
 766 /* Allocate RX descriptor rings for the given receive queue */
 767 static int emac_rx_descs_alloc(struct emac_adapter *adpt)
 768 {
 769         struct emac_ring_header *ring_header = &adpt->ring_header;
 770         struct emac_rx_queue *rx_q = &adpt->rx_q;
 771         size_t size;
 772
 773         size = sizeof(struct emac_buffer) * rx_q->rfd.count;
 774         rx_q->rfd.rfbuff = kzalloc(size, GFP_KERNEL);
 775         if (!rx_q->rfd.rfbuff)
 776                 return -ENOMEM;
 777
 778         rx_q->rrd.size = rx_q->rrd.count * (adpt->rrd_size * 4);
 779         rx_q->rfd.size = rx_q->rfd.count * (adpt->rfd_size * 4);
 780
 781         rx_q->rrd.dma_addr = ring_header->dma_addr + ring_header->used;
 782         rx_q->rrd.v_addr   = ring_header->v_addr + ring_header->used;
 783         ring_header->used += ALIGN(rx_q->rrd.size, 8);
 784
 785         rx_q->rfd.dma_addr = ring_header->dma_addr + ring_header->used;
 786         rx_q->rfd.v_addr   = ring_header->v_addr + ring_header->used;
 787         ring_header->used += ALIGN(rx_q->rfd.size, 8);
 788
 789         rx_q->rrd.produce_idx = 0;
 790         rx_q->rrd.consume_idx = 0;
 791
 792         rx_q->rfd.produce_idx = 0;
 793         rx_q->rfd.consume_idx = 0;
 794
 795         return 0;
 796 }
 797
 798 /* Allocate all TX and RX descriptor rings */
 799 int emac_mac_rx_tx_rings_alloc_all(struct emac_adapter *adpt)
 800 {
 801         struct emac_ring_header *ring_header = &adpt->ring_header;
 802         struct device *dev = adpt->netdev->dev.parent;
 803         unsigned int num_tx_descs = adpt->tx_desc_cnt;
 804         unsigned int num_rx_descs = adpt->rx_desc_cnt;
 805         int ret;
 806
 807         adpt->tx_q.tpd.count = adpt->tx_desc_cnt;
 808
 809         adpt->rx_q.rrd.count = adpt->rx_desc_cnt;
 810         adpt->rx_q.rfd.count = adpt->rx_desc_cnt;
 811
 812         /* Ring DMA buffer. Each ring may need up to 8 bytes for alignment,
 813          * hence the additional padding bytes are allocated.
 814          */
 815         ring_header->size = num_tx_descs * (adpt->tpd_size * 4) +
 816                             num_rx_descs * (adpt->rfd_size * 4) +
 817                             num_rx_descs * (adpt->rrd_size * 4) +
 818                             8 + 2 * 8; /* 8 byte per one Tx and two Rx rings */
 819
 820         ring_header->used = 0;
 821         ring_header->v_addr = dma_zalloc_coherent(dev, ring_header->size,
 822                                                  &ring_header->dma_addr,
 823                                                  GFP_KERNEL);
 824         if (!ring_header->v_addr)
 825                 return -ENOMEM;
 826
 827         ring_header->used = ALIGN(ring_header->dma_addr, 8) -
 828                                                         ring_header->dma_addr;
 829
 830         ret = emac_tx_q_desc_alloc(adpt, &adpt->tx_q);
 831         if (ret) {
 832                 netdev_err(adpt->netdev, "error: Tx Queue alloc failed\n");
 833                 goto err_alloc_tx;
 834         }
 835
 836         ret = emac_rx_descs_alloc(adpt);
 837         if (ret) {
 838                 netdev_err(adpt->netdev, "error: Rx Queue alloc failed\n");
 839                 goto err_alloc_rx;
 840         }
 841
 842         return 0;
 843
 844 err_alloc_rx:
 845         emac_tx_q_bufs_free(adpt);
 846 err_alloc_tx:
 847         dma_free_coherent(dev, ring_header->size,
 848                           ring_header->v_addr, ring_header->dma_addr);
 849
 850         ring_header->v_addr   = NULL;
 851         ring_header->dma_addr = 0;
 852         ring_header->size     = 0;
 853         ring_header->used     = 0;
 854
 855         return ret;
 856 }
 857
 858 /* Free all TX and RX descriptor rings */
 859 void emac_mac_rx_tx_rings_free_all(struct emac_adapter *adpt)
 860 {
 861         struct emac_ring_header *ring_header = &adpt->ring_header;
 862         struct device *dev = adpt->netdev->dev.parent;
 863
 864         emac_tx_q_bufs_free(adpt);
 865         emac_rx_q_bufs_free(adpt);
 866
 867         dma_free_coherent(dev, ring_header->size,
 868                           ring_header->v_addr, ring_header->dma_addr);
 869
 870         ring_header->v_addr   = NULL;
 871         ring_header->dma_addr = 0;
 872         ring_header->size     = 0;
 873         ring_header->used     = 0;
 874 }
 875
 876 /* Initialize descriptor rings */
 877 static void emac_mac_rx_tx_ring_reset_all(struct emac_adapter *adpt)
 878 {
 879         unsigned int i;
 880
 881         adpt->tx_q.tpd.produce_idx = 0;
 882         adpt->tx_q.tpd.consume_idx = 0;
 883         for (i = 0; i < adpt->tx_q.tpd.count; i++)
 884                 adpt->tx_q.tpd.tpbuff[i].dma_addr = 0;
 885
 886         adpt->rx_q.rrd.produce_idx = 0;
 887         adpt->rx_q.rrd.consume_idx = 0;
 888         adpt->rx_q.rfd.produce_idx = 0;
 889         adpt->rx_q.rfd.consume_idx = 0;
 890         for (i = 0; i < adpt->rx_q.rfd.count; i++)
 891                 adpt->rx_q.rfd.rfbuff[i].dma_addr = 0;
 892 }
 893
 894 /* Produce new receive free descriptor */
 895 static void emac_mac_rx_rfd_create(struct emac_adapter *adpt,
 896                                    struct emac_rx_queue *rx_q,
 897                                    dma_addr_t addr)
 898 {
 899         u32 *hw_rfd = EMAC_RFD(rx_q, adpt->rfd_size, rx_q->rfd.produce_idx);
 900
 901         *(hw_rfd++) = lower_32_bits(addr);
 902         *hw_rfd = upper_32_bits(addr);
 903
 904         if (++rx_q->rfd.produce_idx == rx_q->rfd.count)
 905                 rx_q->rfd.produce_idx = 0;
 906 }
 907
 908 /* Fill up receive queue's RFD with preallocated receive buffers */
 909 static void emac_mac_rx_descs_refill(struct emac_adapter *adpt,
 910                                     struct emac_rx_queue *rx_q)
 911 {
 912         struct emac_buffer *curr_rxbuf;
 913         struct emac_buffer *next_rxbuf;
 914         unsigned int count = 0;
 915         u32 next_produce_idx;
 916
 917         next_produce_idx = rx_q->rfd.produce_idx + 1;
 918         if (next_produce_idx == rx_q->rfd.count)
 919                 next_produce_idx = 0;
 920
 921         curr_rxbuf = GET_RFD_BUFFER(rx_q, rx_q->rfd.produce_idx);
 922         next_rxbuf = GET_RFD_BUFFER(rx_q, next_produce_idx);
 923
 924         /* this always has a blank rx_buffer*/
 925         while (!next_rxbuf->dma_addr) {
 926                 struct sk_buff *skb;
 927                 int ret;
 928
 929                 skb = netdev_alloc_skb_ip_align(adpt->netdev, adpt->rxbuf_size);
 930                 if (!skb)
 931                         break;
 932
 933                 curr_rxbuf->dma_addr =
 934                         dma_map_single(adpt->netdev->dev.parent, skb->data,
 935                                        curr_rxbuf->length, DMA_FROM_DEVICE);
 936                 ret = dma_mapping_error(adpt->netdev->dev.parent,
 937                                         curr_rxbuf->dma_addr);
 938                 if (ret) {
 939                         dev_kfree_skb(skb);
 940                         break;
 941                 }
 942                 curr_rxbuf->skb = skb;
 943                 curr_rxbuf->length = adpt->rxbuf_size;
 944
 945                 emac_mac_rx_rfd_create(adpt, rx_q, curr_rxbuf->dma_addr);
 946                 next_produce_idx = rx_q->rfd.produce_idx + 1;
 947                 if (next_produce_idx == rx_q->rfd.count)
 948                         next_produce_idx = 0;
 949
 950                 curr_rxbuf = GET_RFD_BUFFER(rx_q, rx_q->rfd.produce_idx);
 951                 next_rxbuf = GET_RFD_BUFFER(rx_q, next_produce_idx);
 952                 count++;
 953         }
 954
 955         if (count) {
 956                 u32 prod_idx = (rx_q->rfd.produce_idx << rx_q->produce_shift) &
 957                                 rx_q->produce_mask;
 958                 emac_reg_update32(adpt->base + rx_q->produce_reg,
 959                                   rx_q->produce_mask, prod_idx);
 960         }
 961 }
 962
 963 static void emac_adjust_link(struct net_device *netdev)
 964 {
 965         struct emac_adapter *adpt = netdev_priv(netdev);
 966         struct phy_device *phydev = netdev->phydev;
 967
 968         if (phydev->link)
 969                 emac_mac_start(adpt);
 970         else
 971                 emac_mac_stop(adpt);
 972
 973         phy_print_status(phydev);
 974 }
 975
 976 /* Bringup the interface/HW */
 977 int emac_mac_up(struct emac_adapter *adpt)
 978 {
 979         struct net_device *netdev = adpt->netdev;
 980         struct emac_irq *irq = &adpt->irq;
 981         int ret;
 982
 983         emac_mac_rx_tx_ring_reset_all(adpt);
 984         emac_mac_config(adpt);
 985
 986         ret = request_irq(irq->irq, emac_isr, 0, EMAC_MAC_IRQ_RES, irq);
 987         if (ret) {
 988                 netdev_err(adpt->netdev, "could not request %s irq\n",
 989                            EMAC_MAC_IRQ_RES);
 990                 return ret;
 991         }
 992
 993         emac_mac_rx_descs_refill(adpt, &adpt->rx_q);
 994
 995         ret = phy_connect_direct(netdev, adpt->phydev, emac_adjust_link,
 996                                  PHY_INTERFACE_MODE_SGMII);
 997         if (ret) {
 998                 netdev_err(adpt->netdev, "could not connect phy\n");
 999                 free_irq(irq->irq, irq);
1000                 return ret;
1001         }
1002
1003         /* enable mac irq */
1004         writel((u32)~DIS_INT, adpt->base + EMAC_INT_STATUS);
1005         writel(adpt->irq.mask, adpt->base + EMAC_INT_MASK);
1006
1007         /* Enable pause frames.  Without this feature, the EMAC has been shown
1008          * to receive (and drop) frames with FCS errors at gigabit connections.
1009          */
1010         adpt->phydev->supported |= SUPPORTED_Pause | SUPPORTED_Asym_Pause;
1011         adpt->phydev->advertising |= SUPPORTED_Pause | SUPPORTED_Asym_Pause;
1012
1013         adpt->phydev->irq = PHY_IGNORE_INTERRUPT;
1014         phy_start(adpt->phydev);
1015
1016         napi_enable(&adpt->rx_q.napi);
1017         netif_start_queue(netdev);
1018
1019         return 0;
1020 }
1021
1022 /* Bring down the interface/HW */
1023 void emac_mac_down(struct emac_adapter *adpt)
1024 {
1025         struct net_device *netdev = adpt->netdev;
1026
1027         netif_stop_queue(netdev);
1028         napi_disable(&adpt->rx_q.napi);
1029
1030         phy_stop(adpt->phydev);
1031
1032         /* Interrupts must be disabled before the PHY is disconnected, to
1033          * avoid a race condition where adjust_link is null when we get
1034          * an interrupt.
1035          */
1036         writel(DIS_INT, adpt->base + EMAC_INT_STATUS);
1037         writel(0, adpt->base + EMAC_INT_MASK);
1038         synchronize_irq(adpt->irq.irq);
1039         free_irq(adpt->irq.irq, &adpt->irq);
1040
1041         phy_disconnect(adpt->phydev);
1042
1043         emac_mac_reset(adpt);
1044
1045         emac_tx_q_descs_free(adpt);
1046         netdev_reset_queue(adpt->netdev);
1047         emac_rx_q_free_descs(adpt);
1048 }
1049
1050 /* Consume next received packet descriptor */
1051 static bool emac_rx_process_rrd(struct emac_adapter *adpt,
1052                                 struct emac_rx_queue *rx_q,
1053                                 struct emac_rrd *rrd)
1054 {
1055         u32 *hw_rrd = EMAC_RRD(rx_q, adpt->rrd_size, rx_q->rrd.consume_idx);
1056
1057         rrd->word[3] = *(hw_rrd + 3);
1058
1059         if (!RRD_UPDT(rrd))
1060                 return false;
1061
1062         rrd->word[4] = 0;
1063         rrd->word[5] = 0;
1064
1065         rrd->word[0] = *(hw_rrd++);
1066         rrd->word[1] = *(hw_rrd++);
1067         rrd->word[2] = *(hw_rrd++);
1068
1069         if (unlikely(RRD_NOR(rrd) != 1)) {
1070                 netdev_err(adpt->netdev,
1071                            "error: multi-RFD not support yet! nor:%lu\n",
1072                            RRD_NOR(rrd));
1073         }
1074
1075         /* mark rrd as processed */
1076         RRD_UPDT_SET(rrd, 0);
1077         *hw_rrd = rrd->word[3];
1078
1079         if (++rx_q->rrd.consume_idx == rx_q->rrd.count)
1080                 rx_q->rrd.consume_idx = 0;
1081
1082         return true;
1083 }
1084
1085 /* Produce new transmit descriptor */
1086 static void emac_tx_tpd_create(struct emac_adapter *adpt,
1087                                struct emac_tx_queue *tx_q, struct emac_tpd *tpd)
1088 {
1089         u32 *hw_tpd;
1090
1091         tx_q->tpd.last_produce_idx = tx_q->tpd.produce_idx;
1092         hw_tpd = EMAC_TPD(tx_q, adpt->tpd_size, tx_q->tpd.produce_idx);
1093
1094         if (++tx_q->tpd.produce_idx == tx_q->tpd.count)
1095                 tx_q->tpd.produce_idx = 0;
1096
1097         *(hw_tpd++) = tpd->word[0];
1098         *(hw_tpd++) = tpd->word[1];
1099         *(hw_tpd++) = tpd->word[2];
1100         *hw_tpd = tpd->word[3];
1101 }
1102
1103 /* Mark the last transmit descriptor as such (for the transmit packet) */
1104 static void emac_tx_tpd_mark_last(struct emac_adapter *adpt,
1105                                   struct emac_tx_queue *tx_q)
1106 {
1107         u32 *hw_tpd =
1108                 EMAC_TPD(tx_q, adpt->tpd_size, tx_q->tpd.last_produce_idx);
1109         u32 tmp_tpd;
1110
1111         tmp_tpd = *(hw_tpd + 1);
1112         tmp_tpd |= EMAC_TPD_LAST_FRAGMENT;
1113         *(hw_tpd + 1) = tmp_tpd;
1114 }
1115
1116 static void emac_rx_rfd_clean(struct emac_rx_queue *rx_q, struct emac_rrd *rrd)
1117 {
1118         struct emac_buffer *rfbuf = rx_q->rfd.rfbuff;
1119         u32 consume_idx = RRD_SI(rrd);
1120         unsigned int i;
1121
1122         for (i = 0; i < RRD_NOR(rrd); i++) {
1123                 rfbuf[consume_idx].skb = NULL;
1124                 if (++consume_idx == rx_q->rfd.count)
1125                         consume_idx = 0;
1126         }
1127
1128         rx_q->rfd.consume_idx = consume_idx;
1129         rx_q->rfd.process_idx = consume_idx;
1130 }
1131
1132 /* Push the received skb to upper layers */
1133 static void emac_receive_skb(struct emac_rx_queue *rx_q,
1134                              struct sk_buff *skb,
1135                              u16 vlan_tag, bool vlan_flag)
1136 {
1137         if (vlan_flag) {
1138                 u16 vlan;
1139
1140                 EMAC_TAG_TO_VLAN(vlan_tag, vlan);
1141                 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlan);
1142         }
1143
1144         napi_gro_receive(&rx_q->napi, skb);
1145 }
1146
1147 /* Process receive event */
1148 void emac_mac_rx_process(struct emac_adapter *adpt, struct emac_rx_queue *rx_q,
1149                          int *num_pkts, int max_pkts)
1150 {
1151         u32 proc_idx, hw_consume_idx, num_consume_pkts;
1152         struct net_device *netdev  = adpt->netdev;
1153         struct emac_buffer *rfbuf;
1154         unsigned int count = 0;
1155         struct emac_rrd rrd;
1156         struct sk_buff *skb;
1157         u32 reg;
1158
1159         reg = readl_relaxed(adpt->base + rx_q->consume_reg);
1160
1161         hw_consume_idx = (reg & rx_q->consume_mask) >> rx_q->consume_shift;
1162         num_consume_pkts = (hw_consume_idx >= rx_q->rrd.consume_idx) ?
1163                 (hw_consume_idx -  rx_q->rrd.consume_idx) :
1164                 (hw_consume_idx + rx_q->rrd.count - rx_q->rrd.consume_idx);
1165
1166         do {
1167                 if (!num_consume_pkts)
1168                         break;
1169
1170                 if (!emac_rx_process_rrd(adpt, rx_q, &rrd))
1171                         break;
1172
1173                 if (likely(RRD_NOR(&rrd) == 1)) {
1174                         /* good receive */
1175                         rfbuf = GET_RFD_BUFFER(rx_q, RRD_SI(&rrd));
1176                         dma_unmap_single(adpt->netdev->dev.parent,
1177                                          rfbuf->dma_addr, rfbuf->length,
1178                                          DMA_FROM_DEVICE);
1179                         rfbuf->dma_addr = 0;
1180                         skb = rfbuf->skb;
1181                 } else {
1182                         netdev_err(adpt->netdev,
1183                                    "error: multi-RFD not support yet!\n");
1184                         break;
1185                 }
1186                 emac_rx_rfd_clean(rx_q, &rrd);
1187                 num_consume_pkts--;
1188                 count++;
1189
1190                 /* Due to a HW issue in L4 check sum detection (UDP/TCP frags
1191                  * with DF set are marked as error), drop packets based on the
1192                  * error mask rather than the summary bit (ignoring L4F errors)
1193                  */
1194                 if (rrd.word[EMAC_RRD_STATS_DW_IDX] & EMAC_RRD_ERROR) {
1195                         netif_dbg(adpt, rx_status, adpt->netdev,
1196                                   "Drop error packet[RRD: 0x%x:0x%x:0x%x:0x%x]\n",
1197                                   rrd.word[0], rrd.word[1],
1198                                   rrd.word[2], rrd.word[3]);
1199
1200                         dev_kfree_skb(skb);
1201                         continue;
1202                 }
1203
1204                 skb_put(skb, RRD_PKT_SIZE(&rrd) - ETH_FCS_LEN);
1205                 skb->dev = netdev;
1206                 skb->protocol = eth_type_trans(skb, skb->dev);
1207                 if (netdev->features & NETIF_F_RXCSUM)
1208                         skb->ip_summed = RRD_L4F(&rrd) ?
1209                                           CHECKSUM_NONE : CHECKSUM_UNNECESSARY;
1210                 else
1211                         skb_checksum_none_assert(skb);
1212
1213                 emac_receive_skb(rx_q, skb, (u16)RRD_CVALN_TAG(&rrd),
1214                                  (bool)RRD_CVTAG(&rrd));
1215
1216                 netdev->last_rx = jiffies;
1217                 (*num_pkts)++;
1218         } while (*num_pkts < max_pkts);
1219
1220         if (count) {
1221                 proc_idx = (rx_q->rfd.process_idx << rx_q->process_shft) &
1222                                 rx_q->process_mask;
1223                 emac_reg_update32(adpt->base + rx_q->process_reg,
1224                                   rx_q->process_mask, proc_idx);
1225                 emac_mac_rx_descs_refill(adpt, rx_q);
1226         }
1227 }
1228
1229 /* get the number of free transmit descriptors */
1230 static unsigned int emac_tpd_num_free_descs(struct emac_tx_queue *tx_q)
1231 {
1232         u32 produce_idx = tx_q->tpd.produce_idx;
1233         u32 consume_idx = tx_q->tpd.consume_idx;
1234
1235         return (consume_idx > produce_idx) ?
1236                 (consume_idx - produce_idx - 1) :
1237                 (tx_q->tpd.count + consume_idx - produce_idx - 1);
1238 }
1239
1240 /* Process transmit event */
1241 void emac_mac_tx_process(struct emac_adapter *adpt, struct emac_tx_queue *tx_q)
1242 {
1243         u32 reg = readl_relaxed(adpt->base + tx_q->consume_reg);
1244         u32 hw_consume_idx, pkts_compl = 0, bytes_compl = 0;
1245         struct emac_buffer *tpbuf;
1246
1247         hw_consume_idx = (reg & tx_q->consume_mask) >> tx_q->consume_shift;
1248
1249         while (tx_q->tpd.consume_idx != hw_consume_idx) {
1250                 tpbuf = GET_TPD_BUFFER(tx_q, tx_q->tpd.consume_idx);
1251                 if (tpbuf->dma_addr) {
1252                         dma_unmap_single(adpt->netdev->dev.parent,
1253                                          tpbuf->dma_addr, tpbuf->length,
1254                                          DMA_TO_DEVICE);
1255                         tpbuf->dma_addr = 0;
1256                 }
1257
1258                 if (tpbuf->skb) {
1259                         pkts_compl++;
1260                         bytes_compl += tpbuf->skb->len;
1261                         dev_kfree_skb_irq(tpbuf->skb);
1262                         tpbuf->skb = NULL;
1263                 }
1264
1265                 if (++tx_q->tpd.consume_idx == tx_q->tpd.count)
1266                         tx_q->tpd.consume_idx = 0;
1267         }
1268
1269         netdev_completed_queue(adpt->netdev, pkts_compl, bytes_compl);
1270
1271         if (netif_queue_stopped(adpt->netdev))
1272                 if (emac_tpd_num_free_descs(tx_q) > (MAX_SKB_FRAGS + 1))
1273                         netif_wake_queue(adpt->netdev);
1274 }
1275
1276 /* Initialize all queue data structures */
1277 void emac_mac_rx_tx_ring_init_all(struct platform_device *pdev,
1278                                   struct emac_adapter *adpt)
1279 {
1280         adpt->rx_q.netdev = adpt->netdev;
1281
1282         adpt->rx_q.produce_reg  = EMAC_MAILBOX_0;
1283         adpt->rx_q.produce_mask = RFD0_PROD_IDX_BMSK;
1284         adpt->rx_q.produce_shift = RFD0_PROD_IDX_SHFT;
1285
1286         adpt->rx_q.process_reg  = EMAC_MAILBOX_0;
1287         adpt->rx_q.process_mask = RFD0_PROC_IDX_BMSK;
1288         adpt->rx_q.process_shft = RFD0_PROC_IDX_SHFT;
1289
1290         adpt->rx_q.consume_reg  = EMAC_MAILBOX_3;
1291         adpt->rx_q.consume_mask = RFD0_CONS_IDX_BMSK;
1292         adpt->rx_q.consume_shift = RFD0_CONS_IDX_SHFT;
1293
1294         adpt->rx_q.irq          = &adpt->irq;
1295         adpt->rx_q.intr         = adpt->irq.mask & ISR_RX_PKT;
1296
1297         adpt->tx_q.produce_reg  = EMAC_MAILBOX_15;
1298         adpt->tx_q.produce_mask = NTPD_PROD_IDX_BMSK;
1299         adpt->tx_q.produce_shift = NTPD_PROD_IDX_SHFT;
1300
1301         adpt->tx_q.consume_reg  = EMAC_MAILBOX_2;
1302         adpt->tx_q.consume_mask = NTPD_CONS_IDX_BMSK;
1303         adpt->tx_q.consume_shift = NTPD_CONS_IDX_SHFT;
1304 }
1305
1306 /* Fill up transmit descriptors with TSO and Checksum offload information */
1307 static int emac_tso_csum(struct emac_adapter *adpt,
1308                          struct emac_tx_queue *tx_q,
1309                          struct sk_buff *skb,
1310                          struct emac_tpd *tpd)
1311 {
1312         unsigned int hdr_len;
1313         int ret;
1314
1315         if (skb_is_gso(skb)) {
1316                 if (skb_header_cloned(skb)) {
1317                         ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
1318                         if (unlikely(ret))
1319                                 return ret;
1320                 }
1321
1322                 if (skb->protocol == htons(ETH_P_IP)) {
1323                         u32 pkt_len = ((unsigned char *)ip_hdr(skb) - skb->data)
1324                                        + ntohs(ip_hdr(skb)->tot_len);
1325                         if (skb->len > pkt_len)
1326                                 pskb_trim(skb, pkt_len);
1327                 }
1328
1329                 hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb);
1330                 if (unlikely(skb->len == hdr_len)) {
1331                         /* we only need to do csum */
1332                         netif_warn(adpt, tx_err, adpt->netdev,
1333                                    "tso not needed for packet with 0 data\n");
1334                         goto do_csum;
1335                 }
1336
1337                 if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4) {
1338                         ip_hdr(skb)->check = 0;
1339                         tcp_hdr(skb)->check =
1340                                 ~csum_tcpudp_magic(ip_hdr(skb)->saddr,
1341                                                    ip_hdr(skb)->daddr,
1342                                                    0, IPPROTO_TCP, 0);
1343                         TPD_IPV4_SET(tpd, 1);
1344                 }
1345
1346                 if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6) {
1347                         /* ipv6 tso need an extra tpd */
1348                         struct emac_tpd extra_tpd;
1349
1350                         memset(tpd, 0, sizeof(*tpd));
1351                         memset(&extra_tpd, 0, sizeof(extra_tpd));
1352
1353                         ipv6_hdr(skb)->payload_len = 0;
1354                         tcp_hdr(skb)->check =
1355                                 ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
1356                                                  &ipv6_hdr(skb)->daddr,
1357                                                  0, IPPROTO_TCP, 0);
1358                         TPD_PKT_LEN_SET(&extra_tpd, skb->len);
1359                         TPD_LSO_SET(&extra_tpd, 1);
1360                         TPD_LSOV_SET(&extra_tpd, 1);
1361                         emac_tx_tpd_create(adpt, tx_q, &extra_tpd);
1362                         TPD_LSOV_SET(tpd, 1);
1363                 }
1364
1365                 TPD_LSO_SET(tpd, 1);
1366                 TPD_TCPHDR_OFFSET_SET(tpd, skb_transport_offset(skb));
1367                 TPD_MSS_SET(tpd, skb_shinfo(skb)->gso_size);
1368                 return 0;
1369         }
1370
1371 do_csum:
1372         if (likely(skb->ip_summed == CHECKSUM_PARTIAL)) {
1373                 unsigned int css, cso;
1374
1375                 cso = skb_transport_offset(skb);
1376                 if (unlikely(cso & 0x1)) {
1377                         netdev_err(adpt->netdev,
1378                                    "error: payload offset should be even\n");
1379                         return -EINVAL;
1380                 }
1381                 css = cso + skb->csum_offset;
1382
1383                 TPD_PAYLOAD_OFFSET_SET(tpd, cso >> 1);
1384                 TPD_CXSUM_OFFSET_SET(tpd, css >> 1);
1385                 TPD_CSX_SET(tpd, 1);
1386         }
1387
1388         return 0;
1389 }
1390
1391 /* Fill up transmit descriptors */
1392 static void emac_tx_fill_tpd(struct emac_adapter *adpt,
1393                              struct emac_tx_queue *tx_q, struct sk_buff *skb,
1394                              struct emac_tpd *tpd)
1395 {
1396         unsigned int nr_frags = skb_shinfo(skb)->nr_frags;
1397         unsigned int first = tx_q->tpd.produce_idx;
1398         unsigned int len = skb_headlen(skb);
1399         struct emac_buffer *tpbuf = NULL;
1400         unsigned int mapped_len = 0;
1401         unsigned int i;
1402         int count = 0;
1403         int ret;
1404
1405         /* if Large Segment Offload is (in TCP Segmentation Offload struct) */
1406         if (TPD_LSO(tpd)) {
1407                 mapped_len = skb_transport_offset(skb) + tcp_hdrlen(skb);
1408
1409                 tpbuf = GET_TPD_BUFFER(tx_q, tx_q->tpd.produce_idx);
1410                 tpbuf->length = mapped_len;
1411                 tpbuf->dma_addr = dma_map_single(adpt->netdev->dev.parent,
1412                                                  skb->data, tpbuf->length,
1413                                                  DMA_TO_DEVICE);
1414                 ret = dma_mapping_error(adpt->netdev->dev.parent,
1415                                         tpbuf->dma_addr);
1416                 if (ret)
1417                         goto error;
1418
1419                 TPD_BUFFER_ADDR_L_SET(tpd, lower_32_bits(tpbuf->dma_addr));
1420                 TPD_BUFFER_ADDR_H_SET(tpd, upper_32_bits(tpbuf->dma_addr));
1421                 TPD_BUF_LEN_SET(tpd, tpbuf->length);
1422                 emac_tx_tpd_create(adpt, tx_q, tpd);
1423                 count++;
1424         }
1425
1426         if (mapped_len < len) {
1427                 tpbuf = GET_TPD_BUFFER(tx_q, tx_q->tpd.produce_idx);
1428                 tpbuf->length = len - mapped_len;
1429                 tpbuf->dma_addr = dma_map_single(adpt->netdev->dev.parent,
1430                                                  skb->data + mapped_len,
1431                                                  tpbuf->length, DMA_TO_DEVICE);
1432                 ret = dma_mapping_error(adpt->netdev->dev.parent,
1433                                         tpbuf->dma_addr);
1434                 if (ret)
1435                         goto error;
1436
1437                 TPD_BUFFER_ADDR_L_SET(tpd, lower_32_bits(tpbuf->dma_addr));
1438                 TPD_BUFFER_ADDR_H_SET(tpd, upper_32_bits(tpbuf->dma_addr));
1439                 TPD_BUF_LEN_SET(tpd, tpbuf->length);
1440                 emac_tx_tpd_create(adpt, tx_q, tpd);
1441                 count++;
1442         }
1443
1444         for (i = 0; i < nr_frags; i++) {
1445                 struct skb_frag_struct *frag;
1446
1447                 frag = &skb_shinfo(skb)->frags[i];
1448
1449                 tpbuf = GET_TPD_BUFFER(tx_q, tx_q->tpd.produce_idx);
1450                 tpbuf->length = frag->size;
1451                 tpbuf->dma_addr = dma_map_page(adpt->netdev->dev.parent,
1452                                                frag->page.p, frag->page_offset,
1453                                                tpbuf->length, DMA_TO_DEVICE);
1454                 ret = dma_mapping_error(adpt->netdev->dev.parent,
1455                                         tpbuf->dma_addr);
1456                 if (ret)
1457                         goto error;
1458
1459                 TPD_BUFFER_ADDR_L_SET(tpd, lower_32_bits(tpbuf->dma_addr));
1460                 TPD_BUFFER_ADDR_H_SET(tpd, upper_32_bits(tpbuf->dma_addr));
1461                 TPD_BUF_LEN_SET(tpd, tpbuf->length);
1462                 emac_tx_tpd_create(adpt, tx_q, tpd);
1463                 count++;
1464         }
1465
1466         /* The last tpd */
1467         wmb();
1468         emac_tx_tpd_mark_last(adpt, tx_q);
1469
1470         /* The last buffer info contain the skb address,
1471          * so it will be freed after unmap
1472          */
1473         tpbuf->skb = skb;
1474
1475         return;
1476
1477 error:
1478         /* One of the memory mappings failed, so undo everything */
1479         tx_q->tpd.produce_idx = first;
1480
1481         while (count--) {
1482                 tpbuf = GET_TPD_BUFFER(tx_q, first);
1483                 dma_unmap_page(adpt->netdev->dev.parent, tpbuf->dma_addr,
1484                                tpbuf->length, DMA_TO_DEVICE);
1485                 tpbuf->dma_addr = 0;
1486                 tpbuf->length = 0;
1487
1488                 if (++first == tx_q->tpd.count)
1489                         first = 0;
1490         }
1491
1492         dev_kfree_skb(skb);
1493 }
1494
1495 /* Transmit the packet using specified transmit queue */
1496 int emac_mac_tx_buf_send(struct emac_adapter *adpt, struct emac_tx_queue *tx_q,
1497                          struct sk_buff *skb)
1498 {
1499         struct emac_tpd tpd;
1500         u32 prod_idx;
1501
1502         memset(&tpd, 0, sizeof(tpd));
1503
1504         if (emac_tso_csum(adpt, tx_q, skb, &tpd) != 0) {
1505                 dev_kfree_skb_any(skb);
1506                 return NETDEV_TX_OK;
1507         }
1508
1509         if (skb_vlan_tag_present(skb)) {
1510                 u16 tag;
1511
1512                 EMAC_VLAN_TO_TAG(skb_vlan_tag_get(skb), tag);
1513                 TPD_CVLAN_TAG_SET(&tpd, tag);
1514                 TPD_INSTC_SET(&tpd, 1);
1515         }
1516
1517         if (skb_network_offset(skb) != ETH_HLEN)
1518                 TPD_TYP_SET(&tpd, 1);
1519
1520         emac_tx_fill_tpd(adpt, tx_q, skb, &tpd);
1521
1522         netdev_sent_queue(adpt->netdev, skb->len);
1523
1524         /* Make sure the are enough free descriptors to hold one
1525          * maximum-sized SKB.  We need one desc for each fragment,
1526          * one for the checksum (emac_tso_csum), one for TSO, and
1527          * and one for the SKB header.
1528          */
1529         if (emac_tpd_num_free_descs(tx_q) < (MAX_SKB_FRAGS + 3))
1530                 netif_stop_queue(adpt->netdev);
1531
1532         /* update produce idx */
1533         prod_idx = (tx_q->tpd.produce_idx << tx_q->produce_shift) &
1534                     tx_q->produce_mask;
1535         emac_reg_update32(adpt->base + tx_q->produce_reg,
1536                           tx_q->produce_mask, prod_idx);
1537
1538         return NETDEV_TX_OK;
1539 }