drivers/net/ethernet/huawei/hinic/hinic_hw_cmdq.c

   1 /*
   2  * Huawei HiNIC PCI Express Linux driver
   3  * Copyright(c) 2017 Huawei Technologies Co., Ltd
   4  *
   5  * This program is free software; you can redistribute it and/or modify it
   6  * under the terms and conditions of the GNU General Public License,
   7  * version 2, as published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * for more details.
  13  *
  14  */
  15
  16 #include <linux/kernel.h>
  17 #include <linux/types.h>
  18 #include <linux/errno.h>
  19 #include <linux/pci.h>
  20 #include <linux/device.h>
  21 #include <linux/slab.h>
  22 #include <linux/vmalloc.h>
  23 #include <linux/spinlock.h>
  24 #include <linux/sizes.h>
  25 #include <linux/atomic.h>
  26 #include <linux/log2.h>
  27 #include <linux/io.h>
  28 #include <linux/completion.h>
  29 #include <linux/err.h>
  30 #include <asm/byteorder.h>
  31 #include <asm/barrier.h>
  32
  33 #include "hinic_common.h"
  34 #include "hinic_hw_if.h"
  35 #include "hinic_hw_eqs.h"
  36 #include "hinic_hw_mgmt.h"
  37 #include "hinic_hw_wqe.h"
  38 #include "hinic_hw_wq.h"
  39 #include "hinic_hw_cmdq.h"
  40 #include "hinic_hw_io.h"
  41 #include "hinic_hw_dev.h"
  42
  43 #define CMDQ_CEQE_TYPE_SHIFT                    0
  44
  45 #define CMDQ_CEQE_TYPE_MASK                     0x7
  46
  47 #define CMDQ_CEQE_GET(val, member)              \
  48                         (((val) >> CMDQ_CEQE_##member##_SHIFT) \
  49                          & CMDQ_CEQE_##member##_MASK)
  50
  51 #define CMDQ_WQE_ERRCODE_VAL_SHIFT              20
  52
  53 #define CMDQ_WQE_ERRCODE_VAL_MASK               0xF
  54
  55 #define CMDQ_WQE_ERRCODE_GET(val, member)       \
  56                         (((val) >> CMDQ_WQE_ERRCODE_##member##_SHIFT) \
  57                          & CMDQ_WQE_ERRCODE_##member##_MASK)
  58
  59 #define CMDQ_DB_PI_OFF(pi)              (((u16)LOWER_8_BITS(pi)) << 3)
  60
  61 #define CMDQ_DB_ADDR(db_base, pi)       ((db_base) + CMDQ_DB_PI_OFF(pi))
  62
  63 #define CMDQ_WQE_HEADER(wqe)            ((struct hinic_cmdq_header *)(wqe))
  64
  65 #define CMDQ_WQE_COMPLETED(ctrl_info)   \
  66                         HINIC_CMDQ_CTRL_GET(ctrl_info, HW_BUSY_BIT)
  67
  68 #define FIRST_DATA_TO_WRITE_LAST        sizeof(u64)
  69
  70 #define CMDQ_DB_OFF                     SZ_2K
  71
  72 #define CMDQ_WQEBB_SIZE                 64
  73 #define CMDQ_WQE_SIZE                   64
  74 #define CMDQ_DEPTH                      SZ_4K
  75
  76 #define CMDQ_WQ_PAGE_SIZE               SZ_4K
  77
  78 #define WQE_LCMD_SIZE                   64
  79 #define WQE_SCMD_SIZE                   64
  80
  81 #define COMPLETE_LEN                    3
  82
  83 #define CMDQ_TIMEOUT                    1000
  84
  85 #define CMDQ_PFN(addr, page_size)       ((addr) >> (ilog2(page_size)))
  86
  87 #define cmdq_to_cmdqs(cmdq)     container_of((cmdq) - (cmdq)->cmdq_type, \
  88                                              struct hinic_cmdqs, cmdq[0])
  89
  90 #define cmdqs_to_func_to_io(cmdqs)      container_of(cmdqs, \
  91                                                      struct hinic_func_to_io, \
  92                                                      cmdqs)
  93
  94 enum cmdq_wqe_type {
  95         WQE_LCMD_TYPE = 0,
  96         WQE_SCMD_TYPE = 1,
  97 };
  98
  99 enum completion_format {
 100         COMPLETE_DIRECT = 0,
 101         COMPLETE_SGE    = 1,
 102 };
 103
 104 enum data_format {
 105         DATA_SGE        = 0,
 106         DATA_DIRECT     = 1,
 107 };
 108
 109 enum bufdesc_len {
 110         BUFDESC_LCMD_LEN = 2,   /* 16 bytes - 2(8 byte unit) */
 111         BUFDESC_SCMD_LEN = 3,   /* 24 bytes - 3(8 byte unit) */
 112 };
 113
 114 enum ctrl_sect_len {
 115         CTRL_SECT_LEN        = 1, /* 4 bytes (ctrl) - 1(8 byte unit) */
 116         CTRL_DIRECT_SECT_LEN = 2, /* 12 bytes (ctrl + rsvd) - 2(8 byte unit) */
 117 };
 118
 119 enum cmdq_scmd_type {
 120         CMDQ_SET_ARM_CMD = 2,
 121 };
 122
 123 enum cmdq_cmd_type {
 124         CMDQ_CMD_SYNC_DIRECT_RESP = 0,
 125         CMDQ_CMD_SYNC_SGE_RESP    = 1,
 126 };
 127
 128 enum completion_request {
 129         NO_CEQ  = 0,
 130         CEQ_SET = 1,
 131 };
 132
 133 /**
 134  * hinic_alloc_cmdq_buf - alloc buffer for sending command
 135  * @cmdqs: the cmdqs
 136  * @cmdq_buf: the buffer returned in this struct
 137  *
 138  * Return 0 - Success, negative - Failure
 139  **/
 140 int hinic_alloc_cmdq_buf(struct hinic_cmdqs *cmdqs,
 141                          struct hinic_cmdq_buf *cmdq_buf)
 142 {
 143         struct hinic_hwif *hwif = cmdqs->hwif;
 144         struct pci_dev *pdev = hwif->pdev;
 145
 146         cmdq_buf->buf = pci_pool_alloc(cmdqs->cmdq_buf_pool, GFP_KERNEL,
 147                                        &cmdq_buf->dma_addr);
 148         if (!cmdq_buf->buf) {
 149                 dev_err(&pdev->dev, "Failed to allocate cmd from the pool\n");
 150                 return -ENOMEM;
 151         }
 152
 153         return 0;
 154 }
 155
 156 /**
 157  * hinic_free_cmdq_buf - free buffer
 158  * @cmdqs: the cmdqs
 159  * @cmdq_buf: the buffer to free that is in this struct
 160  **/
 161 void hinic_free_cmdq_buf(struct hinic_cmdqs *cmdqs,
 162                          struct hinic_cmdq_buf *cmdq_buf)
 163 {
 164         pci_pool_free(cmdqs->cmdq_buf_pool, cmdq_buf->buf, cmdq_buf->dma_addr);
 165 }
 166
 167 static unsigned int cmdq_wqe_size_from_bdlen(enum bufdesc_len len)
 168 {
 169         unsigned int wqe_size = 0;
 170
 171         switch (len) {
 172         case BUFDESC_LCMD_LEN:
 173                 wqe_size = WQE_LCMD_SIZE;
 174                 break;
 175         case BUFDESC_SCMD_LEN:
 176                 wqe_size = WQE_SCMD_SIZE;
 177                 break;
 178         }
 179
 180         return wqe_size;
 181 }
 182
 183 static void cmdq_set_sge_completion(struct hinic_cmdq_completion *completion,
 184                                     struct hinic_cmdq_buf *buf_out)
 185 {
 186         struct hinic_sge_resp *sge_resp = &completion->sge_resp;
 187
 188         hinic_set_sge(&sge_resp->sge, buf_out->dma_addr, buf_out->size);
 189 }
 190
 191 static void cmdq_prepare_wqe_ctrl(struct hinic_cmdq_wqe *wqe, int wrapped,
 192                                   enum hinic_cmd_ack_type ack_type,
 193                                   enum hinic_mod_type mod, u8 cmd, u16 prod_idx,
 194                                   enum completion_format complete_format,
 195                                   enum data_format data_format,
 196                                   enum bufdesc_len buf_len)
 197 {
 198         struct hinic_cmdq_wqe_lcmd *wqe_lcmd;
 199         struct hinic_cmdq_wqe_scmd *wqe_scmd;
 200         enum ctrl_sect_len ctrl_len;
 201         struct hinic_ctrl *ctrl;
 202         u32 saved_data;
 203
 204         if (data_format == DATA_SGE) {
 205                 wqe_lcmd = &wqe->wqe_lcmd;
 206
 207                 wqe_lcmd->status.status_info = 0;
 208                 ctrl = &wqe_lcmd->ctrl;
 209                 ctrl_len = CTRL_SECT_LEN;
 210         } else {
 211                 wqe_scmd = &wqe->direct_wqe.wqe_scmd;
 212
 213                 wqe_scmd->status.status_info = 0;
 214                 ctrl = &wqe_scmd->ctrl;
 215                 ctrl_len = CTRL_DIRECT_SECT_LEN;
 216         }
 217
 218         ctrl->ctrl_info = HINIC_CMDQ_CTRL_SET(prod_idx, PI)             |
 219                           HINIC_CMDQ_CTRL_SET(cmd, CMD)                 |
 220                           HINIC_CMDQ_CTRL_SET(mod, MOD)                 |
 221                           HINIC_CMDQ_CTRL_SET(ack_type, ACK_TYPE);
 222
 223         CMDQ_WQE_HEADER(wqe)->header_info =
 224                 HINIC_CMDQ_WQE_HEADER_SET(buf_len, BUFDESC_LEN)            |
 225                 HINIC_CMDQ_WQE_HEADER_SET(complete_format, COMPLETE_FMT)   |
 226                 HINIC_CMDQ_WQE_HEADER_SET(data_format, DATA_FMT)           |
 227                 HINIC_CMDQ_WQE_HEADER_SET(CEQ_SET, COMPLETE_REQ)           |
 228                 HINIC_CMDQ_WQE_HEADER_SET(COMPLETE_LEN, COMPLETE_SECT_LEN) |
 229                 HINIC_CMDQ_WQE_HEADER_SET(ctrl_len, CTRL_LEN)              |
 230                 HINIC_CMDQ_WQE_HEADER_SET(wrapped, TOGGLED_WRAPPED);
 231
 232         saved_data = CMDQ_WQE_HEADER(wqe)->saved_data;
 233         saved_data = HINIC_SAVED_DATA_CLEAR(saved_data, ARM);
 234
 235         if ((cmd == CMDQ_SET_ARM_CMD) && (mod == HINIC_MOD_COMM))
 236                 CMDQ_WQE_HEADER(wqe)->saved_data |=
 237                                                 HINIC_SAVED_DATA_SET(1, ARM);
 238         else
 239                 CMDQ_WQE_HEADER(wqe)->saved_data = saved_data;
 240 }
 241
 242 static void cmdq_set_lcmd_bufdesc(struct hinic_cmdq_wqe_lcmd *wqe_lcmd,
 243                                   struct hinic_cmdq_buf *buf_in)
 244 {
 245         hinic_set_sge(&wqe_lcmd->buf_desc.sge, buf_in->dma_addr, buf_in->size);
 246 }
 247
 248 static void cmdq_set_direct_wqe_data(struct hinic_cmdq_direct_wqe *wqe,
 249                                      void *buf_in, u32 in_size)
 250 {
 251         struct hinic_cmdq_wqe_scmd *wqe_scmd = &wqe->wqe_scmd;
 252
 253         wqe_scmd->buf_desc.buf_len = in_size;
 254         memcpy(wqe_scmd->buf_desc.data, buf_in, in_size);
 255 }
 256
 257 static void cmdq_set_lcmd_wqe(struct hinic_cmdq_wqe *wqe,
 258                               enum cmdq_cmd_type cmd_type,
 259                               struct hinic_cmdq_buf *buf_in,
 260                               struct hinic_cmdq_buf *buf_out, int wrapped,
 261                               enum hinic_cmd_ack_type ack_type,
 262                               enum hinic_mod_type mod, u8 cmd, u16 prod_idx)
 263 {
 264         struct hinic_cmdq_wqe_lcmd *wqe_lcmd = &wqe->wqe_lcmd;
 265         enum completion_format complete_format;
 266
 267         switch (cmd_type) {
 268         case CMDQ_CMD_SYNC_SGE_RESP:
 269                 complete_format = COMPLETE_SGE;
 270                 cmdq_set_sge_completion(&wqe_lcmd->completion, buf_out);
 271                 break;
 272         case CMDQ_CMD_SYNC_DIRECT_RESP:
 273                 complete_format = COMPLETE_DIRECT;
 274                 wqe_lcmd->completion.direct_resp = 0;
 275                 break;
 276         }
 277
 278         cmdq_prepare_wqe_ctrl(wqe, wrapped, ack_type, mod, cmd,
 279                               prod_idx, complete_format, DATA_SGE,
 280                               BUFDESC_LCMD_LEN);
 281
 282         cmdq_set_lcmd_bufdesc(wqe_lcmd, buf_in);
 283 }
 284
 285 static void cmdq_set_direct_wqe(struct hinic_cmdq_wqe *wqe,
 286                                 enum cmdq_cmd_type cmd_type,
 287                                 void *buf_in, u16 in_size,
 288                                 struct hinic_cmdq_buf *buf_out, int wrapped,
 289                                 enum hinic_cmd_ack_type ack_type,
 290                                 enum hinic_mod_type mod, u8 cmd, u16 prod_idx)
 291 {
 292         struct hinic_cmdq_direct_wqe *direct_wqe = &wqe->direct_wqe;
 293         enum completion_format complete_format;
 294         struct hinic_cmdq_wqe_scmd *wqe_scmd;
 295
 296         wqe_scmd = &direct_wqe->wqe_scmd;
 297
 298         switch (cmd_type) {
 299         case CMDQ_CMD_SYNC_SGE_RESP:
 300                 complete_format = COMPLETE_SGE;
 301                 cmdq_set_sge_completion(&wqe_scmd->completion, buf_out);
 302                 break;
 303         case CMDQ_CMD_SYNC_DIRECT_RESP:
 304                 complete_format = COMPLETE_DIRECT;
 305                 wqe_scmd->completion.direct_resp = 0;
 306                 break;
 307         }
 308
 309         cmdq_prepare_wqe_ctrl(wqe, wrapped, ack_type, mod, cmd, prod_idx,
 310                               complete_format, DATA_DIRECT, BUFDESC_SCMD_LEN);
 311
 312         cmdq_set_direct_wqe_data(direct_wqe, buf_in, in_size);
 313 }
 314
 315 static void cmdq_wqe_fill(void *dst, void *src)
 316 {
 317         memcpy(dst + FIRST_DATA_TO_WRITE_LAST, src + FIRST_DATA_TO_WRITE_LAST,
 318                CMDQ_WQE_SIZE - FIRST_DATA_TO_WRITE_LAST);
 319
 320         wmb();          /* The first 8 bytes should be written last */
 321
 322         *(u64 *)dst = *(u64 *)src;
 323 }
 324
 325 static void cmdq_fill_db(u32 *db_info,
 326                          enum hinic_cmdq_type cmdq_type, u16 prod_idx)
 327 {
 328         *db_info = HINIC_CMDQ_DB_INFO_SET(UPPER_8_BITS(prod_idx), HI_PROD_IDX) |
 329                    HINIC_CMDQ_DB_INFO_SET(HINIC_CTRL_PATH, PATH)               |
 330                    HINIC_CMDQ_DB_INFO_SET(cmdq_type, CMDQ_TYPE)                |
 331                    HINIC_CMDQ_DB_INFO_SET(HINIC_DB_CMDQ_TYPE, DB_TYPE);
 332 }
 333
 334 static void cmdq_set_db(struct hinic_cmdq *cmdq,
 335                         enum hinic_cmdq_type cmdq_type, u16 prod_idx)
 336 {
 337         u32 db_info;
 338
 339         cmdq_fill_db(&db_info, cmdq_type, prod_idx);
 340
 341         /* The data that is written to HW should be in Big Endian Format */
 342         db_info = cpu_to_be32(db_info);
 343
 344         wmb();  /* write all before the doorbell */
 345
 346         writel(db_info, CMDQ_DB_ADDR(cmdq->db_base, prod_idx));
 347 }
 348
 349 static int cmdq_sync_cmd_direct_resp(struct hinic_cmdq *cmdq,
 350                                      enum hinic_mod_type mod, u8 cmd,
 351                                      struct hinic_cmdq_buf *buf_in,
 352                                      u64 *resp)
 353 {
 354         struct hinic_cmdq_wqe *curr_cmdq_wqe, cmdq_wqe;
 355         u16 curr_prod_idx, next_prod_idx;
 356         int errcode, wrapped, num_wqebbs;
 357         struct hinic_wq *wq = cmdq->wq;
 358         struct hinic_hw_wqe *hw_wqe;
 359         struct completion done;
 360
 361         /* Keep doorbell index correct. bh - for tasklet(ceq). */
 362         spin_lock_bh(&cmdq->cmdq_lock);
 363
 364         /* WQE_SIZE = WQEBB_SIZE, we will get the wq element and not shadow*/
 365         hw_wqe = hinic_get_wqe(wq, WQE_LCMD_SIZE, &curr_prod_idx);
 366         if (IS_ERR(hw_wqe)) {
 367                 spin_unlock_bh(&cmdq->cmdq_lock);
 368                 return -EBUSY;
 369         }
 370
 371         curr_cmdq_wqe = &hw_wqe->cmdq_wqe;
 372
 373         wrapped = cmdq->wrapped;
 374
 375         num_wqebbs = ALIGN(WQE_LCMD_SIZE, wq->wqebb_size) / wq->wqebb_size;
 376         next_prod_idx = curr_prod_idx + num_wqebbs;
 377         if (next_prod_idx >= wq->q_depth) {
 378                 cmdq->wrapped = !cmdq->wrapped;
 379                 next_prod_idx -= wq->q_depth;
 380         }
 381
 382         cmdq->errcode[curr_prod_idx] = &errcode;
 383
 384         init_completion(&done);
 385         cmdq->done[curr_prod_idx] = &done;
 386
 387         cmdq_set_lcmd_wqe(&cmdq_wqe, CMDQ_CMD_SYNC_DIRECT_RESP, buf_in, NULL,
 388                           wrapped, HINIC_CMD_ACK_TYPE_CMDQ, mod, cmd,
 389                           curr_prod_idx);
 390
 391         /* The data that is written to HW should be in Big Endian Format */
 392         hinic_cpu_to_be32(&cmdq_wqe, WQE_LCMD_SIZE);
 393
 394         /* CMDQ WQE is not shadow, therefore wqe will be written to wq */
 395         cmdq_wqe_fill(curr_cmdq_wqe, &cmdq_wqe);
 396
 397         cmdq_set_db(cmdq, HINIC_CMDQ_SYNC, next_prod_idx);
 398
 399         spin_unlock_bh(&cmdq->cmdq_lock);
 400
 401         if (!wait_for_completion_timeout(&done,
 402                                          msecs_to_jiffies(CMDQ_TIMEOUT))) {
 403                 spin_lock_bh(&cmdq->cmdq_lock);
 404
 405                 if (cmdq->errcode[curr_prod_idx] == &errcode)
 406                         cmdq->errcode[curr_prod_idx] = NULL;
 407
 408                 if (cmdq->done[curr_prod_idx] == &done)
 409                         cmdq->done[curr_prod_idx] = NULL;
 410
 411                 spin_unlock_bh(&cmdq->cmdq_lock);
 412
 413                 return -ETIMEDOUT;
 414         }
 415
 416         smp_rmb();      /* read error code after completion */
 417
 418         if (resp) {
 419                 struct hinic_cmdq_wqe_lcmd *wqe_lcmd = &curr_cmdq_wqe->wqe_lcmd;
 420
 421                 *resp = cpu_to_be64(wqe_lcmd->completion.direct_resp);
 422         }
 423
 424         if (errcode != 0)
 425                 return -EFAULT;
 426
 427         return 0;
 428 }
 429
 430 static int cmdq_set_arm_bit(struct hinic_cmdq *cmdq, void *buf_in,
 431                             u16 in_size)
 432 {
 433         struct hinic_cmdq_wqe *curr_cmdq_wqe, cmdq_wqe;
 434         u16 curr_prod_idx, next_prod_idx;
 435         struct hinic_wq *wq = cmdq->wq;
 436         struct hinic_hw_wqe *hw_wqe;
 437         int wrapped, num_wqebbs;
 438
 439         /* Keep doorbell index correct */
 440         spin_lock(&cmdq->cmdq_lock);
 441
 442         /* WQE_SIZE = WQEBB_SIZE, we will get the wq element and not shadow*/
 443         hw_wqe = hinic_get_wqe(wq, WQE_SCMD_SIZE, &curr_prod_idx);
 444         if (IS_ERR(hw_wqe)) {
 445                 spin_unlock(&cmdq->cmdq_lock);
 446                 return -EBUSY;
 447         }
 448
 449         curr_cmdq_wqe = &hw_wqe->cmdq_wqe;
 450
 451         wrapped = cmdq->wrapped;
 452
 453         num_wqebbs = ALIGN(WQE_SCMD_SIZE, wq->wqebb_size) / wq->wqebb_size;
 454         next_prod_idx = curr_prod_idx + num_wqebbs;
 455         if (next_prod_idx >= wq->q_depth) {
 456                 cmdq->wrapped = !cmdq->wrapped;
 457                 next_prod_idx -= wq->q_depth;
 458         }
 459
 460         cmdq_set_direct_wqe(&cmdq_wqe, CMDQ_CMD_SYNC_DIRECT_RESP, buf_in,
 461                             in_size, NULL, wrapped, HINIC_CMD_ACK_TYPE_CMDQ,
 462                             HINIC_MOD_COMM, CMDQ_SET_ARM_CMD, curr_prod_idx);
 463
 464         /* The data that is written to HW should be in Big Endian Format */
 465         hinic_cpu_to_be32(&cmdq_wqe, WQE_SCMD_SIZE);
 466
 467         /* cmdq wqe is not shadow, therefore wqe will be written to wq */
 468         cmdq_wqe_fill(curr_cmdq_wqe, &cmdq_wqe);
 469
 470         cmdq_set_db(cmdq, HINIC_CMDQ_SYNC, next_prod_idx);
 471
 472         spin_unlock(&cmdq->cmdq_lock);
 473         return 0;
 474 }
 475
 476 static int cmdq_params_valid(struct hinic_cmdq_buf *buf_in)
 477 {
 478         if (buf_in->size > HINIC_CMDQ_MAX_DATA_SIZE)
 479                 return -EINVAL;
 480
 481         return 0;
 482 }
 483
 484 /**
 485  * hinic_cmdq_direct_resp - send command with direct data as resp
 486  * @cmdqs: the cmdqs
 487  * @mod: module on the card that will handle the command
 488  * @cmd: the command
 489  * @buf_in: the buffer for the command
 490  * @resp: the response to return
 491  *
 492  * Return 0 - Success, negative - Failure
 493  **/
 494 int hinic_cmdq_direct_resp(struct hinic_cmdqs *cmdqs,
 495                            enum hinic_mod_type mod, u8 cmd,
 496                            struct hinic_cmdq_buf *buf_in, u64 *resp)
 497 {
 498         struct hinic_hwif *hwif = cmdqs->hwif;
 499         struct pci_dev *pdev = hwif->pdev;
 500         int err;
 501
 502         err = cmdq_params_valid(buf_in);
 503         if (err) {
 504                 dev_err(&pdev->dev, "Invalid CMDQ parameters\n");
 505                 return err;
 506         }
 507
 508         return cmdq_sync_cmd_direct_resp(&cmdqs->cmdq[HINIC_CMDQ_SYNC],
 509                                          mod, cmd, buf_in, resp);
 510 }
 511
 512 /**
 513  * hinic_set_arm_bit - set arm bit for enable interrupt again
 514  * @cmdqs: the cmdqs
 515  * @q_type: type of queue to set the arm bit for
 516  * @q_id: the queue number
 517  *
 518  * Return 0 - Success, negative - Failure
 519  **/
 520 int hinic_set_arm_bit(struct hinic_cmdqs *cmdqs,
 521                       enum hinic_set_arm_qtype q_type, u32 q_id)
 522 {
 523         struct hinic_cmdq *cmdq = &cmdqs->cmdq[HINIC_CMDQ_SYNC];
 524         struct hinic_hwif *hwif = cmdqs->hwif;
 525         struct pci_dev *pdev = hwif->pdev;
 526         struct hinic_cmdq_arm_bit arm_bit;
 527         int err;
 528
 529         arm_bit.q_type = q_type;
 530         arm_bit.q_id   = q_id;
 531
 532         err = cmdq_set_arm_bit(cmdq, &arm_bit, sizeof(arm_bit));
 533         if (err) {
 534                 dev_err(&pdev->dev, "Failed to set arm for qid %d\n", q_id);
 535                 return err;
 536         }
 537
 538         return 0;
 539 }
 540
 541 static void clear_wqe_complete_bit(struct hinic_cmdq *cmdq,
 542                                    struct hinic_cmdq_wqe *wqe)
 543 {
 544         u32 header_info = be32_to_cpu(CMDQ_WQE_HEADER(wqe)->header_info);
 545         unsigned int bufdesc_len, wqe_size;
 546         struct hinic_ctrl *ctrl;
 547
 548         bufdesc_len = HINIC_CMDQ_WQE_HEADER_GET(header_info, BUFDESC_LEN);
 549         wqe_size = cmdq_wqe_size_from_bdlen(bufdesc_len);
 550         if (wqe_size == WQE_LCMD_SIZE) {
 551                 struct hinic_cmdq_wqe_lcmd *wqe_lcmd = &wqe->wqe_lcmd;
 552
 553                 ctrl = &wqe_lcmd->ctrl;
 554         } else {
 555                 struct hinic_cmdq_direct_wqe *direct_wqe = &wqe->direct_wqe;
 556                 struct hinic_cmdq_wqe_scmd *wqe_scmd;
 557
 558                 wqe_scmd = &direct_wqe->wqe_scmd;
 559                 ctrl = &wqe_scmd->ctrl;
 560         }
 561
 562         /* clear HW busy bit */
 563         ctrl->ctrl_info = 0;
 564
 565         wmb();  /* verify wqe is clear */
 566 }
 567
 568 /**
 569  * cmdq_arm_ceq_handler - cmdq completion event handler for arm command
 570  * @cmdq: the cmdq of the arm command
 571  * @wqe: the wqe of the arm command
 572  *
 573  * Return 0 - Success, negative - Failure
 574  **/
 575 static int cmdq_arm_ceq_handler(struct hinic_cmdq *cmdq,
 576                                 struct hinic_cmdq_wqe *wqe)
 577 {
 578         struct hinic_cmdq_direct_wqe *direct_wqe = &wqe->direct_wqe;
 579         struct hinic_cmdq_wqe_scmd *wqe_scmd;
 580         struct hinic_ctrl *ctrl;
 581         u32 ctrl_info;
 582
 583         wqe_scmd = &direct_wqe->wqe_scmd;
 584         ctrl = &wqe_scmd->ctrl;
 585         ctrl_info = be32_to_cpu(ctrl->ctrl_info);
 586
 587         /* HW should toggle the HW BUSY BIT */
 588         if (!CMDQ_WQE_COMPLETED(ctrl_info))
 589                 return -EBUSY;
 590
 591         clear_wqe_complete_bit(cmdq, wqe);
 592
 593         hinic_put_wqe(cmdq->wq, WQE_SCMD_SIZE);
 594         return 0;
 595 }
 596
 597 static void cmdq_update_errcode(struct hinic_cmdq *cmdq, u16 prod_idx,
 598                                 int errcode)
 599 {
 600         if (cmdq->errcode[prod_idx])
 601                 *cmdq->errcode[prod_idx] = errcode;
 602 }
 603
 604 /**
 605  * cmdq_arm_ceq_handler - cmdq completion event handler for sync command
 606  * @cmdq: the cmdq of the command
 607  * @cons_idx: the consumer index to update the error code for
 608  * @errcode: the error code
 609  **/
 610 static void cmdq_sync_cmd_handler(struct hinic_cmdq *cmdq, u16 cons_idx,
 611                                   int errcode)
 612 {
 613         u16 prod_idx = cons_idx;
 614
 615         spin_lock(&cmdq->cmdq_lock);
 616         cmdq_update_errcode(cmdq, prod_idx, errcode);
 617
 618         wmb();  /* write all before update for the command request */
 619
 620         if (cmdq->done[prod_idx])
 621                 complete(cmdq->done[prod_idx]);
 622         spin_unlock(&cmdq->cmdq_lock);
 623 }
 624
 625 static int cmdq_cmd_ceq_handler(struct hinic_cmdq *cmdq, u16 ci,
 626                                 struct hinic_cmdq_wqe *cmdq_wqe)
 627 {
 628         struct hinic_cmdq_wqe_lcmd *wqe_lcmd = &cmdq_wqe->wqe_lcmd;
 629         struct hinic_status *status = &wqe_lcmd->status;
 630         struct hinic_ctrl *ctrl = &wqe_lcmd->ctrl;
 631         int errcode;
 632
 633         if (!CMDQ_WQE_COMPLETED(be32_to_cpu(ctrl->ctrl_info)))
 634                 return -EBUSY;
 635
 636         errcode = CMDQ_WQE_ERRCODE_GET(be32_to_cpu(status->status_info), VAL);
 637
 638         cmdq_sync_cmd_handler(cmdq, ci, errcode);
 639
 640         clear_wqe_complete_bit(cmdq, cmdq_wqe);
 641         hinic_put_wqe(cmdq->wq, WQE_LCMD_SIZE);
 642         return 0;
 643 }
 644
 645 /**
 646  * cmdq_ceq_handler - cmdq completion event handler
 647  * @handle: private data for the handler(cmdqs)
 648  * @ceqe_data: ceq element data
 649  **/
 650 static void cmdq_ceq_handler(void *handle, u32 ceqe_data)
 651 {
 652         enum hinic_cmdq_type cmdq_type = CMDQ_CEQE_GET(ceqe_data, TYPE);
 653         struct hinic_cmdqs *cmdqs = (struct hinic_cmdqs *)handle;
 654         struct hinic_cmdq *cmdq = &cmdqs->cmdq[cmdq_type];
 655         struct hinic_cmdq_header *header;
 656         struct hinic_hw_wqe *hw_wqe;
 657         int err, set_arm = 0;
 658         u32 saved_data;
 659         u16 ci;
 660
 661         /* Read the smallest wqe size for getting wqe size */
 662         while ((hw_wqe = hinic_read_wqe(cmdq->wq, WQE_SCMD_SIZE, &ci))) {
 663                 if (IS_ERR(hw_wqe))
 664                         break;
 665
 666                 header = CMDQ_WQE_HEADER(&hw_wqe->cmdq_wqe);
 667                 saved_data = be32_to_cpu(header->saved_data);
 668
 669                 if (HINIC_SAVED_DATA_GET(saved_data, ARM)) {
 670                         /* arm_bit was set until here */
 671                         set_arm = 0;
 672
 673                         if (cmdq_arm_ceq_handler(cmdq, &hw_wqe->cmdq_wqe))
 674                                 break;
 675                 } else {
 676                         set_arm = 1;
 677
 678                         hw_wqe = hinic_read_wqe(cmdq->wq, WQE_LCMD_SIZE, &ci);
 679                         if (IS_ERR(hw_wqe))
 680                                 break;
 681
 682                         if (cmdq_cmd_ceq_handler(cmdq, ci, &hw_wqe->cmdq_wqe))
 683                                 break;
 684                 }
 685         }
 686
 687         if (set_arm) {
 688                 struct hinic_hwif *hwif = cmdqs->hwif;
 689                 struct pci_dev *pdev = hwif->pdev;
 690
 691                 err = hinic_set_arm_bit(cmdqs, HINIC_SET_ARM_CMDQ, cmdq_type);
 692                 if (err)
 693                         dev_err(&pdev->dev, "Failed to set arm for CMDQ\n");
 694         }
 695 }
 696
 697 /**
 698  * cmdq_init_queue_ctxt - init the queue ctxt of a cmdq
 699  * @cmdq_ctxt: cmdq ctxt to initialize
 700  * @cmdq: the cmdq
 701  * @cmdq_pages: the memory of the queue
 702  **/
 703 static void cmdq_init_queue_ctxt(struct hinic_cmdq_ctxt *cmdq_ctxt,
 704                                  struct hinic_cmdq *cmdq,
 705                                  struct hinic_cmdq_pages *cmdq_pages)
 706 {
 707         struct hinic_cmdq_ctxt_info *ctxt_info = &cmdq_ctxt->ctxt_info;
 708         u64 wq_first_page_paddr, cmdq_first_block_paddr, pfn;
 709         struct hinic_cmdqs *cmdqs = cmdq_to_cmdqs(cmdq);
 710         struct hinic_wq *wq = cmdq->wq;
 711
 712         /* The data in the HW is in Big Endian Format */
 713         wq_first_page_paddr = be64_to_cpu(*wq->block_vaddr);
 714
 715         pfn = CMDQ_PFN(wq_first_page_paddr, wq->wq_page_size);
 716
 717         ctxt_info->curr_wqe_page_pfn =
 718                 HINIC_CMDQ_CTXT_PAGE_INFO_SET(pfn, CURR_WQE_PAGE_PFN)   |
 719                 HINIC_CMDQ_CTXT_PAGE_INFO_SET(HINIC_CEQ_ID_CMDQ, EQ_ID) |
 720                 HINIC_CMDQ_CTXT_PAGE_INFO_SET(1, CEQ_ARM)               |
 721                 HINIC_CMDQ_CTXT_PAGE_INFO_SET(1, CEQ_EN)                |
 722                 HINIC_CMDQ_CTXT_PAGE_INFO_SET(cmdq->wrapped, WRAPPED);
 723
 724         /* block PFN - Read Modify Write */
 725         cmdq_first_block_paddr = cmdq_pages->page_paddr;
 726
 727         pfn = CMDQ_PFN(cmdq_first_block_paddr, wq->wq_page_size);
 728
 729         ctxt_info->wq_block_pfn =
 730                 HINIC_CMDQ_CTXT_BLOCK_INFO_SET(pfn, WQ_BLOCK_PFN) |
 731                 HINIC_CMDQ_CTXT_BLOCK_INFO_SET(atomic_read(&wq->cons_idx), CI);
 732
 733         cmdq_ctxt->func_idx = HINIC_HWIF_FUNC_IDX(cmdqs->hwif);
 734         cmdq_ctxt->cmdq_type  = cmdq->cmdq_type;
 735 }
 736
 737 /**
 738  * init_cmdq - initialize cmdq
 739  * @cmdq: the cmdq
 740  * @wq: the wq attaced to the cmdq
 741  * @q_type: the cmdq type of the cmdq
 742  * @db_area: doorbell area for the cmdq
 743  *
 744  * Return 0 - Success, negative - Failure
 745  **/
 746 static int init_cmdq(struct hinic_cmdq *cmdq, struct hinic_wq *wq,
 747                      enum hinic_cmdq_type q_type, void __iomem *db_area)
 748 {
 749         int err;
 750
 751         cmdq->wq = wq;
 752         cmdq->cmdq_type = q_type;
 753         cmdq->wrapped = 1;
 754
 755         spin_lock_init(&cmdq->cmdq_lock);
 756
 757         cmdq->done = vzalloc(wq->q_depth * sizeof(*cmdq->done));
 758         if (!cmdq->done)
 759                 return -ENOMEM;
 760
 761         cmdq->errcode = vzalloc(wq->q_depth * sizeof(*cmdq->errcode));
 762         if (!cmdq->errcode) {
 763                 err = -ENOMEM;
 764                 goto err_errcode;
 765         }
 766
 767         cmdq->db_base = db_area + CMDQ_DB_OFF;
 768         return 0;
 769
 770 err_errcode:
 771         vfree(cmdq->done);
 772         return err;
 773 }
 774
 775 /**
 776  * free_cmdq - Free cmdq
 777  * @cmdq: the cmdq to free
 778  **/
 779 static void free_cmdq(struct hinic_cmdq *cmdq)
 780 {
 781         vfree(cmdq->errcode);
 782         vfree(cmdq->done);
 783 }
 784
 785 /**
 786  * init_cmdqs_ctxt - write the cmdq ctxt to HW after init all cmdq
 787  * @hwdev: the NIC HW device
 788  * @cmdqs: cmdqs to write the ctxts for
 789  * &db_area: db_area for all the cmdqs
 790  *
 791  * Return 0 - Success, negative - Failure
 792  **/
 793 static int init_cmdqs_ctxt(struct hinic_hwdev *hwdev,
 794                            struct hinic_cmdqs *cmdqs, void __iomem **db_area)
 795 {
 796         struct hinic_hwif *hwif = hwdev->hwif;
 797         enum hinic_cmdq_type type, cmdq_type;
 798         struct hinic_cmdq_ctxt *cmdq_ctxts;
 799         struct pci_dev *pdev = hwif->pdev;
 800         struct hinic_pfhwdev *pfhwdev;
 801         size_t cmdq_ctxts_size;
 802         int err;
 803
 804         if (!HINIC_IS_PF(hwif) && !HINIC_IS_PPF(hwif)) {
 805                 dev_err(&pdev->dev, "Unsupported PCI function type\n");
 806                 return -EINVAL;
 807         }
 808
 809         cmdq_ctxts_size = HINIC_MAX_CMDQ_TYPES * sizeof(*cmdq_ctxts);
 810         cmdq_ctxts = devm_kzalloc(&pdev->dev, cmdq_ctxts_size, GFP_KERNEL);
 811         if (!cmdq_ctxts)
 812                 return -ENOMEM;
 813
 814         pfhwdev = container_of(hwdev, struct hinic_pfhwdev, hwdev);
 815
 816         cmdq_type = HINIC_CMDQ_SYNC;
 817         for (; cmdq_type < HINIC_MAX_CMDQ_TYPES; cmdq_type++) {
 818                 err = init_cmdq(&cmdqs->cmdq[cmdq_type],
 819                                 &cmdqs->saved_wqs[cmdq_type], cmdq_type,
 820                                 db_area[cmdq_type]);
 821                 if (err) {
 822                         dev_err(&pdev->dev, "Failed to initialize cmdq\n");
 823                         goto err_init_cmdq;
 824                 }
 825
 826                 cmdq_init_queue_ctxt(&cmdq_ctxts[cmdq_type],
 827                                      &cmdqs->cmdq[cmdq_type],
 828                                      &cmdqs->cmdq_pages);
 829         }
 830
 831         /* Write the CMDQ ctxts */
 832         cmdq_type = HINIC_CMDQ_SYNC;
 833         for (; cmdq_type < HINIC_MAX_CMDQ_TYPES; cmdq_type++) {
 834                 err = hinic_msg_to_mgmt(&pfhwdev->pf_to_mgmt, HINIC_MOD_COMM,
 835                                         HINIC_COMM_CMD_CMDQ_CTXT_SET,
 836                                         &cmdq_ctxts[cmdq_type],
 837                                         sizeof(cmdq_ctxts[cmdq_type]),
 838                                         NULL, NULL, HINIC_MGMT_MSG_SYNC);
 839                 if (err) {
 840                         dev_err(&pdev->dev, "Failed to set CMDQ CTXT type = %d\n",
 841                                 cmdq_type);
 842                         goto err_write_cmdq_ctxt;
 843                 }
 844         }
 845
 846         devm_kfree(&pdev->dev, cmdq_ctxts);
 847         return 0;
 848
 849 err_write_cmdq_ctxt:
 850         cmdq_type = HINIC_MAX_CMDQ_TYPES;
 851
 852 err_init_cmdq:
 853         for (type = HINIC_CMDQ_SYNC; type < cmdq_type; type++)
 854                 free_cmdq(&cmdqs->cmdq[type]);
 855
 856         devm_kfree(&pdev->dev, cmdq_ctxts);
 857         return err;
 858 }
 859
 860 /**
 861  * hinic_init_cmdqs - init all cmdqs
 862  * @cmdqs: cmdqs to init
 863  * @hwif: HW interface for accessing cmdqs
 864  * @db_area: doorbell areas for all the cmdqs
 865  *
 866  * Return 0 - Success, negative - Failure
 867  **/
 868 int hinic_init_cmdqs(struct hinic_cmdqs *cmdqs, struct hinic_hwif *hwif,
 869                      void __iomem **db_area)
 870 {
 871         struct hinic_func_to_io *func_to_io = cmdqs_to_func_to_io(cmdqs);
 872         struct pci_dev *pdev = hwif->pdev;
 873         struct hinic_hwdev *hwdev;
 874         size_t saved_wqs_size;
 875         u16 max_wqe_size;
 876         int err;
 877
 878         cmdqs->hwif = hwif;
 879         cmdqs->cmdq_buf_pool = pci_pool_create("hinic_cmdq", pdev,
 880                                                HINIC_CMDQ_BUF_SIZE,
 881                                                HINIC_CMDQ_BUF_SIZE, 0);
 882         if (!cmdqs->cmdq_buf_pool)
 883                 return -ENOMEM;
 884
 885         saved_wqs_size = HINIC_MAX_CMDQ_TYPES * sizeof(struct hinic_wq);
 886         cmdqs->saved_wqs = devm_kzalloc(&pdev->dev, saved_wqs_size, GFP_KERNEL);
 887         if (!cmdqs->saved_wqs) {
 888                 err = -ENOMEM;
 889                 goto err_saved_wqs;
 890         }
 891
 892         max_wqe_size = WQE_LCMD_SIZE;
 893         err = hinic_wqs_cmdq_alloc(&cmdqs->cmdq_pages, cmdqs->saved_wqs, hwif,
 894                                    HINIC_MAX_CMDQ_TYPES, CMDQ_WQEBB_SIZE,
 895                                    CMDQ_WQ_PAGE_SIZE, CMDQ_DEPTH, max_wqe_size);
 896         if (err) {
 897                 dev_err(&pdev->dev, "Failed to allocate CMDQ wqs\n");
 898                 goto err_cmdq_wqs;
 899         }
 900
 901         hwdev = container_of(func_to_io, struct hinic_hwdev, func_to_io);
 902         err = init_cmdqs_ctxt(hwdev, cmdqs, db_area);
 903         if (err) {
 904                 dev_err(&pdev->dev, "Failed to write cmdq ctxt\n");
 905                 goto err_cmdq_ctxt;
 906         }
 907
 908         hinic_ceq_register_cb(&func_to_io->ceqs, HINIC_CEQ_CMDQ, cmdqs,
 909                               cmdq_ceq_handler);
 910         return 0;
 911
 912 err_cmdq_ctxt:
 913         hinic_wqs_cmdq_free(&cmdqs->cmdq_pages, cmdqs->saved_wqs,
 914                             HINIC_MAX_CMDQ_TYPES);
 915
 916 err_cmdq_wqs:
 917         devm_kfree(&pdev->dev, cmdqs->saved_wqs);
 918
 919 err_saved_wqs:
 920         pci_pool_destroy(cmdqs->cmdq_buf_pool);
 921         return err;
 922 }
 923
 924 /**
 925  * hinic_free_cmdqs - free all cmdqs
 926  * @cmdqs: cmdqs to free
 927  **/
 928 void hinic_free_cmdqs(struct hinic_cmdqs *cmdqs)
 929 {
 930         struct hinic_func_to_io *func_to_io = cmdqs_to_func_to_io(cmdqs);
 931         struct hinic_hwif *hwif = cmdqs->hwif;
 932         struct pci_dev *pdev = hwif->pdev;
 933         enum hinic_cmdq_type cmdq_type;
 934
 935         hinic_ceq_unregister_cb(&func_to_io->ceqs, HINIC_CEQ_CMDQ);
 936
 937         cmdq_type = HINIC_CMDQ_SYNC;
 938         for (; cmdq_type < HINIC_MAX_CMDQ_TYPES; cmdq_type++)
 939                 free_cmdq(&cmdqs->cmdq[cmdq_type]);
 940
 941         hinic_wqs_cmdq_free(&cmdqs->cmdq_pages, cmdqs->saved_wqs,
 942                             HINIC_MAX_CMDQ_TYPES);
 943
 944         devm_kfree(&pdev->dev, cmdqs->saved_wqs);
 945
 946         pci_pool_destroy(cmdqs->cmdq_buf_pool);
 947 }