drivers/net/ethernet/huawei/hinic/hinic_hw_wq.c

   1 /*
   2  * Huawei HiNIC PCI Express Linux driver
   3  * Copyright(c) 2017 Huawei Technologies Co., Ltd
   4  *
   5  * This program is free software; you can redistribute it and/or modify it
   6  * under the terms and conditions of the GNU General Public License,
   7  * version 2, as published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * for more details.
  13  *
  14  */
  15
  16 #include <linux/kernel.h>
  17 #include <linux/types.h>
  18 #include <linux/pci.h>
  19 #include <linux/device.h>
  20 #include <linux/dma-mapping.h>
  21 #include <linux/slab.h>
  22 #include <linux/atomic.h>
  23 #include <linux/semaphore.h>
  24 #include <linux/errno.h>
  25 #include <linux/vmalloc.h>
  26 #include <linux/err.h>
  27 #include <asm/byteorder.h>
  28
  29 #include "hinic_hw_if.h"
  30 #include "hinic_hw_wqe.h"
  31 #include "hinic_hw_wq.h"
  32 #include "hinic_hw_cmdq.h"
  33
  34 #define WQS_BLOCKS_PER_PAGE             4
  35
  36 #define WQ_BLOCK_SIZE                   4096
  37 #define WQS_PAGE_SIZE                   (WQS_BLOCKS_PER_PAGE * WQ_BLOCK_SIZE)
  38
  39 #define WQS_MAX_NUM_BLOCKS              128
  40 #define WQS_FREE_BLOCKS_SIZE(wqs)       (WQS_MAX_NUM_BLOCKS * \
  41                                          sizeof((wqs)->free_blocks[0]))
  42
  43 #define WQ_SIZE(wq)                     ((wq)->q_depth * (wq)->wqebb_size)
  44
  45 #define WQ_PAGE_ADDR_SIZE               sizeof(u64)
  46 #define WQ_MAX_PAGES                    (WQ_BLOCK_SIZE / WQ_PAGE_ADDR_SIZE)
  47
  48 #define CMDQ_BLOCK_SIZE                 512
  49 #define CMDQ_PAGE_SIZE                  4096
  50
  51 #define CMDQ_WQ_MAX_PAGES               (CMDQ_BLOCK_SIZE / WQ_PAGE_ADDR_SIZE)
  52
  53 #define WQ_BASE_VADDR(wqs, wq)          \
  54                         ((void *)((wqs)->page_vaddr[(wq)->page_idx]) \
  55                                 + (wq)->block_idx * WQ_BLOCK_SIZE)
  56
  57 #define WQ_BASE_PADDR(wqs, wq)          \
  58                         ((wqs)->page_paddr[(wq)->page_idx] \
  59                                 + (wq)->block_idx * WQ_BLOCK_SIZE)
  60
  61 #define WQ_BASE_ADDR(wqs, wq)           \
  62                         ((void *)((wqs)->shadow_page_vaddr[(wq)->page_idx]) \
  63                                 + (wq)->block_idx * WQ_BLOCK_SIZE)
  64
  65 #define CMDQ_BASE_VADDR(cmdq_pages, wq) \
  66                         ((void *)((cmdq_pages)->page_vaddr) \
  67                                 + (wq)->block_idx * CMDQ_BLOCK_SIZE)
  68
  69 #define CMDQ_BASE_PADDR(cmdq_pages, wq) \
  70                         ((cmdq_pages)->page_paddr \
  71                                 + (wq)->block_idx * CMDQ_BLOCK_SIZE)
  72
  73 #define CMDQ_BASE_ADDR(cmdq_pages, wq)  \
  74                         ((void *)((cmdq_pages)->shadow_page_vaddr) \
  75                                 + (wq)->block_idx * CMDQ_BLOCK_SIZE)
  76
  77 #define WQE_PAGE_OFF(wq, idx)   (((idx) & ((wq)->num_wqebbs_per_page - 1)) * \
  78                                         (wq)->wqebb_size)
  79
  80 #define WQE_PAGE_NUM(wq, idx)   (((idx) / ((wq)->num_wqebbs_per_page)) \
  81                                         & ((wq)->num_q_pages - 1))
  82
  83 #define WQ_PAGE_ADDR(wq, idx)           \
  84                         ((wq)->shadow_block_vaddr[WQE_PAGE_NUM(wq, idx)])
  85
  86 #define MASKED_WQE_IDX(wq, idx)         ((idx) & (wq)->mask)
  87
  88 #define WQE_IN_RANGE(wqe, start, end)   \
  89                 (((unsigned long)(wqe) >= (unsigned long)(start)) && \
  90                  ((unsigned long)(wqe) < (unsigned long)(end)))
  91
  92 #define WQE_SHADOW_PAGE(wq, wqe)        \
  93                 (((unsigned long)(wqe) - (unsigned long)(wq)->shadow_wqe) \
  94                         / (wq)->max_wqe_size)
  95
  96 /**
  97  * queue_alloc_page - allocate page for Queue
  98  * @hwif: HW interface for allocating DMA
  99  * @vaddr: virtual address will be returned in this address
 100  * @paddr: physical address will be returned in this address
 101  * @shadow_vaddr: VM area will be return here for holding WQ page addresses
 102  * @page_sz: page size of each WQ page
 103  *
 104  * Return 0 - Success, negative - Failure
 105  **/
 106 static int queue_alloc_page(struct hinic_hwif *hwif, u64 **vaddr, u64 *paddr,
 107                             void ***shadow_vaddr, size_t page_sz)
 108 {
 109         struct pci_dev *pdev = hwif->pdev;
 110         dma_addr_t dma_addr;
 111
 112         *vaddr = dma_zalloc_coherent(&pdev->dev, page_sz, &dma_addr,
 113                                      GFP_KERNEL);
 114         if (!*vaddr) {
 115                 dev_err(&pdev->dev, "Failed to allocate dma for wqs page\n");
 116                 return -ENOMEM;
 117         }
 118
 119         *paddr = (u64)dma_addr;
 120
 121         /* use vzalloc for big mem */
 122         *shadow_vaddr = vzalloc(page_sz);
 123         if (!*shadow_vaddr)
 124                 goto err_shadow_vaddr;
 125
 126         return 0;
 127
 128 err_shadow_vaddr:
 129         dma_free_coherent(&pdev->dev, page_sz, *vaddr, dma_addr);
 130         return -ENOMEM;
 131 }
 132
 133 /**
 134  * wqs_allocate_page - allocate page for WQ set
 135  * @wqs: Work Queue Set
 136  * @page_idx: the page index of the page will be allocated
 137  *
 138  * Return 0 - Success, negative - Failure
 139  **/
 140 static int wqs_allocate_page(struct hinic_wqs *wqs, int page_idx)
 141 {
 142         return queue_alloc_page(wqs->hwif, &wqs->page_vaddr[page_idx],
 143                                 &wqs->page_paddr[page_idx],
 144                                 &wqs->shadow_page_vaddr[page_idx],
 145                                 WQS_PAGE_SIZE);
 146 }
 147
 148 /**
 149  * wqs_free_page - free page of WQ set
 150  * @wqs: Work Queue Set
 151  * @page_idx: the page index of the page will be freed
 152  **/
 153 static void wqs_free_page(struct hinic_wqs *wqs, int page_idx)
 154 {
 155         struct hinic_hwif *hwif = wqs->hwif;
 156         struct pci_dev *pdev = hwif->pdev;
 157
 158         dma_free_coherent(&pdev->dev, WQS_PAGE_SIZE,
 159                           wqs->page_vaddr[page_idx],
 160                           (dma_addr_t)wqs->page_paddr[page_idx]);
 161         vfree(wqs->shadow_page_vaddr[page_idx]);
 162 }
 163
 164 /**
 165  * cmdq_allocate_page - allocate page for cmdq
 166  * @cmdq_pages: the pages of the cmdq queue struct to hold the page
 167  *
 168  * Return 0 - Success, negative - Failure
 169  **/
 170 static int cmdq_allocate_page(struct hinic_cmdq_pages *cmdq_pages)
 171 {
 172         return queue_alloc_page(cmdq_pages->hwif, &cmdq_pages->page_vaddr,
 173                                 &cmdq_pages->page_paddr,
 174                                 &cmdq_pages->shadow_page_vaddr,
 175                                 CMDQ_PAGE_SIZE);
 176 }
 177
 178 /**
 179  * cmdq_free_page - free page from cmdq
 180  * @cmdq_pages: the pages of the cmdq queue struct that hold the page
 181  *
 182  * Return 0 - Success, negative - Failure
 183  **/
 184 static void cmdq_free_page(struct hinic_cmdq_pages *cmdq_pages)
 185 {
 186         struct hinic_hwif *hwif = cmdq_pages->hwif;
 187         struct pci_dev *pdev = hwif->pdev;
 188
 189         dma_free_coherent(&pdev->dev, CMDQ_PAGE_SIZE,
 190                           cmdq_pages->page_vaddr,
 191                           (dma_addr_t)cmdq_pages->page_paddr);
 192         vfree(cmdq_pages->shadow_page_vaddr);
 193 }
 194
 195 static int alloc_page_arrays(struct hinic_wqs *wqs)
 196 {
 197         struct hinic_hwif *hwif = wqs->hwif;
 198         struct pci_dev *pdev = hwif->pdev;
 199         size_t size;
 200
 201         size = wqs->num_pages * sizeof(*wqs->page_paddr);
 202         wqs->page_paddr = devm_kzalloc(&pdev->dev, size, GFP_KERNEL);
 203         if (!wqs->page_paddr)
 204                 return -ENOMEM;
 205
 206         size = wqs->num_pages * sizeof(*wqs->page_vaddr);
 207         wqs->page_vaddr = devm_kzalloc(&pdev->dev, size, GFP_KERNEL);
 208         if (!wqs->page_vaddr)
 209                 goto err_page_vaddr;
 210
 211         size = wqs->num_pages * sizeof(*wqs->shadow_page_vaddr);
 212         wqs->shadow_page_vaddr = devm_kzalloc(&pdev->dev, size, GFP_KERNEL);
 213         if (!wqs->shadow_page_vaddr)
 214                 goto err_page_shadow_vaddr;
 215
 216         return 0;
 217
 218 err_page_shadow_vaddr:
 219         devm_kfree(&pdev->dev, wqs->page_vaddr);
 220
 221 err_page_vaddr:
 222         devm_kfree(&pdev->dev, wqs->page_paddr);
 223         return -ENOMEM;
 224 }
 225
 226 static void free_page_arrays(struct hinic_wqs *wqs)
 227 {
 228         struct hinic_hwif *hwif = wqs->hwif;
 229         struct pci_dev *pdev = hwif->pdev;
 230
 231         devm_kfree(&pdev->dev, wqs->shadow_page_vaddr);
 232         devm_kfree(&pdev->dev, wqs->page_vaddr);
 233         devm_kfree(&pdev->dev, wqs->page_paddr);
 234 }
 235
 236 static int wqs_next_block(struct hinic_wqs *wqs, int *page_idx,
 237                           int *block_idx)
 238 {
 239         int pos;
 240
 241         down(&wqs->alloc_blocks_lock);
 242
 243         wqs->num_free_blks--;
 244
 245         if (wqs->num_free_blks < 0) {
 246                 wqs->num_free_blks++;
 247                 up(&wqs->alloc_blocks_lock);
 248                 return -ENOMEM;
 249         }
 250
 251         pos = wqs->alloc_blk_pos++;
 252         pos &= WQS_MAX_NUM_BLOCKS - 1;
 253
 254         *page_idx = wqs->free_blocks[pos].page_idx;
 255         *block_idx = wqs->free_blocks[pos].block_idx;
 256
 257         wqs->free_blocks[pos].page_idx = -1;
 258         wqs->free_blocks[pos].block_idx = -1;
 259
 260         up(&wqs->alloc_blocks_lock);
 261         return 0;
 262 }
 263
 264 static void wqs_return_block(struct hinic_wqs *wqs, int page_idx,
 265                              int block_idx)
 266 {
 267         int pos;
 268
 269         down(&wqs->alloc_blocks_lock);
 270
 271         pos = wqs->return_blk_pos++;
 272         pos &= WQS_MAX_NUM_BLOCKS - 1;
 273
 274         wqs->free_blocks[pos].page_idx = page_idx;
 275         wqs->free_blocks[pos].block_idx = block_idx;
 276
 277         wqs->num_free_blks++;
 278
 279         up(&wqs->alloc_blocks_lock);
 280 }
 281
 282 static void init_wqs_blocks_arr(struct hinic_wqs *wqs)
 283 {
 284         int page_idx, blk_idx, pos = 0;
 285
 286         for (page_idx = 0; page_idx < wqs->num_pages; page_idx++) {
 287                 for (blk_idx = 0; blk_idx < WQS_BLOCKS_PER_PAGE; blk_idx++) {
 288                         wqs->free_blocks[pos].page_idx = page_idx;
 289                         wqs->free_blocks[pos].block_idx = blk_idx;
 290                         pos++;
 291                 }
 292         }
 293
 294         wqs->alloc_blk_pos = 0;
 295         wqs->return_blk_pos = pos;
 296         wqs->num_free_blks = pos;
 297
 298         sema_init(&wqs->alloc_blocks_lock, 1);
 299 }
 300
 301 /**
 302  * hinic_wqs_alloc - allocate Work Queues set
 303  * @wqs: Work Queue Set
 304  * @max_wqs: maximum wqs to allocate
 305  * @hwif: HW interface for use for the allocation
 306  *
 307  * Return 0 - Success, negative - Failure
 308  **/
 309 int hinic_wqs_alloc(struct hinic_wqs *wqs, int max_wqs,
 310                     struct hinic_hwif *hwif)
 311 {
 312         struct pci_dev *pdev = hwif->pdev;
 313         int err, i, page_idx;
 314
 315         max_wqs = ALIGN(max_wqs, WQS_BLOCKS_PER_PAGE);
 316         if (max_wqs > WQS_MAX_NUM_BLOCKS)  {
 317                 dev_err(&pdev->dev, "Invalid max_wqs = %d\n", max_wqs);
 318                 return -EINVAL;
 319         }
 320
 321         wqs->hwif = hwif;
 322         wqs->num_pages = max_wqs / WQS_BLOCKS_PER_PAGE;
 323
 324         if (alloc_page_arrays(wqs)) {
 325                 dev_err(&pdev->dev,
 326                         "Failed to allocate mem for page addresses\n");
 327                 return -ENOMEM;
 328         }
 329
 330         for (page_idx = 0; page_idx < wqs->num_pages; page_idx++) {
 331                 err = wqs_allocate_page(wqs, page_idx);
 332                 if (err) {
 333                         dev_err(&pdev->dev, "Failed wq page allocation\n");
 334                         goto err_wq_allocate_page;
 335                 }
 336         }
 337
 338         wqs->free_blocks = devm_kzalloc(&pdev->dev, WQS_FREE_BLOCKS_SIZE(wqs),
 339                                         GFP_KERNEL);
 340         if (!wqs->free_blocks) {
 341                 err = -ENOMEM;
 342                 goto err_alloc_blocks;
 343         }
 344
 345         init_wqs_blocks_arr(wqs);
 346         return 0;
 347
 348 err_alloc_blocks:
 349 err_wq_allocate_page:
 350         for (i = 0; i < page_idx; i++)
 351                 wqs_free_page(wqs, i);
 352
 353         free_page_arrays(wqs);
 354         return err;
 355 }
 356
 357 /**
 358  * hinic_wqs_free - free Work Queues set
 359  * @wqs: Work Queue Set
 360  **/
 361 void hinic_wqs_free(struct hinic_wqs *wqs)
 362 {
 363         struct hinic_hwif *hwif = wqs->hwif;
 364         struct pci_dev *pdev = hwif->pdev;
 365         int page_idx;
 366
 367         devm_kfree(&pdev->dev, wqs->free_blocks);
 368
 369         for (page_idx = 0; page_idx < wqs->num_pages; page_idx++)
 370                 wqs_free_page(wqs, page_idx);
 371
 372         free_page_arrays(wqs);
 373 }
 374
 375 /**
 376  * alloc_wqes_shadow - allocate WQE shadows for WQ
 377  * @wq: WQ to allocate shadows for
 378  *
 379  * Return 0 - Success, negative - Failure
 380  **/
 381 static int alloc_wqes_shadow(struct hinic_wq *wq)
 382 {
 383         struct hinic_hwif *hwif = wq->hwif;
 384         struct pci_dev *pdev = hwif->pdev;
 385         size_t size;
 386
 387         size = wq->num_q_pages * wq->max_wqe_size;
 388         wq->shadow_wqe = devm_kzalloc(&pdev->dev, size, GFP_KERNEL);
 389         if (!wq->shadow_wqe)
 390                 return -ENOMEM;
 391
 392         size = wq->num_q_pages * sizeof(wq->prod_idx);
 393         wq->shadow_idx = devm_kzalloc(&pdev->dev, size, GFP_KERNEL);
 394         if (!wq->shadow_idx)
 395                 goto err_shadow_idx;
 396
 397         return 0;
 398
 399 err_shadow_idx:
 400         devm_kfree(&pdev->dev, wq->shadow_wqe);
 401         return -ENOMEM;
 402 }
 403
 404 /**
 405  * free_wqes_shadow - free WQE shadows of WQ
 406  * @wq: WQ to free shadows from
 407  **/
 408 static void free_wqes_shadow(struct hinic_wq *wq)
 409 {
 410         struct hinic_hwif *hwif = wq->hwif;
 411         struct pci_dev *pdev = hwif->pdev;
 412
 413         devm_kfree(&pdev->dev, wq->shadow_idx);
 414         devm_kfree(&pdev->dev, wq->shadow_wqe);
 415 }
 416
 417 /**
 418  * free_wq_pages - free pages of WQ
 419  * @hwif: HW interface for releasing dma addresses
 420  * @wq: WQ to free pages from
 421  * @num_q_pages: number pages to free
 422  **/
 423 static void free_wq_pages(struct hinic_wq *wq, struct hinic_hwif *hwif,
 424                           int num_q_pages)
 425 {
 426         struct pci_dev *pdev = hwif->pdev;
 427         int i;
 428
 429         for (i = 0; i < num_q_pages; i++) {
 430                 void **vaddr = &wq->shadow_block_vaddr[i];
 431                 u64 *paddr = &wq->block_vaddr[i];
 432                 dma_addr_t dma_addr;
 433
 434                 dma_addr = (dma_addr_t)be64_to_cpu(*paddr);
 435                 dma_free_coherent(&pdev->dev, wq->wq_page_size, *vaddr,
 436                                   dma_addr);
 437         }
 438
 439         free_wqes_shadow(wq);
 440 }
 441
 442 /**
 443  * alloc_wq_pages - alloc pages for WQ
 444  * @hwif: HW interface for allocating dma addresses
 445  * @wq: WQ to allocate pages for
 446  * @max_pages: maximum pages allowed
 447  *
 448  * Return 0 - Success, negative - Failure
 449  **/
 450 static int alloc_wq_pages(struct hinic_wq *wq, struct hinic_hwif *hwif,
 451                           int max_pages)
 452 {
 453         struct pci_dev *pdev = hwif->pdev;
 454         int i, err, num_q_pages;
 455
 456         num_q_pages = ALIGN(WQ_SIZE(wq), wq->wq_page_size) / wq->wq_page_size;
 457         if (num_q_pages > max_pages) {
 458                 dev_err(&pdev->dev, "Number wq pages exceeds the limit\n");
 459                 return -EINVAL;
 460         }
 461
 462         if (num_q_pages & (num_q_pages - 1)) {
 463                 dev_err(&pdev->dev, "Number wq pages must be power of 2\n");
 464                 return -EINVAL;
 465         }
 466
 467         wq->num_q_pages = num_q_pages;
 468
 469         err = alloc_wqes_shadow(wq);
 470         if (err) {
 471                 dev_err(&pdev->dev, "Failed to allocate wqe shadow\n");
 472                 return err;
 473         }
 474
 475         for (i = 0; i < num_q_pages; i++) {
 476                 void **vaddr = &wq->shadow_block_vaddr[i];
 477                 u64 *paddr = &wq->block_vaddr[i];
 478                 dma_addr_t dma_addr;
 479
 480                 *vaddr = dma_zalloc_coherent(&pdev->dev, wq->wq_page_size,
 481                                              &dma_addr, GFP_KERNEL);
 482                 if (!*vaddr) {
 483                         dev_err(&pdev->dev, "Failed to allocate wq page\n");
 484                         goto err_alloc_wq_pages;
 485                 }
 486
 487                 /* HW uses Big Endian Format */
 488                 *paddr = cpu_to_be64(dma_addr);
 489         }
 490
 491         return 0;
 492
 493 err_alloc_wq_pages:
 494         free_wq_pages(wq, hwif, i);
 495         return -ENOMEM;
 496 }
 497
 498 /**
 499  * hinic_wq_allocate - Allocate the WQ resources from the WQS
 500  * @wqs: WQ set from which to allocate the WQ resources
 501  * @wq: WQ to allocate resources for it from the WQ set
 502  * @wqebb_size: Work Queue Block Byte Size
 503  * @wq_page_size: the page size in the Work Queue
 504  * @q_depth: number of wqebbs in WQ
 505  * @max_wqe_size: maximum WQE size that will be used in the WQ
 506  *
 507  * Return 0 - Success, negative - Failure
 508  **/
 509 int hinic_wq_allocate(struct hinic_wqs *wqs, struct hinic_wq *wq,
 510                       u16 wqebb_size, u16 wq_page_size, u16 q_depth,
 511                       u16 max_wqe_size)
 512 {
 513         struct hinic_hwif *hwif = wqs->hwif;
 514         struct pci_dev *pdev = hwif->pdev;
 515         u16 num_wqebbs_per_page;
 516         int err;
 517
 518         if (wqebb_size == 0) {
 519                 dev_err(&pdev->dev, "wqebb_size must be > 0\n");
 520                 return -EINVAL;
 521         }
 522
 523         if (wq_page_size == 0) {
 524                 dev_err(&pdev->dev, "wq_page_size must be > 0\n");
 525                 return -EINVAL;
 526         }
 527
 528         if (q_depth & (q_depth - 1)) {
 529                 dev_err(&pdev->dev, "WQ q_depth must be power of 2\n");
 530                 return -EINVAL;
 531         }
 532
 533         num_wqebbs_per_page = ALIGN(wq_page_size, wqebb_size) / wqebb_size;
 534
 535         if (num_wqebbs_per_page & (num_wqebbs_per_page - 1)) {
 536                 dev_err(&pdev->dev, "num wqebbs per page must be power of 2\n");
 537                 return -EINVAL;
 538         }
 539
 540         wq->hwif = hwif;
 541
 542         err = wqs_next_block(wqs, &wq->page_idx, &wq->block_idx);
 543         if (err) {
 544                 dev_err(&pdev->dev, "Failed to get free wqs next block\n");
 545                 return err;
 546         }
 547
 548         wq->wqebb_size = wqebb_size;
 549         wq->wq_page_size = wq_page_size;
 550         wq->q_depth = q_depth;
 551         wq->max_wqe_size = max_wqe_size;
 552         wq->num_wqebbs_per_page = num_wqebbs_per_page;
 553
 554         wq->block_vaddr = WQ_BASE_VADDR(wqs, wq);
 555         wq->shadow_block_vaddr = WQ_BASE_ADDR(wqs, wq);
 556         wq->block_paddr = WQ_BASE_PADDR(wqs, wq);
 557
 558         err = alloc_wq_pages(wq, wqs->hwif, WQ_MAX_PAGES);
 559         if (err) {
 560                 dev_err(&pdev->dev, "Failed to allocate wq pages\n");
 561                 goto err_alloc_wq_pages;
 562         }
 563
 564         atomic_set(&wq->cons_idx, 0);
 565         atomic_set(&wq->prod_idx, 0);
 566         atomic_set(&wq->delta, q_depth);
 567         wq->mask = q_depth - 1;
 568
 569         return 0;
 570
 571 err_alloc_wq_pages:
 572         wqs_return_block(wqs, wq->page_idx, wq->block_idx);
 573         return err;
 574 }
 575
 576 /**
 577  * hinic_wq_free - Free the WQ resources to the WQS
 578  * @wqs: WQ set to free the WQ resources to it
 579  * @wq: WQ to free its resources to the WQ set resources
 580  **/
 581 void hinic_wq_free(struct hinic_wqs *wqs, struct hinic_wq *wq)
 582 {
 583         free_wq_pages(wq, wqs->hwif, wq->num_q_pages);
 584
 585         wqs_return_block(wqs, wq->page_idx, wq->block_idx);
 586 }
 587
 588 /**
 589  * hinic_wqs_cmdq_alloc - Allocate wqs for cmdqs
 590  * @cmdq_pages: will hold the pages of the cmdq
 591  * @wq: returned wqs
 592  * @hwif: HW interface
 593  * @cmdq_blocks: number of cmdq blocks/wq to allocate
 594  * @wqebb_size: Work Queue Block Byte Size
 595  * @wq_page_size: the page size in the Work Queue
 596  * @q_depth: number of wqebbs in WQ
 597  * @max_wqe_size: maximum WQE size that will be used in the WQ
 598  *
 599  * Return 0 - Success, negative - Failure
 600  **/
 601 int hinic_wqs_cmdq_alloc(struct hinic_cmdq_pages *cmdq_pages,
 602                          struct hinic_wq *wq, struct hinic_hwif *hwif,
 603                          int cmdq_blocks, u16 wqebb_size, u16 wq_page_size,
 604                          u16 q_depth, u16 max_wqe_size)
 605 {
 606         struct pci_dev *pdev = hwif->pdev;
 607         u16 num_wqebbs_per_page;
 608         int i, j, err = -ENOMEM;
 609
 610         if (wqebb_size == 0) {
 611                 dev_err(&pdev->dev, "wqebb_size must be > 0\n");
 612                 return -EINVAL;
 613         }
 614
 615         if (wq_page_size == 0) {
 616                 dev_err(&pdev->dev, "wq_page_size must be > 0\n");
 617                 return -EINVAL;
 618         }
 619
 620         if (q_depth & (q_depth - 1)) {
 621                 dev_err(&pdev->dev, "WQ q_depth must be power of 2\n");
 622                 return -EINVAL;
 623         }
 624
 625         num_wqebbs_per_page = ALIGN(wq_page_size, wqebb_size) / wqebb_size;
 626
 627         if (num_wqebbs_per_page & (num_wqebbs_per_page - 1)) {
 628                 dev_err(&pdev->dev, "num wqebbs per page must be power of 2\n");
 629                 return -EINVAL;
 630         }
 631
 632         cmdq_pages->hwif = hwif;
 633
 634         err = cmdq_allocate_page(cmdq_pages);
 635         if (err) {
 636                 dev_err(&pdev->dev, "Failed to allocate CMDQ page\n");
 637                 return err;
 638         }
 639
 640         for (i = 0; i < cmdq_blocks; i++) {
 641                 wq[i].hwif = hwif;
 642                 wq[i].page_idx = 0;
 643                 wq[i].block_idx = i;
 644
 645                 wq[i].wqebb_size = wqebb_size;
 646                 wq[i].wq_page_size = wq_page_size;
 647                 wq[i].q_depth = q_depth;
 648                 wq[i].max_wqe_size = max_wqe_size;
 649                 wq[i].num_wqebbs_per_page = num_wqebbs_per_page;
 650
 651                 wq[i].block_vaddr = CMDQ_BASE_VADDR(cmdq_pages, &wq[i]);
 652                 wq[i].shadow_block_vaddr = CMDQ_BASE_ADDR(cmdq_pages, &wq[i]);
 653                 wq[i].block_paddr = CMDQ_BASE_PADDR(cmdq_pages, &wq[i]);
 654
 655                 err = alloc_wq_pages(&wq[i], cmdq_pages->hwif,
 656                                      CMDQ_WQ_MAX_PAGES);
 657                 if (err) {
 658                         dev_err(&pdev->dev, "Failed to alloc CMDQ blocks\n");
 659                         goto err_cmdq_block;
 660                 }
 661
 662                 atomic_set(&wq[i].cons_idx, 0);
 663                 atomic_set(&wq[i].prod_idx, 0);
 664                 atomic_set(&wq[i].delta, q_depth);
 665                 wq[i].mask = q_depth - 1;
 666         }
 667
 668         return 0;
 669
 670 err_cmdq_block:
 671         for (j = 0; j < i; j++)
 672                 free_wq_pages(&wq[j], cmdq_pages->hwif, wq[j].num_q_pages);
 673
 674         cmdq_free_page(cmdq_pages);
 675         return err;
 676 }
 677
 678 /**
 679  * hinic_wqs_cmdq_free - Free wqs from cmdqs
 680  * @cmdq_pages: hold the pages of the cmdq
 681  * @wq: wqs to free
 682  * @cmdq_blocks: number of wqs to free
 683  **/
 684 void hinic_wqs_cmdq_free(struct hinic_cmdq_pages *cmdq_pages,
 685                          struct hinic_wq *wq, int cmdq_blocks)
 686 {
 687         int i;
 688
 689         for (i = 0; i < cmdq_blocks; i++)
 690                 free_wq_pages(&wq[i], cmdq_pages->hwif, wq[i].num_q_pages);
 691
 692         cmdq_free_page(cmdq_pages);
 693 }
 694
 695 static void copy_wqe_to_shadow(struct hinic_wq *wq, void *shadow_addr,
 696                                int num_wqebbs, u16 idx)
 697 {
 698         void *wqebb_addr;
 699         int i;
 700
 701         for (i = 0; i < num_wqebbs; i++, idx++) {
 702                 idx = MASKED_WQE_IDX(wq, idx);
 703                 wqebb_addr = WQ_PAGE_ADDR(wq, idx) +
 704                              WQE_PAGE_OFF(wq, idx);
 705
 706                 memcpy(shadow_addr, wqebb_addr, wq->wqebb_size);
 707
 708                 shadow_addr += wq->wqebb_size;
 709         }
 710 }
 711
 712 static void copy_wqe_from_shadow(struct hinic_wq *wq, void *shadow_addr,
 713                                  int num_wqebbs, u16 idx)
 714 {
 715         void *wqebb_addr;
 716         int i;
 717
 718         for (i = 0; i < num_wqebbs; i++, idx++) {
 719                 idx = MASKED_WQE_IDX(wq, idx);
 720                 wqebb_addr = WQ_PAGE_ADDR(wq, idx) +
 721                              WQE_PAGE_OFF(wq, idx);
 722
 723                 memcpy(wqebb_addr, shadow_addr, wq->wqebb_size);
 724                 shadow_addr += wq->wqebb_size;
 725         }
 726 }
 727
 728 /**
 729  * hinic_get_wqe - get wqe ptr in the current pi and update the pi
 730  * @wq: wq to get wqe from
 731  * @wqe_size: wqe size
 732  * @prod_idx: returned pi
 733  *
 734  * Return wqe pointer
 735  **/
 736 struct hinic_hw_wqe *hinic_get_wqe(struct hinic_wq *wq, unsigned int wqe_size,
 737                                    u16 *prod_idx)
 738 {
 739         int curr_pg, end_pg, num_wqebbs;
 740         u16 curr_prod_idx, end_prod_idx;
 741
 742         *prod_idx = MASKED_WQE_IDX(wq, atomic_read(&wq->prod_idx));
 743
 744         num_wqebbs = ALIGN(wqe_size, wq->wqebb_size) / wq->wqebb_size;
 745
 746         if (atomic_sub_return(num_wqebbs, &wq->delta) <= 0) {
 747                 atomic_add(num_wqebbs, &wq->delta);
 748                 return ERR_PTR(-EBUSY);
 749         }
 750
 751         end_prod_idx = atomic_add_return(num_wqebbs, &wq->prod_idx);
 752
 753         end_prod_idx = MASKED_WQE_IDX(wq, end_prod_idx);
 754         curr_prod_idx = end_prod_idx - num_wqebbs;
 755         curr_prod_idx = MASKED_WQE_IDX(wq, curr_prod_idx);
 756
 757         /* end prod index points to the next wqebb, therefore minus 1 */
 758         end_prod_idx = MASKED_WQE_IDX(wq, end_prod_idx - 1);
 759
 760         curr_pg = WQE_PAGE_NUM(wq, curr_prod_idx);
 761         end_pg = WQE_PAGE_NUM(wq, end_prod_idx);
 762
 763         *prod_idx = curr_prod_idx;
 764
 765         if (curr_pg != end_pg) {
 766                 void *shadow_addr = &wq->shadow_wqe[curr_pg * wq->max_wqe_size];
 767
 768                 copy_wqe_to_shadow(wq, shadow_addr, num_wqebbs, *prod_idx);
 769
 770                 wq->shadow_idx[curr_pg] = *prod_idx;
 771                 return shadow_addr;
 772         }
 773
 774         return WQ_PAGE_ADDR(wq, *prod_idx) + WQE_PAGE_OFF(wq, *prod_idx);
 775 }
 776
 777 /**
 778  * hinic_put_wqe - return the wqe place to use for a new wqe
 779  * @wq: wq to return wqe
 780  * @wqe_size: wqe size
 781  **/
 782 void hinic_put_wqe(struct hinic_wq *wq, unsigned int wqe_size)
 783 {
 784         int num_wqebbs = ALIGN(wqe_size, wq->wqebb_size) / wq->wqebb_size;
 785
 786         atomic_add(num_wqebbs, &wq->cons_idx);
 787
 788         atomic_add(num_wqebbs, &wq->delta);
 789 }
 790
 791 /**
 792  * hinic_read_wqe - read wqe ptr in the current ci
 793  * @wq: wq to get read from
 794  * @wqe_size: wqe size
 795  * @cons_idx: returned ci
 796  *
 797  * Return wqe pointer
 798  **/
 799 struct hinic_hw_wqe *hinic_read_wqe(struct hinic_wq *wq, unsigned int wqe_size,
 800                                     u16 *cons_idx)
 801 {
 802         int num_wqebbs = ALIGN(wqe_size, wq->wqebb_size) / wq->wqebb_size;
 803         u16 curr_cons_idx, end_cons_idx;
 804         int curr_pg, end_pg;
 805
 806         if ((atomic_read(&wq->delta) + num_wqebbs) > wq->q_depth)
 807                 return ERR_PTR(-EBUSY);
 808
 809         curr_cons_idx = atomic_read(&wq->cons_idx);
 810
 811         curr_cons_idx = MASKED_WQE_IDX(wq, curr_cons_idx);
 812         end_cons_idx = MASKED_WQE_IDX(wq, curr_cons_idx + num_wqebbs - 1);
 813
 814         curr_pg = WQE_PAGE_NUM(wq, curr_cons_idx);
 815         end_pg = WQE_PAGE_NUM(wq, end_cons_idx);
 816
 817         *cons_idx = curr_cons_idx;
 818
 819         if (curr_pg != end_pg) {
 820                 void *shadow_addr = &wq->shadow_wqe[curr_pg * wq->max_wqe_size];
 821
 822                 copy_wqe_to_shadow(wq, shadow_addr, num_wqebbs, *cons_idx);
 823                 return shadow_addr;
 824         }
 825
 826         return WQ_PAGE_ADDR(wq, *cons_idx) + WQE_PAGE_OFF(wq, *cons_idx);
 827 }
 828
 829 /**
 830  * hinic_read_wqe_direct - read wqe directly from ci position
 831  * @wq: wq
 832  * @cons_idx: ci position
 833  *
 834  * Return wqe
 835  **/
 836 struct hinic_hw_wqe *hinic_read_wqe_direct(struct hinic_wq *wq, u16 cons_idx)
 837 {
 838         return WQ_PAGE_ADDR(wq, cons_idx) + WQE_PAGE_OFF(wq, cons_idx);
 839 }
 840
 841 /**
 842  * wqe_shadow - check if a wqe is shadow
 843  * @wq: wq of the wqe
 844  * @wqe: the wqe for shadow checking
 845  *
 846  * Return true - shadow, false - Not shadow
 847  **/
 848 static inline bool wqe_shadow(struct hinic_wq *wq, struct hinic_hw_wqe *wqe)
 849 {
 850         size_t wqe_shadow_size = wq->num_q_pages * wq->max_wqe_size;
 851
 852         return WQE_IN_RANGE(wqe, wq->shadow_wqe,
 853                             &wq->shadow_wqe[wqe_shadow_size]);
 854 }
 855
 856 /**
 857  * hinic_write_wqe - write the wqe to the wq
 858  * @wq: wq to write wqe to
 859  * @wqe: wqe to write
 860  * @wqe_size: wqe size
 861  **/
 862 void hinic_write_wqe(struct hinic_wq *wq, struct hinic_hw_wqe *wqe,
 863                      unsigned int wqe_size)
 864 {
 865         int curr_pg, num_wqebbs;
 866         void *shadow_addr;
 867         u16 prod_idx;
 868
 869         if (wqe_shadow(wq, wqe)) {
 870                 curr_pg = WQE_SHADOW_PAGE(wq, wqe);
 871
 872                 prod_idx = wq->shadow_idx[curr_pg];
 873                 num_wqebbs = ALIGN(wqe_size, wq->wqebb_size) / wq->wqebb_size;
 874                 shadow_addr = &wq->shadow_wqe[curr_pg * wq->max_wqe_size];
 875
 876                 copy_wqe_from_shadow(wq, shadow_addr, num_wqebbs, prod_idx);
 877         }
 878 }