drivers/net/ethernet/cavium/liquidio/request_manager.c

   1 /**********************************************************************
   2  * Author: Cavium, Inc.
   3  *
   4  * Contact: support@cavium.com
   5  *          Please include "LiquidIO" in the subject.
   6  *
   7  * Copyright (c) 2003-2016 Cavium, Inc.
   8  *
   9  * This file is free software; you can redistribute it and/or modify
  10  * it under the terms of the GNU General Public License, Version 2, as
  11  * published by the Free Software Foundation.
  12  *
  13  * This file is distributed in the hope that it will be useful, but
  14  * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty
  15  * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or
  16  * NONINFRINGEMENT.  See the GNU General Public License for more
  17  * details.
  18  **********************************************************************/
  19 #include <linux/pci.h>
  20 #include <linux/netdevice.h>
  21 #include <linux/vmalloc.h>
  22 #include "liquidio_common.h"
  23 #include "octeon_droq.h"
  24 #include "octeon_iq.h"
  25 #include "response_manager.h"
  26 #include "octeon_device.h"
  27 #include "octeon_main.h"
  28 #include "octeon_network.h"
  29 #include "cn66xx_device.h"
  30 #include "cn23xx_pf_device.h"
  31 #include "cn23xx_vf_device.h"
  32
  33 struct iq_post_status {
  34         int status;
  35         int index;
  36 };
  37
  38 static void check_db_timeout(struct work_struct *work);
  39 static void  __check_db_timeout(struct octeon_device *oct, u64 iq_no);
  40
  41 static void (*reqtype_free_fn[MAX_OCTEON_DEVICES][REQTYPE_LAST + 1]) (void *);
  42
  43 static inline int IQ_INSTR_MODE_64B(struct octeon_device *oct, int iq_no)
  44 {
  45         struct octeon_instr_queue *iq =
  46             (struct octeon_instr_queue *)oct->instr_queue[iq_no];
  47         return iq->iqcmd_64B;
  48 }
  49
  50 #define IQ_INSTR_MODE_32B(oct, iq_no)  (!IQ_INSTR_MODE_64B(oct, iq_no))
  51
  52 /* Define this to return the request status comaptible to old code */
  53 /*#define OCTEON_USE_OLD_REQ_STATUS*/
  54
  55 /* Return 0 on success, 1 on failure */
  56 int octeon_init_instr_queue(struct octeon_device *oct,
  57                             union oct_txpciq txpciq,
  58                             u32 num_descs)
  59 {
  60         struct octeon_instr_queue *iq;
  61         struct octeon_iq_config *conf = NULL;
  62         u32 iq_no = (u32)txpciq.s.q_no;
  63         u32 q_size;
  64         struct cavium_wq *db_wq;
  65         int numa_node = dev_to_node(&oct->pci_dev->dev);
  66
  67         if (OCTEON_CN6XXX(oct))
  68                 conf = &(CFG_GET_IQ_CFG(CHIP_CONF(oct, cn6xxx)));
  69         else if (OCTEON_CN23XX_PF(oct))
  70                 conf = &(CFG_GET_IQ_CFG(CHIP_CONF(oct, cn23xx_pf)));
  71         else if (OCTEON_CN23XX_VF(oct))
  72                 conf = &(CFG_GET_IQ_CFG(CHIP_CONF(oct, cn23xx_vf)));
  73
  74         if (!conf) {
  75                 dev_err(&oct->pci_dev->dev, "Unsupported Chip %x\n",
  76                         oct->chip_id);
  77                 return 1;
  78         }
  79
  80         q_size = (u32)conf->instr_type * num_descs;
  81
  82         iq = oct->instr_queue[iq_no];
  83
  84         iq->oct_dev = oct;
  85
  86         iq->base_addr = lio_dma_alloc(oct, q_size, &iq->base_addr_dma);
  87         if (!iq->base_addr) {
  88                 dev_err(&oct->pci_dev->dev, "Cannot allocate memory for instr queue %d\n",
  89                         iq_no);
  90                 return 1;
  91         }
  92
  93         iq->max_count = num_descs;
  94
  95         /* Initialize a list to holds requests that have been posted to Octeon
  96          * but has yet to be fetched by octeon
  97          */
  98         iq->request_list = vmalloc_node((sizeof(*iq->request_list) * num_descs),
  99                                                numa_node);
 100         if (!iq->request_list)
 101                 iq->request_list =
 102                         vmalloc(array_size(num_descs,
 103                                            sizeof(*iq->request_list)));
 104         if (!iq->request_list) {
 105                 lio_dma_free(oct, q_size, iq->base_addr, iq->base_addr_dma);
 106                 dev_err(&oct->pci_dev->dev, "Alloc failed for IQ[%d] nr free list\n",
 107                         iq_no);
 108                 return 1;
 109         }
 110
 111         memset(iq->request_list, 0, sizeof(*iq->request_list) * num_descs);
 112
 113         dev_dbg(&oct->pci_dev->dev, "IQ[%d]: base: %p basedma: %pad count: %d\n",
 114                 iq_no, iq->base_addr, &iq->base_addr_dma, iq->max_count);
 115
 116         iq->txpciq.u64 = txpciq.u64;
 117         iq->fill_threshold = (u32)conf->db_min;
 118         iq->fill_cnt = 0;
 119         iq->host_write_index = 0;
 120         iq->octeon_read_index = 0;
 121         iq->flush_index = 0;
 122         iq->last_db_time = 0;
 123         iq->do_auto_flush = 1;
 124         iq->db_timeout = (u32)conf->db_timeout;
 125         atomic_set(&iq->instr_pending, 0);
 126         iq->pkts_processed = 0;
 127
 128         /* Initialize the spinlock for this instruction queue */
 129         spin_lock_init(&iq->lock);
 130         if (iq_no == 0) {
 131                 iq->allow_soft_cmds = true;
 132                 spin_lock_init(&iq->post_lock);
 133         } else {
 134                 iq->allow_soft_cmds = false;
 135         }
 136
 137         spin_lock_init(&iq->iq_flush_running_lock);
 138
 139         oct->io_qmask.iq |= BIT_ULL(iq_no);
 140
 141         /* Set the 32B/64B mode for each input queue */
 142         oct->io_qmask.iq64B |= ((conf->instr_type == 64) << iq_no);
 143         iq->iqcmd_64B = (conf->instr_type == 64);
 144
 145         oct->fn_list.setup_iq_regs(oct, iq_no);
 146
 147         oct->check_db_wq[iq_no].wq = alloc_workqueue("check_iq_db",
 148                                                      WQ_MEM_RECLAIM,
 149                                                      0);
 150         if (!oct->check_db_wq[iq_no].wq) {
 151                 vfree(iq->request_list);
 152                 iq->request_list = NULL;
 153                 lio_dma_free(oct, q_size, iq->base_addr, iq->base_addr_dma);
 154                 dev_err(&oct->pci_dev->dev, "check db wq create failed for iq %d\n",
 155                         iq_no);
 156                 return 1;
 157         }
 158
 159         db_wq = &oct->check_db_wq[iq_no];
 160
 161         INIT_DELAYED_WORK(&db_wq->wk.work, check_db_timeout);
 162         db_wq->wk.ctxptr = oct;
 163         db_wq->wk.ctxul = iq_no;
 164         queue_delayed_work(db_wq->wq, &db_wq->wk.work, msecs_to_jiffies(1));
 165
 166         return 0;
 167 }
 168
 169 int octeon_delete_instr_queue(struct octeon_device *oct, u32 iq_no)
 170 {
 171         u64 desc_size = 0, q_size;
 172         struct octeon_instr_queue *iq = oct->instr_queue[iq_no];
 173
 174         cancel_delayed_work_sync(&oct->check_db_wq[iq_no].wk.work);
 175         destroy_workqueue(oct->check_db_wq[iq_no].wq);
 176
 177         if (OCTEON_CN6XXX(oct))
 178                 desc_size =
 179                     CFG_GET_IQ_INSTR_TYPE(CHIP_CONF(oct, cn6xxx));
 180         else if (OCTEON_CN23XX_PF(oct))
 181                 desc_size =
 182                     CFG_GET_IQ_INSTR_TYPE(CHIP_CONF(oct, cn23xx_pf));
 183         else if (OCTEON_CN23XX_VF(oct))
 184                 desc_size =
 185                     CFG_GET_IQ_INSTR_TYPE(CHIP_CONF(oct, cn23xx_vf));
 186
 187         vfree(iq->request_list);
 188
 189         if (iq->base_addr) {
 190                 q_size = iq->max_count * desc_size;
 191                 lio_dma_free(oct, (u32)q_size, iq->base_addr,
 192                              iq->base_addr_dma);
 193                 oct->io_qmask.iq &= ~(1ULL << iq_no);
 194                 vfree(oct->instr_queue[iq_no]);
 195                 oct->instr_queue[iq_no] = NULL;
 196                 oct->num_iqs--;
 197                 return 0;
 198         }
 199         return 1;
 200 }
 201
 202 /* Return 0 on success, 1 on failure */
 203 int octeon_setup_iq(struct octeon_device *oct,
 204                     int ifidx,
 205                     int q_index,
 206                     union oct_txpciq txpciq,
 207                     u32 num_descs,
 208                     void *app_ctx)
 209 {
 210         u32 iq_no = (u32)txpciq.s.q_no;
 211         int numa_node = dev_to_node(&oct->pci_dev->dev);
 212
 213         if (oct->instr_queue[iq_no]) {
 214                 dev_dbg(&oct->pci_dev->dev, "IQ is in use. Cannot create the IQ: %d again\n",
 215                         iq_no);
 216                 oct->instr_queue[iq_no]->txpciq.u64 = txpciq.u64;
 217                 oct->instr_queue[iq_no]->app_ctx = app_ctx;
 218                 return 0;
 219         }
 220         oct->instr_queue[iq_no] =
 221             vmalloc_node(sizeof(struct octeon_instr_queue), numa_node);
 222         if (!oct->instr_queue[iq_no])
 223                 oct->instr_queue[iq_no] =
 224                     vmalloc(sizeof(struct octeon_instr_queue));
 225         if (!oct->instr_queue[iq_no])
 226                 return 1;
 227
 228         memset(oct->instr_queue[iq_no], 0,
 229                sizeof(struct octeon_instr_queue));
 230
 231         oct->instr_queue[iq_no]->q_index = q_index;
 232         oct->instr_queue[iq_no]->app_ctx = app_ctx;
 233         oct->instr_queue[iq_no]->ifidx = ifidx;
 234
 235         if (octeon_init_instr_queue(oct, txpciq, num_descs)) {
 236                 vfree(oct->instr_queue[iq_no]);
 237                 oct->instr_queue[iq_no] = NULL;
 238                 return 1;
 239         }
 240
 241         oct->num_iqs++;
 242         if (oct->fn_list.enable_io_queues(oct)) {
 243                 octeon_delete_instr_queue(oct, iq_no);
 244                 return 1;
 245         }
 246
 247         return 0;
 248 }
 249
 250 int lio_wait_for_instr_fetch(struct octeon_device *oct)
 251 {
 252         int i, retry = 1000, pending, instr_cnt = 0;
 253
 254         do {
 255                 instr_cnt = 0;
 256
 257                 for (i = 0; i < MAX_OCTEON_INSTR_QUEUES(oct); i++) {
 258                         if (!(oct->io_qmask.iq & BIT_ULL(i)))
 259                                 continue;
 260                         pending =
 261                             atomic_read(&oct->instr_queue[i]->instr_pending);
 262                         if (pending)
 263                                 __check_db_timeout(oct, i);
 264                         instr_cnt += pending;
 265                 }
 266
 267                 if (instr_cnt == 0)
 268                         break;
 269
 270                 schedule_timeout_uninterruptible(1);
 271
 272         } while (retry-- && instr_cnt);
 273
 274         return instr_cnt;
 275 }
 276
 277 static inline void
 278 ring_doorbell(struct octeon_device *oct, struct octeon_instr_queue *iq)
 279 {
 280         if (atomic_read(&oct->status) == OCT_DEV_RUNNING) {
 281                 writel(iq->fill_cnt, iq->doorbell_reg);
 282                 /* make sure doorbell write goes through */
 283                 mmiowb();
 284                 iq->fill_cnt = 0;
 285                 iq->last_db_time = jiffies;
 286                 return;
 287         }
 288 }
 289
 290 void
 291 octeon_ring_doorbell_locked(struct octeon_device *oct, u32 iq_no)
 292 {
 293         struct octeon_instr_queue *iq;
 294
 295         iq = oct->instr_queue[iq_no];
 296         spin_lock(&iq->post_lock);
 297         if (iq->fill_cnt)
 298                 ring_doorbell(oct, iq);
 299         spin_unlock(&iq->post_lock);
 300 }
 301
 302 static inline void __copy_cmd_into_iq(struct octeon_instr_queue *iq,
 303                                       u8 *cmd)
 304 {
 305         u8 *iqptr, cmdsize;
 306
 307         cmdsize = ((iq->iqcmd_64B) ? 64 : 32);
 308         iqptr = iq->base_addr + (cmdsize * iq->host_write_index);
 309
 310         memcpy(iqptr, cmd, cmdsize);
 311 }
 312
 313 static inline struct iq_post_status
 314 __post_command2(struct octeon_instr_queue *iq, u8 *cmd)
 315 {
 316         struct iq_post_status st;
 317
 318         st.status = IQ_SEND_OK;
 319
 320         /* This ensures that the read index does not wrap around to the same
 321          * position if queue gets full before Octeon could fetch any instr.
 322          */
 323         if (atomic_read(&iq->instr_pending) >= (s32)(iq->max_count - 1)) {
 324                 st.status = IQ_SEND_FAILED;
 325                 st.index = -1;
 326                 return st;
 327         }
 328
 329         if (atomic_read(&iq->instr_pending) >= (s32)(iq->max_count - 2))
 330                 st.status = IQ_SEND_STOP;
 331
 332         __copy_cmd_into_iq(iq, cmd);
 333
 334         /* "index" is returned, host_write_index is modified. */
 335         st.index = iq->host_write_index;
 336         iq->host_write_index = incr_index(iq->host_write_index, 1,
 337                                           iq->max_count);
 338         iq->fill_cnt++;
 339
 340         /* Flush the command into memory. We need to be sure the data is in
 341          * memory before indicating that the instruction is pending.
 342          */
 343         wmb();
 344
 345         atomic_inc(&iq->instr_pending);
 346
 347         return st;
 348 }
 349
 350 int
 351 octeon_register_reqtype_free_fn(struct octeon_device *oct, int reqtype,
 352                                 void (*fn)(void *))
 353 {
 354         if (reqtype > REQTYPE_LAST) {
 355                 dev_err(&oct->pci_dev->dev, "%s: Invalid reqtype: %d\n",
 356                         __func__, reqtype);
 357                 return -EINVAL;
 358         }
 359
 360         reqtype_free_fn[oct->octeon_id][reqtype] = fn;
 361
 362         return 0;
 363 }
 364
 365 static inline void
 366 __add_to_request_list(struct octeon_instr_queue *iq,
 367                       int idx, void *buf, int reqtype)
 368 {
 369         iq->request_list[idx].buf = buf;
 370         iq->request_list[idx].reqtype = reqtype;
 371 }
 372
 373 /* Can only run in process context */
 374 int
 375 lio_process_iq_request_list(struct octeon_device *oct,
 376                             struct octeon_instr_queue *iq, u32 napi_budget)
 377 {
 378         struct cavium_wq *cwq = &oct->dma_comp_wq;
 379         int reqtype;
 380         void *buf;
 381         u32 old = iq->flush_index;
 382         u32 inst_count = 0;
 383         unsigned int pkts_compl = 0, bytes_compl = 0;
 384         struct octeon_soft_command *sc;
 385         struct octeon_instr_irh *irh;
 386         unsigned long flags;
 387
 388         while (old != iq->octeon_read_index) {
 389                 reqtype = iq->request_list[old].reqtype;
 390                 buf     = iq->request_list[old].buf;
 391
 392                 if (reqtype == REQTYPE_NONE)
 393                         goto skip_this;
 394
 395                 octeon_update_tx_completion_counters(buf, reqtype, &pkts_compl,
 396                                                      &bytes_compl);
 397
 398                 switch (reqtype) {
 399                 case REQTYPE_NORESP_NET:
 400                 case REQTYPE_NORESP_NET_SG:
 401                 case REQTYPE_RESP_NET_SG:
 402                         reqtype_free_fn[oct->octeon_id][reqtype](buf);
 403                         break;
 404                 case REQTYPE_RESP_NET:
 405                 case REQTYPE_SOFT_COMMAND:
 406                         sc = buf;
 407
 408                         if (OCTEON_CN23XX_PF(oct) || OCTEON_CN23XX_VF(oct))
 409                                 irh = (struct octeon_instr_irh *)
 410                                         &sc->cmd.cmd3.irh;
 411                         else
 412                                 irh = (struct octeon_instr_irh *)
 413                                         &sc->cmd.cmd2.irh;
 414                         if (irh->rflag) {
 415                                 /* We're expecting a response from Octeon.
 416                                  * It's up to lio_process_ordered_list() to
 417                                  * process  sc. Add sc to the ordered soft
 418                                  * command response list because we expect
 419                                  * a response from Octeon.
 420                                  */
 421                                 spin_lock_irqsave
 422                                         (&oct->response_list
 423                                          [OCTEON_ORDERED_SC_LIST].lock,
 424                                          flags);
 425                                 atomic_inc(&oct->response_list
 426                                         [OCTEON_ORDERED_SC_LIST].
 427                                         pending_req_count);
 428                                 list_add_tail(&sc->node, &oct->response_list
 429                                         [OCTEON_ORDERED_SC_LIST].head);
 430                                 spin_unlock_irqrestore
 431                                         (&oct->response_list
 432                                          [OCTEON_ORDERED_SC_LIST].lock,
 433                                          flags);
 434                         } else {
 435                                 if (sc->callback) {
 436                                         /* This callback must not sleep */
 437                                         sc->callback(oct, OCTEON_REQUEST_DONE,
 438                                                      sc->callback_arg);
 439                                 }
 440                         }
 441                         break;
 442                 default:
 443                         dev_err(&oct->pci_dev->dev,
 444                                 "%s Unknown reqtype: %d buf: %p at idx %d\n",
 445                                 __func__, reqtype, buf, old);
 446                 }
 447
 448                 iq->request_list[old].buf = NULL;
 449                 iq->request_list[old].reqtype = 0;
 450
 451  skip_this:
 452                 inst_count++;
 453                 old = incr_index(old, 1, iq->max_count);
 454
 455                 if ((napi_budget) && (inst_count >= napi_budget))
 456                         break;
 457         }
 458         if (bytes_compl)
 459                 octeon_report_tx_completion_to_bql(iq->app_ctx, pkts_compl,
 460                                                    bytes_compl);
 461         iq->flush_index = old;
 462
 463         if (atomic_read(&oct->response_list
 464                         [OCTEON_ORDERED_SC_LIST].pending_req_count))
 465                 queue_delayed_work(cwq->wq, &cwq->wk.work, msecs_to_jiffies(1));
 466
 467         return inst_count;
 468 }
 469
 470 /* Can only be called from process context */
 471 int
 472 octeon_flush_iq(struct octeon_device *oct, struct octeon_instr_queue *iq,
 473                 u32 napi_budget)
 474 {
 475         u32 inst_processed = 0;
 476         u32 tot_inst_processed = 0;
 477         int tx_done = 1;
 478
 479         if (!spin_trylock(&iq->iq_flush_running_lock))
 480                 return tx_done;
 481
 482         spin_lock_bh(&iq->lock);
 483
 484         iq->octeon_read_index = oct->fn_list.update_iq_read_idx(iq);
 485
 486         do {
 487                 /* Process any outstanding IQ packets. */
 488                 if (iq->flush_index == iq->octeon_read_index)
 489                         break;
 490
 491                 if (napi_budget)
 492                         inst_processed =
 493                                 lio_process_iq_request_list(oct, iq,
 494                                                             napi_budget -
 495                                                             tot_inst_processed);
 496                 else
 497                         inst_processed =
 498                                 lio_process_iq_request_list(oct, iq, 0);
 499
 500                 if (inst_processed) {
 501                         iq->pkts_processed += inst_processed;
 502                         atomic_sub(inst_processed, &iq->instr_pending);
 503                         iq->stats.instr_processed += inst_processed;
 504                 }
 505
 506                 tot_inst_processed += inst_processed;
 507         } while (tot_inst_processed < napi_budget);
 508
 509         if (napi_budget && (tot_inst_processed >= napi_budget))
 510                 tx_done = 0;
 511
 512         iq->last_db_time = jiffies;
 513
 514         spin_unlock_bh(&iq->lock);
 515
 516         spin_unlock(&iq->iq_flush_running_lock);
 517
 518         return tx_done;
 519 }
 520
 521 /* Process instruction queue after timeout.
 522  * This routine gets called from a workqueue or when removing the module.
 523  */
 524 static void __check_db_timeout(struct octeon_device *oct, u64 iq_no)
 525 {
 526         struct octeon_instr_queue *iq;
 527         u64 next_time;
 528
 529         if (!oct)
 530                 return;
 531
 532         iq = oct->instr_queue[iq_no];
 533         if (!iq)
 534                 return;
 535
 536         /* return immediately, if no work pending */
 537         if (!atomic_read(&iq->instr_pending))
 538                 return;
 539         /* If jiffies - last_db_time < db_timeout do nothing  */
 540         next_time = iq->last_db_time + iq->db_timeout;
 541         if (!time_after(jiffies, (unsigned long)next_time))
 542                 return;
 543         iq->last_db_time = jiffies;
 544
 545         /* Flush the instruction queue */
 546         octeon_flush_iq(oct, iq, 0);
 547
 548         lio_enable_irq(NULL, iq);
 549 }
 550
 551 /* Called by the Poll thread at regular intervals to check the instruction
 552  * queue for commands to be posted and for commands that were fetched by Octeon.
 553  */
 554 static void check_db_timeout(struct work_struct *work)
 555 {
 556         struct cavium_wk *wk = (struct cavium_wk *)work;
 557         struct octeon_device *oct = (struct octeon_device *)wk->ctxptr;
 558         u64 iq_no = wk->ctxul;
 559         struct cavium_wq *db_wq = &oct->check_db_wq[iq_no];
 560         u32 delay = 10;
 561
 562         __check_db_timeout(oct, iq_no);
 563         queue_delayed_work(db_wq->wq, &db_wq->wk.work, msecs_to_jiffies(delay));
 564 }
 565
 566 int
 567 octeon_send_command(struct octeon_device *oct, u32 iq_no,
 568                     u32 force_db, void *cmd, void *buf,
 569                     u32 datasize, u32 reqtype)
 570 {
 571         int xmit_stopped;
 572         struct iq_post_status st;
 573         struct octeon_instr_queue *iq = oct->instr_queue[iq_no];
 574
 575         /* Get the lock and prevent other tasks and tx interrupt handler from
 576          * running.
 577          */
 578         if (iq->allow_soft_cmds)
 579                 spin_lock_bh(&iq->post_lock);
 580
 581         st = __post_command2(iq, cmd);
 582
 583         if (st.status != IQ_SEND_FAILED) {
 584                 xmit_stopped = octeon_report_sent_bytes_to_bql(buf, reqtype);
 585                 __add_to_request_list(iq, st.index, buf, reqtype);
 586                 INCR_INSTRQUEUE_PKT_COUNT(oct, iq_no, bytes_sent, datasize);
 587                 INCR_INSTRQUEUE_PKT_COUNT(oct, iq_no, instr_posted, 1);
 588
 589                 if (iq->fill_cnt >= MAX_OCTEON_FILL_COUNT || force_db ||
 590                     xmit_stopped || st.status == IQ_SEND_STOP)
 591                         ring_doorbell(oct, iq);
 592         } else {
 593                 INCR_INSTRQUEUE_PKT_COUNT(oct, iq_no, instr_dropped, 1);
 594         }
 595
 596         if (iq->allow_soft_cmds)
 597                 spin_unlock_bh(&iq->post_lock);
 598
 599         /* This is only done here to expedite packets being flushed
 600          * for cases where there are no IQ completion interrupts.
 601          */
 602
 603         return st.status;
 604 }
 605
 606 void
 607 octeon_prepare_soft_command(struct octeon_device *oct,
 608                             struct octeon_soft_command *sc,
 609                             u8 opcode,
 610                             u8 subcode,
 611                             u32 irh_ossp,
 612                             u64 ossp0,
 613                             u64 ossp1)
 614 {
 615         struct octeon_config *oct_cfg;
 616         struct octeon_instr_ih2 *ih2;
 617         struct octeon_instr_ih3 *ih3;
 618         struct octeon_instr_pki_ih3 *pki_ih3;
 619         struct octeon_instr_irh *irh;
 620         struct octeon_instr_rdp *rdp;
 621
 622         WARN_ON(opcode > 15);
 623         WARN_ON(subcode > 127);
 624
 625         oct_cfg = octeon_get_conf(oct);
 626
 627         if (OCTEON_CN23XX_PF(oct) || OCTEON_CN23XX_VF(oct)) {
 628                 ih3 = (struct octeon_instr_ih3 *)&sc->cmd.cmd3.ih3;
 629
 630                 ih3->pkind = oct->instr_queue[sc->iq_no]->txpciq.s.pkind;
 631
 632                 pki_ih3 = (struct octeon_instr_pki_ih3 *)&sc->cmd.cmd3.pki_ih3;
 633
 634                 pki_ih3->w           = 1;
 635                 pki_ih3->raw         = 1;
 636                 pki_ih3->utag        = 1;
 637                 pki_ih3->uqpg        =
 638                         oct->instr_queue[sc->iq_no]->txpciq.s.use_qpg;
 639                 pki_ih3->utt         = 1;
 640                 pki_ih3->tag     = LIO_CONTROL;
 641                 pki_ih3->tagtype = ATOMIC_TAG;
 642                 pki_ih3->qpg         =
 643                         oct->instr_queue[sc->iq_no]->txpciq.s.ctrl_qpg;
 644
 645                 pki_ih3->pm          = 0x7;
 646                 pki_ih3->sl          = 8;
 647
 648                 if (sc->datasize)
 649                         ih3->dlengsz = sc->datasize;
 650
 651                 irh            = (struct octeon_instr_irh *)&sc->cmd.cmd3.irh;
 652                 irh->opcode    = opcode;
 653                 irh->subcode   = subcode;
 654
 655                 /* opcode/subcode specific parameters (ossp) */
 656                 irh->ossp       = irh_ossp;
 657                 sc->cmd.cmd3.ossp[0] = ossp0;
 658                 sc->cmd.cmd3.ossp[1] = ossp1;
 659
 660                 if (sc->rdatasize) {
 661                         rdp = (struct octeon_instr_rdp *)&sc->cmd.cmd3.rdp;
 662                         rdp->pcie_port = oct->pcie_port;
 663                         rdp->rlen      = sc->rdatasize;
 664
 665                         irh->rflag =  1;
 666                         /*PKI IH3*/
 667                         /* pki_ih3 irh+ossp[0]+ossp[1]+rdp+rptr = 48 bytes */
 668                         ih3->fsz    = LIO_SOFTCMDRESP_IH3;
 669                 } else {
 670                         irh->rflag =  0;
 671                         /*PKI IH3*/
 672                         /* pki_h3 + irh + ossp[0] + ossp[1] = 32 bytes */
 673                         ih3->fsz    = LIO_PCICMD_O3;
 674                 }
 675
 676         } else {
 677                 ih2          = (struct octeon_instr_ih2 *)&sc->cmd.cmd2.ih2;
 678                 ih2->tagtype = ATOMIC_TAG;
 679                 ih2->tag     = LIO_CONTROL;
 680                 ih2->raw     = 1;
 681                 ih2->grp     = CFG_GET_CTRL_Q_GRP(oct_cfg);
 682
 683                 if (sc->datasize) {
 684                         ih2->dlengsz = sc->datasize;
 685                         ih2->rs = 1;
 686                 }
 687
 688                 irh            = (struct octeon_instr_irh *)&sc->cmd.cmd2.irh;
 689                 irh->opcode    = opcode;
 690                 irh->subcode   = subcode;
 691
 692                 /* opcode/subcode specific parameters (ossp) */
 693                 irh->ossp       = irh_ossp;
 694                 sc->cmd.cmd2.ossp[0] = ossp0;
 695                 sc->cmd.cmd2.ossp[1] = ossp1;
 696
 697                 if (sc->rdatasize) {
 698                         rdp = (struct octeon_instr_rdp *)&sc->cmd.cmd2.rdp;
 699                         rdp->pcie_port = oct->pcie_port;
 700                         rdp->rlen      = sc->rdatasize;
 701
 702                         irh->rflag =  1;
 703                         /* irh+ossp[0]+ossp[1]+rdp+rptr = 40 bytes */
 704                         ih2->fsz   = LIO_SOFTCMDRESP_IH2;
 705                 } else {
 706                         irh->rflag =  0;
 707                         /* irh + ossp[0] + ossp[1] = 24 bytes */
 708                         ih2->fsz   = LIO_PCICMD_O2;
 709                 }
 710         }
 711 }
 712
 713 int octeon_send_soft_command(struct octeon_device *oct,
 714                              struct octeon_soft_command *sc)
 715 {
 716         struct octeon_instr_queue *iq;
 717         struct octeon_instr_ih2 *ih2;
 718         struct octeon_instr_ih3 *ih3;
 719         struct octeon_instr_irh *irh;
 720         u32 len;
 721
 722         iq = oct->instr_queue[sc->iq_no];
 723         if (!iq->allow_soft_cmds) {
 724                 dev_err(&oct->pci_dev->dev, "Soft commands are not allowed on Queue %d\n",
 725                         sc->iq_no);
 726                 INCR_INSTRQUEUE_PKT_COUNT(oct, sc->iq_no, instr_dropped, 1);
 727                 return IQ_SEND_FAILED;
 728         }
 729
 730         if (OCTEON_CN23XX_PF(oct) || OCTEON_CN23XX_VF(oct)) {
 731                 ih3 =  (struct octeon_instr_ih3 *)&sc->cmd.cmd3.ih3;
 732                 if (ih3->dlengsz) {
 733                         WARN_ON(!sc->dmadptr);
 734                         sc->cmd.cmd3.dptr = sc->dmadptr;
 735                 }
 736                 irh = (struct octeon_instr_irh *)&sc->cmd.cmd3.irh;
 737                 if (irh->rflag) {
 738                         WARN_ON(!sc->dmarptr);
 739                         WARN_ON(!sc->status_word);
 740                         *sc->status_word = COMPLETION_WORD_INIT;
 741                         sc->cmd.cmd3.rptr = sc->dmarptr;
 742                 }
 743                 len = (u32)ih3->dlengsz;
 744         } else {
 745                 ih2 = (struct octeon_instr_ih2 *)&sc->cmd.cmd2.ih2;
 746                 if (ih2->dlengsz) {
 747                         WARN_ON(!sc->dmadptr);
 748                         sc->cmd.cmd2.dptr = sc->dmadptr;
 749                 }
 750                 irh = (struct octeon_instr_irh *)&sc->cmd.cmd2.irh;
 751                 if (irh->rflag) {
 752                         WARN_ON(!sc->dmarptr);
 753                         WARN_ON(!sc->status_word);
 754                         *sc->status_word = COMPLETION_WORD_INIT;
 755                         sc->cmd.cmd2.rptr = sc->dmarptr;
 756                 }
 757                 len = (u32)ih2->dlengsz;
 758         }
 759
 760         if (sc->wait_time)
 761                 sc->timeout = jiffies + sc->wait_time;
 762
 763         return (octeon_send_command(oct, sc->iq_no, 1, &sc->cmd, sc,
 764                                     len, REQTYPE_SOFT_COMMAND));
 765 }
 766
 767 int octeon_setup_sc_buffer_pool(struct octeon_device *oct)
 768 {
 769         int i;
 770         u64 dma_addr;
 771         struct octeon_soft_command *sc;
 772
 773         INIT_LIST_HEAD(&oct->sc_buf_pool.head);
 774         spin_lock_init(&oct->sc_buf_pool.lock);
 775         atomic_set(&oct->sc_buf_pool.alloc_buf_count, 0);
 776
 777         for (i = 0; i < MAX_SOFT_COMMAND_BUFFERS; i++) {
 778                 sc = (struct octeon_soft_command *)
 779                         lio_dma_alloc(oct,
 780                                       SOFT_COMMAND_BUFFER_SIZE,
 781                                           (dma_addr_t *)&dma_addr);
 782                 if (!sc) {
 783                         octeon_free_sc_buffer_pool(oct);
 784                         return 1;
 785                 }
 786
 787                 sc->dma_addr = dma_addr;
 788                 sc->size = SOFT_COMMAND_BUFFER_SIZE;
 789
 790                 list_add_tail(&sc->node, &oct->sc_buf_pool.head);
 791         }
 792
 793         return 0;
 794 }
 795
 796 int octeon_free_sc_buffer_pool(struct octeon_device *oct)
 797 {
 798         struct list_head *tmp, *tmp2;
 799         struct octeon_soft_command *sc;
 800
 801         spin_lock_bh(&oct->sc_buf_pool.lock);
 802
 803         list_for_each_safe(tmp, tmp2, &oct->sc_buf_pool.head) {
 804                 list_del(tmp);
 805
 806                 sc = (struct octeon_soft_command *)tmp;
 807
 808                 lio_dma_free(oct, sc->size, sc, sc->dma_addr);
 809         }
 810
 811         INIT_LIST_HEAD(&oct->sc_buf_pool.head);
 812
 813         spin_unlock_bh(&oct->sc_buf_pool.lock);
 814
 815         return 0;
 816 }
 817
 818 struct octeon_soft_command *octeon_alloc_soft_command(struct octeon_device *oct,
 819                                                       u32 datasize,
 820                                                       u32 rdatasize,
 821                                                       u32 ctxsize)
 822 {
 823         u64 dma_addr;
 824         u32 size;
 825         u32 offset = sizeof(struct octeon_soft_command);
 826         struct octeon_soft_command *sc = NULL;
 827         struct list_head *tmp;
 828
 829         WARN_ON((offset + datasize + rdatasize + ctxsize) >
 830                SOFT_COMMAND_BUFFER_SIZE);
 831
 832         spin_lock_bh(&oct->sc_buf_pool.lock);
 833
 834         if (list_empty(&oct->sc_buf_pool.head)) {
 835                 spin_unlock_bh(&oct->sc_buf_pool.lock);
 836                 return NULL;
 837         }
 838
 839         list_for_each(tmp, &oct->sc_buf_pool.head)
 840                 break;
 841
 842         list_del(tmp);
 843
 844         atomic_inc(&oct->sc_buf_pool.alloc_buf_count);
 845
 846         spin_unlock_bh(&oct->sc_buf_pool.lock);
 847
 848         sc = (struct octeon_soft_command *)tmp;
 849
 850         dma_addr = sc->dma_addr;
 851         size = sc->size;
 852
 853         memset(sc, 0, sc->size);
 854
 855         sc->dma_addr = dma_addr;
 856         sc->size = size;
 857
 858         if (ctxsize) {
 859                 sc->ctxptr = (u8 *)sc + offset;
 860                 sc->ctxsize = ctxsize;
 861         }
 862
 863         /* Start data at 128 byte boundary */
 864         offset = (offset + ctxsize + 127) & 0xffffff80;
 865
 866         if (datasize) {
 867                 sc->virtdptr = (u8 *)sc + offset;
 868                 sc->dmadptr = dma_addr + offset;
 869                 sc->datasize = datasize;
 870         }
 871
 872         /* Start rdata at 128 byte boundary */
 873         offset = (offset + datasize + 127) & 0xffffff80;
 874
 875         if (rdatasize) {
 876                 WARN_ON(rdatasize < 16);
 877                 sc->virtrptr = (u8 *)sc + offset;
 878                 sc->dmarptr = dma_addr + offset;
 879                 sc->rdatasize = rdatasize;
 880                 sc->status_word = (u64 *)((u8 *)(sc->virtrptr) + rdatasize - 8);
 881         }
 882
 883         return sc;
 884 }
 885
 886 void octeon_free_soft_command(struct octeon_device *oct,
 887                               struct octeon_soft_command *sc)
 888 {
 889         spin_lock_bh(&oct->sc_buf_pool.lock);
 890
 891         list_add_tail(&sc->node, &oct->sc_buf_pool.head);
 892
 893         atomic_dec(&oct->sc_buf_pool.alloc_buf_count);
 894
 895         spin_unlock_bh(&oct->sc_buf_pool.lock);
 896 }