drivers/infiniband/hw/qib/qib_sdma.c

   1 /*
   2  * Copyright (c) 2012 Intel Corporation. All rights reserved.
   3  * Copyright (c) 2007 - 2012 QLogic Corporation. All rights reserved.
   4  *
   5  * This software is available to you under a choice of one of two
   6  * licenses.  You may choose to be licensed under the terms of the GNU
   7  * General Public License (GPL) Version 2, available from the file
   8  * COPYING in the main directory of this source tree, or the
   9  * OpenIB.org BSD license below:
  10  *
  11  *     Redistribution and use in source and binary forms, with or
  12  *     without modification, are permitted provided that the following
  13  *     conditions are met:
  14  *
  15  *      - Redistributions of source code must retain the above
  16  *        copyright notice, this list of conditions and the following
  17  *        disclaimer.
  18  *
  19  *      - Redistributions in binary form must reproduce the above
  20  *        copyright notice, this list of conditions and the following
  21  *        disclaimer in the documentation and/or other materials
  22  *        provided with the distribution.
  23  *
  24  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  25  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  26  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  27  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  28  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  29  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  30  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  31  * SOFTWARE.
  32  */
  33
  34 #include <linux/spinlock.h>
  35 #include <linux/netdevice.h>
  36 #include <linux/moduleparam.h>
  37
  38 #include "qib.h"
  39 #include "qib_common.h"
  40
  41 /* default pio off, sdma on */
  42 static ushort sdma_descq_cnt = 256;
  43 module_param_named(sdma_descq_cnt, sdma_descq_cnt, ushort, S_IRUGO);
  44 MODULE_PARM_DESC(sdma_descq_cnt, "Number of SDMA descq entries");
  45
  46 /*
  47  * Bits defined in the send DMA descriptor.
  48  */
  49 #define SDMA_DESC_LAST          (1ULL << 11)
  50 #define SDMA_DESC_FIRST         (1ULL << 12)
  51 #define SDMA_DESC_DMA_HEAD      (1ULL << 13)
  52 #define SDMA_DESC_USE_LARGE_BUF (1ULL << 14)
  53 #define SDMA_DESC_INTR          (1ULL << 15)
  54 #define SDMA_DESC_COUNT_LSB     16
  55 #define SDMA_DESC_GEN_LSB       30
  56
  57 /* declare all statics here rather than keep sorting */
  58 static int alloc_sdma(struct qib_pportdata *);
  59 static void sdma_complete(struct kref *);
  60 static void sdma_finalput(struct qib_sdma_state *);
  61 static void sdma_get(struct qib_sdma_state *);
  62 static void sdma_put(struct qib_sdma_state *);
  63 static void sdma_set_state(struct qib_pportdata *, enum qib_sdma_states);
  64 static void sdma_start_sw_clean_up(struct qib_pportdata *);
  65 static void sdma_sw_clean_up_task(unsigned long);
  66 static void unmap_desc(struct qib_pportdata *, unsigned);
  67
  68 static void sdma_get(struct qib_sdma_state *ss)
  69 {
  70         kref_get(&ss->kref);
  71 }
  72
  73 static void sdma_complete(struct kref *kref)
  74 {
  75         struct qib_sdma_state *ss =
  76                 container_of(kref, struct qib_sdma_state, kref);
  77
  78         complete(&ss->comp);
  79 }
  80
  81 static void sdma_put(struct qib_sdma_state *ss)
  82 {
  83         kref_put(&ss->kref, sdma_complete);
  84 }
  85
  86 static void sdma_finalput(struct qib_sdma_state *ss)
  87 {
  88         sdma_put(ss);
  89         wait_for_completion(&ss->comp);
  90 }
  91
  92 /*
  93  * Complete all the sdma requests on the active list, in the correct
  94  * order, and with appropriate processing.   Called when cleaning up
  95  * after sdma shutdown, and when new sdma requests are submitted for
  96  * a link that is down.   This matches what is done for requests
  97  * that complete normally, it's just the full list.
  98  *
  99  * Must be called with sdma_lock held
 100  */
 101 static void clear_sdma_activelist(struct qib_pportdata *ppd)
 102 {
 103         struct qib_sdma_txreq *txp, *txp_next;
 104
 105         list_for_each_entry_safe(txp, txp_next, &ppd->sdma_activelist, list) {
 106                 list_del_init(&txp->list);
 107                 if (txp->flags & QIB_SDMA_TXREQ_F_FREEDESC) {
 108                         unsigned idx;
 109
 110                         idx = txp->start_idx;
 111                         while (idx != txp->next_descq_idx) {
 112                                 unmap_desc(ppd, idx);
 113                                 if (++idx == ppd->sdma_descq_cnt)
 114                                         idx = 0;
 115                         }
 116                 }
 117                 if (txp->callback)
 118                         (*txp->callback)(txp, QIB_SDMA_TXREQ_S_ABORTED);
 119         }
 120 }
 121
 122 static void sdma_sw_clean_up_task(unsigned long opaque)
 123 {
 124         struct qib_pportdata *ppd = (struct qib_pportdata *) opaque;
 125         unsigned long flags;
 126
 127         spin_lock_irqsave(&ppd->sdma_lock, flags);
 128
 129         /*
 130          * At this point, the following should always be true:
 131          * - We are halted, so no more descriptors are getting retired.
 132          * - We are not running, so no one is submitting new work.
 133          * - Only we can send the e40_sw_cleaned, so we can't start
 134          *   running again until we say so.  So, the active list and
 135          *   descq are ours to play with.
 136          */
 137
 138         /* Process all retired requests. */
 139         qib_sdma_make_progress(ppd);
 140
 141         clear_sdma_activelist(ppd);
 142
 143         /*
 144          * Resync count of added and removed.  It is VERY important that
 145          * sdma_descq_removed NEVER decrement - user_sdma depends on it.
 146          */
 147         ppd->sdma_descq_removed = ppd->sdma_descq_added;
 148
 149         /*
 150          * Reset our notion of head and tail.
 151          * Note that the HW registers will be reset when switching states
 152          * due to calling __qib_sdma_process_event() below.
 153          */
 154         ppd->sdma_descq_tail = 0;
 155         ppd->sdma_descq_head = 0;
 156         ppd->sdma_head_dma[0] = 0;
 157         ppd->sdma_generation = 0;
 158
 159         __qib_sdma_process_event(ppd, qib_sdma_event_e40_sw_cleaned);
 160
 161         spin_unlock_irqrestore(&ppd->sdma_lock, flags);
 162 }
 163
 164 /*
 165  * This is called when changing to state qib_sdma_state_s10_hw_start_up_wait
 166  * as a result of send buffer errors or send DMA descriptor errors.
 167  * We want to disarm the buffers in these cases.
 168  */
 169 static void sdma_hw_start_up(struct qib_pportdata *ppd)
 170 {
 171         struct qib_sdma_state *ss = &ppd->sdma_state;
 172         unsigned bufno;
 173
 174         for (bufno = ss->first_sendbuf; bufno < ss->last_sendbuf; ++bufno)
 175                 ppd->dd->f_sendctrl(ppd, QIB_SENDCTRL_DISARM_BUF(bufno));
 176
 177         ppd->dd->f_sdma_hw_start_up(ppd);
 178 }
 179
 180 static void sdma_sw_tear_down(struct qib_pportdata *ppd)
 181 {
 182         struct qib_sdma_state *ss = &ppd->sdma_state;
 183
 184         /* Releasing this reference means the state machine has stopped. */
 185         sdma_put(ss);
 186 }
 187
 188 static void sdma_start_sw_clean_up(struct qib_pportdata *ppd)
 189 {
 190         tasklet_hi_schedule(&ppd->sdma_sw_clean_up_task);
 191 }
 192
 193 static void sdma_set_state(struct qib_pportdata *ppd,
 194         enum qib_sdma_states next_state)
 195 {
 196         struct qib_sdma_state *ss = &ppd->sdma_state;
 197         struct sdma_set_state_action *action = ss->set_state_action;
 198         unsigned op = 0;
 199
 200         /* debugging bookkeeping */
 201         ss->previous_state = ss->current_state;
 202         ss->previous_op = ss->current_op;
 203
 204         ss->current_state = next_state;
 205
 206         if (action[next_state].op_enable)
 207                 op |= QIB_SDMA_SENDCTRL_OP_ENABLE;
 208
 209         if (action[next_state].op_intenable)
 210                 op |= QIB_SDMA_SENDCTRL_OP_INTENABLE;
 211
 212         if (action[next_state].op_halt)
 213                 op |= QIB_SDMA_SENDCTRL_OP_HALT;
 214
 215         if (action[next_state].op_drain)
 216                 op |= QIB_SDMA_SENDCTRL_OP_DRAIN;
 217
 218         if (action[next_state].go_s99_running_tofalse)
 219                 ss->go_s99_running = 0;
 220
 221         if (action[next_state].go_s99_running_totrue)
 222                 ss->go_s99_running = 1;
 223
 224         ss->current_op = op;
 225
 226         ppd->dd->f_sdma_sendctrl(ppd, ss->current_op);
 227 }
 228
 229 static void unmap_desc(struct qib_pportdata *ppd, unsigned head)
 230 {
 231         __le64 *descqp = &ppd->sdma_descq[head].qw[0];
 232         u64 desc[2];
 233         dma_addr_t addr;
 234         size_t len;
 235
 236         desc[0] = le64_to_cpu(descqp[0]);
 237         desc[1] = le64_to_cpu(descqp[1]);
 238
 239         addr = (desc[1] << 32) | (desc[0] >> 32);
 240         len = (desc[0] >> 14) & (0x7ffULL << 2);
 241         dma_unmap_single(&ppd->dd->pcidev->dev, addr, len, DMA_TO_DEVICE);
 242 }
 243
 244 static int alloc_sdma(struct qib_pportdata *ppd)
 245 {
 246         ppd->sdma_descq_cnt = sdma_descq_cnt;
 247         if (!ppd->sdma_descq_cnt)
 248                 ppd->sdma_descq_cnt = 256;
 249
 250         /* Allocate memory for SendDMA descriptor FIFO */
 251         ppd->sdma_descq = dma_alloc_coherent(&ppd->dd->pcidev->dev,
 252                 ppd->sdma_descq_cnt * sizeof(u64[2]), &ppd->sdma_descq_phys,
 253                 GFP_KERNEL);
 254
 255         if (!ppd->sdma_descq) {
 256                 qib_dev_err(ppd->dd,
 257                         "failed to allocate SendDMA descriptor FIFO memory\n");
 258                 goto bail;
 259         }
 260
 261         /* Allocate memory for DMA of head register to memory */
 262         ppd->sdma_head_dma = dma_alloc_coherent(&ppd->dd->pcidev->dev,
 263                 PAGE_SIZE, &ppd->sdma_head_phys, GFP_KERNEL);
 264         if (!ppd->sdma_head_dma) {
 265                 qib_dev_err(ppd->dd,
 266                         "failed to allocate SendDMA head memory\n");
 267                 goto cleanup_descq;
 268         }
 269         ppd->sdma_head_dma[0] = 0;
 270         return 0;
 271
 272 cleanup_descq:
 273         dma_free_coherent(&ppd->dd->pcidev->dev,
 274                 ppd->sdma_descq_cnt * sizeof(u64[2]), (void *)ppd->sdma_descq,
 275                 ppd->sdma_descq_phys);
 276         ppd->sdma_descq = NULL;
 277         ppd->sdma_descq_phys = 0;
 278 bail:
 279         ppd->sdma_descq_cnt = 0;
 280         return -ENOMEM;
 281 }
 282
 283 static void free_sdma(struct qib_pportdata *ppd)
 284 {
 285         struct qib_devdata *dd = ppd->dd;
 286
 287         if (ppd->sdma_head_dma) {
 288                 dma_free_coherent(&dd->pcidev->dev, PAGE_SIZE,
 289                                   (void *)ppd->sdma_head_dma,
 290                                   ppd->sdma_head_phys);
 291                 ppd->sdma_head_dma = NULL;
 292                 ppd->sdma_head_phys = 0;
 293         }
 294
 295         if (ppd->sdma_descq) {
 296                 dma_free_coherent(&dd->pcidev->dev,
 297                                   ppd->sdma_descq_cnt * sizeof(u64[2]),
 298                                   ppd->sdma_descq, ppd->sdma_descq_phys);
 299                 ppd->sdma_descq = NULL;
 300                 ppd->sdma_descq_phys = 0;
 301         }
 302 }
 303
 304 static inline void make_sdma_desc(struct qib_pportdata *ppd,
 305                                   u64 *sdmadesc, u64 addr, u64 dwlen,
 306                                   u64 dwoffset)
 307 {
 308
 309         WARN_ON(addr & 3);
 310         /* SDmaPhyAddr[47:32] */
 311         sdmadesc[1] = addr >> 32;
 312         /* SDmaPhyAddr[31:0] */
 313         sdmadesc[0] = (addr & 0xfffffffcULL) << 32;
 314         /* SDmaGeneration[1:0] */
 315         sdmadesc[0] |= (ppd->sdma_generation & 3ULL) <<
 316                 SDMA_DESC_GEN_LSB;
 317         /* SDmaDwordCount[10:0] */
 318         sdmadesc[0] |= (dwlen & 0x7ffULL) << SDMA_DESC_COUNT_LSB;
 319         /* SDmaBufOffset[12:2] */
 320         sdmadesc[0] |= dwoffset & 0x7ffULL;
 321 }
 322
 323 /* sdma_lock must be held */
 324 int qib_sdma_make_progress(struct qib_pportdata *ppd)
 325 {
 326         struct list_head *lp = NULL;
 327         struct qib_sdma_txreq *txp = NULL;
 328         struct qib_devdata *dd = ppd->dd;
 329         int progress = 0;
 330         u16 hwhead;
 331         u16 idx = 0;
 332
 333         hwhead = dd->f_sdma_gethead(ppd);
 334
 335         /* The reason for some of the complexity of this code is that
 336          * not all descriptors have corresponding txps.  So, we have to
 337          * be able to skip over descs until we wander into the range of
 338          * the next txp on the list.
 339          */
 340
 341         if (!list_empty(&ppd->sdma_activelist)) {
 342                 lp = ppd->sdma_activelist.next;
 343                 txp = list_entry(lp, struct qib_sdma_txreq, list);
 344                 idx = txp->start_idx;
 345         }
 346
 347         while (ppd->sdma_descq_head != hwhead) {
 348                 /* if desc is part of this txp, unmap if needed */
 349                 if (txp && (txp->flags & QIB_SDMA_TXREQ_F_FREEDESC) &&
 350                     (idx == ppd->sdma_descq_head)) {
 351                         unmap_desc(ppd, ppd->sdma_descq_head);
 352                         if (++idx == ppd->sdma_descq_cnt)
 353                                 idx = 0;
 354                 }
 355
 356                 /* increment dequed desc count */
 357                 ppd->sdma_descq_removed++;
 358
 359                 /* advance head, wrap if needed */
 360                 if (++ppd->sdma_descq_head == ppd->sdma_descq_cnt)
 361                         ppd->sdma_descq_head = 0;
 362
 363                 /* if now past this txp's descs, do the callback */
 364                 if (txp && txp->next_descq_idx == ppd->sdma_descq_head) {
 365                         /* remove from active list */
 366                         list_del_init(&txp->list);
 367                         if (txp->callback)
 368                                 (*txp->callback)(txp, QIB_SDMA_TXREQ_S_OK);
 369                         /* see if there is another txp */
 370                         if (list_empty(&ppd->sdma_activelist))
 371                                 txp = NULL;
 372                         else {
 373                                 lp = ppd->sdma_activelist.next;
 374                                 txp = list_entry(lp, struct qib_sdma_txreq,
 375                                         list);
 376                                 idx = txp->start_idx;
 377                         }
 378                 }
 379                 progress = 1;
 380         }
 381         if (progress)
 382                 qib_verbs_sdma_desc_avail(ppd, qib_sdma_descq_freecnt(ppd));
 383         return progress;
 384 }
 385
 386 /*
 387  * This is called from interrupt context.
 388  */
 389 void qib_sdma_intr(struct qib_pportdata *ppd)
 390 {
 391         unsigned long flags;
 392
 393         spin_lock_irqsave(&ppd->sdma_lock, flags);
 394
 395         __qib_sdma_intr(ppd);
 396
 397         spin_unlock_irqrestore(&ppd->sdma_lock, flags);
 398 }
 399
 400 void __qib_sdma_intr(struct qib_pportdata *ppd)
 401 {
 402         if (__qib_sdma_running(ppd)) {
 403                 qib_sdma_make_progress(ppd);
 404                 if (!list_empty(&ppd->sdma_userpending))
 405                         qib_user_sdma_send_desc(ppd, &ppd->sdma_userpending);
 406         }
 407 }
 408
 409 int qib_setup_sdma(struct qib_pportdata *ppd)
 410 {
 411         struct qib_devdata *dd = ppd->dd;
 412         unsigned long flags;
 413         int ret = 0;
 414
 415         ret = alloc_sdma(ppd);
 416         if (ret)
 417                 goto bail;
 418
 419         /* set consistent sdma state */
 420         ppd->dd->f_sdma_init_early(ppd);
 421         spin_lock_irqsave(&ppd->sdma_lock, flags);
 422         sdma_set_state(ppd, qib_sdma_state_s00_hw_down);
 423         spin_unlock_irqrestore(&ppd->sdma_lock, flags);
 424
 425         /* set up reference counting */
 426         kref_init(&ppd->sdma_state.kref);
 427         init_completion(&ppd->sdma_state.comp);
 428
 429         ppd->sdma_generation = 0;
 430         ppd->sdma_descq_head = 0;
 431         ppd->sdma_descq_removed = 0;
 432         ppd->sdma_descq_added = 0;
 433
 434         ppd->sdma_intrequest = 0;
 435         INIT_LIST_HEAD(&ppd->sdma_userpending);
 436
 437         INIT_LIST_HEAD(&ppd->sdma_activelist);
 438
 439         tasklet_init(&ppd->sdma_sw_clean_up_task, sdma_sw_clean_up_task,
 440                 (unsigned long)ppd);
 441
 442         ret = dd->f_init_sdma_regs(ppd);
 443         if (ret)
 444                 goto bail_alloc;
 445
 446         qib_sdma_process_event(ppd, qib_sdma_event_e10_go_hw_start);
 447
 448         return 0;
 449
 450 bail_alloc:
 451         qib_teardown_sdma(ppd);
 452 bail:
 453         return ret;
 454 }
 455
 456 void qib_teardown_sdma(struct qib_pportdata *ppd)
 457 {
 458         qib_sdma_process_event(ppd, qib_sdma_event_e00_go_hw_down);
 459
 460         /*
 461          * This waits for the state machine to exit so it is not
 462          * necessary to kill the sdma_sw_clean_up_task to make sure
 463          * it is not running.
 464          */
 465         sdma_finalput(&ppd->sdma_state);
 466
 467         free_sdma(ppd);
 468 }
 469
 470 int qib_sdma_running(struct qib_pportdata *ppd)
 471 {
 472         unsigned long flags;
 473         int ret;
 474
 475         spin_lock_irqsave(&ppd->sdma_lock, flags);
 476         ret = __qib_sdma_running(ppd);
 477         spin_unlock_irqrestore(&ppd->sdma_lock, flags);
 478
 479         return ret;
 480 }
 481
 482 /*
 483  * Complete a request when sdma not running; likely only request
 484  * but to simplify the code, always queue it, then process the full
 485  * activelist.  We process the entire list to ensure that this particular
 486  * request does get it's callback, but in the correct order.
 487  * Must be called with sdma_lock held
 488  */
 489 static void complete_sdma_err_req(struct qib_pportdata *ppd,
 490                                   struct qib_verbs_txreq *tx)
 491 {
 492         struct qib_qp_priv *priv = tx->qp->priv;
 493
 494         atomic_inc(&priv->s_dma_busy);
 495         /* no sdma descriptors, so no unmap_desc */
 496         tx->txreq.start_idx = 0;
 497         tx->txreq.next_descq_idx = 0;
 498         list_add_tail(&tx->txreq.list, &ppd->sdma_activelist);
 499         clear_sdma_activelist(ppd);
 500 }
 501
 502 /*
 503  * This function queues one IB packet onto the send DMA queue per call.
 504  * The caller is responsible for checking:
 505  * 1) The number of send DMA descriptor entries is less than the size of
 506  *    the descriptor queue.
 507  * 2) The IB SGE addresses and lengths are 32-bit aligned
 508  *    (except possibly the last SGE's length)
 509  * 3) The SGE addresses are suitable for passing to dma_map_single().
 510  */
 511 int qib_sdma_verbs_send(struct qib_pportdata *ppd,
 512                         struct rvt_sge_state *ss, u32 dwords,
 513                         struct qib_verbs_txreq *tx)
 514 {
 515         unsigned long flags;
 516         struct rvt_sge *sge;
 517         struct rvt_qp *qp;
 518         int ret = 0;
 519         u16 tail;
 520         __le64 *descqp;
 521         u64 sdmadesc[2];
 522         u32 dwoffset;
 523         dma_addr_t addr;
 524         struct qib_qp_priv *priv;
 525
 526         spin_lock_irqsave(&ppd->sdma_lock, flags);
 527
 528 retry:
 529         if (unlikely(!__qib_sdma_running(ppd))) {
 530                 complete_sdma_err_req(ppd, tx);
 531                 goto unlock;
 532         }
 533
 534         if (tx->txreq.sg_count > qib_sdma_descq_freecnt(ppd)) {
 535                 if (qib_sdma_make_progress(ppd))
 536                         goto retry;
 537                 if (ppd->dd->flags & QIB_HAS_SDMA_TIMEOUT)
 538                         ppd->dd->f_sdma_set_desc_cnt(ppd,
 539                                         ppd->sdma_descq_cnt / 2);
 540                 goto busy;
 541         }
 542
 543         dwoffset = tx->hdr_dwords;
 544         make_sdma_desc(ppd, sdmadesc, (u64) tx->txreq.addr, dwoffset, 0);
 545
 546         sdmadesc[0] |= SDMA_DESC_FIRST;
 547         if (tx->txreq.flags & QIB_SDMA_TXREQ_F_USELARGEBUF)
 548                 sdmadesc[0] |= SDMA_DESC_USE_LARGE_BUF;
 549
 550         /* write to the descq */
 551         tail = ppd->sdma_descq_tail;
 552         descqp = &ppd->sdma_descq[tail].qw[0];
 553         *descqp++ = cpu_to_le64(sdmadesc[0]);
 554         *descqp++ = cpu_to_le64(sdmadesc[1]);
 555
 556         /* increment the tail */
 557         if (++tail == ppd->sdma_descq_cnt) {
 558                 tail = 0;
 559                 descqp = &ppd->sdma_descq[0].qw[0];
 560                 ++ppd->sdma_generation;
 561         }
 562
 563         tx->txreq.start_idx = tail;
 564
 565         sge = &ss->sge;
 566         while (dwords) {
 567                 u32 dw;
 568                 u32 len;
 569
 570                 len = dwords << 2;
 571                 if (len > sge->length)
 572                         len = sge->length;
 573                 if (len > sge->sge_length)
 574                         len = sge->sge_length;
 575                 BUG_ON(len == 0);
 576                 dw = (len + 3) >> 2;
 577                 addr = dma_map_single(&ppd->dd->pcidev->dev, sge->vaddr,
 578                                       dw << 2, DMA_TO_DEVICE);
 579                 if (dma_mapping_error(&ppd->dd->pcidev->dev, addr)) {
 580                         ret = -ENOMEM;
 581                         goto unmap;
 582                 }
 583                 sdmadesc[0] = 0;
 584                 make_sdma_desc(ppd, sdmadesc, (u64) addr, dw, dwoffset);
 585                 /* SDmaUseLargeBuf has to be set in every descriptor */
 586                 if (tx->txreq.flags & QIB_SDMA_TXREQ_F_USELARGEBUF)
 587                         sdmadesc[0] |= SDMA_DESC_USE_LARGE_BUF;
 588                 /* write to the descq */
 589                 *descqp++ = cpu_to_le64(sdmadesc[0]);
 590                 *descqp++ = cpu_to_le64(sdmadesc[1]);
 591
 592                 /* increment the tail */
 593                 if (++tail == ppd->sdma_descq_cnt) {
 594                         tail = 0;
 595                         descqp = &ppd->sdma_descq[0].qw[0];
 596                         ++ppd->sdma_generation;
 597                 }
 598                 sge->vaddr += len;
 599                 sge->length -= len;
 600                 sge->sge_length -= len;
 601                 if (sge->sge_length == 0) {
 602                         if (--ss->num_sge)
 603                                 *sge = *ss->sg_list++;
 604                 } else if (sge->length == 0 && sge->mr->lkey) {
 605                         if (++sge->n >= RVT_SEGSZ) {
 606                                 if (++sge->m >= sge->mr->mapsz)
 607                                         break;
 608                                 sge->n = 0;
 609                         }
 610                         sge->vaddr =
 611                                 sge->mr->map[sge->m]->segs[sge->n].vaddr;
 612                         sge->length =
 613                                 sge->mr->map[sge->m]->segs[sge->n].length;
 614                 }
 615
 616                 dwoffset += dw;
 617                 dwords -= dw;
 618         }
 619
 620         if (!tail)
 621                 descqp = &ppd->sdma_descq[ppd->sdma_descq_cnt].qw[0];
 622         descqp -= 2;
 623         descqp[0] |= cpu_to_le64(SDMA_DESC_LAST);
 624         if (tx->txreq.flags & QIB_SDMA_TXREQ_F_HEADTOHOST)
 625                 descqp[0] |= cpu_to_le64(SDMA_DESC_DMA_HEAD);
 626         if (tx->txreq.flags & QIB_SDMA_TXREQ_F_INTREQ)
 627                 descqp[0] |= cpu_to_le64(SDMA_DESC_INTR);
 628         priv = tx->qp->priv;
 629         atomic_inc(&priv->s_dma_busy);
 630         tx->txreq.next_descq_idx = tail;
 631         ppd->dd->f_sdma_update_tail(ppd, tail);
 632         ppd->sdma_descq_added += tx->txreq.sg_count;
 633         list_add_tail(&tx->txreq.list, &ppd->sdma_activelist);
 634         goto unlock;
 635
 636 unmap:
 637         for (;;) {
 638                 if (!tail)
 639                         tail = ppd->sdma_descq_cnt - 1;
 640                 else
 641                         tail--;
 642                 if (tail == ppd->sdma_descq_tail)
 643                         break;
 644                 unmap_desc(ppd, tail);
 645         }
 646         qp = tx->qp;
 647         priv = qp->priv;
 648         qib_put_txreq(tx);
 649         spin_lock(&qp->r_lock);
 650         spin_lock(&qp->s_lock);
 651         if (qp->ibqp.qp_type == IB_QPT_RC) {
 652                 /* XXX what about error sending RDMA read responses? */
 653                 if (ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK)
 654                         rvt_error_qp(qp, IB_WC_GENERAL_ERR);
 655         } else if (qp->s_wqe)
 656                 qib_send_complete(qp, qp->s_wqe, IB_WC_GENERAL_ERR);
 657         spin_unlock(&qp->s_lock);
 658         spin_unlock(&qp->r_lock);
 659         /* return zero to process the next send work request */
 660         goto unlock;
 661
 662 busy:
 663         qp = tx->qp;
 664         priv = qp->priv;
 665         spin_lock(&qp->s_lock);
 666         if (ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK) {
 667                 struct qib_ibdev *dev;
 668
 669                 /*
 670                  * If we couldn't queue the DMA request, save the info
 671                  * and try again later rather than destroying the
 672                  * buffer and undoing the side effects of the copy.
 673                  */
 674                 tx->ss = ss;
 675                 tx->dwords = dwords;
 676                 priv->s_tx = tx;
 677                 dev = &ppd->dd->verbs_dev;
 678                 spin_lock(&dev->rdi.pending_lock);
 679                 if (list_empty(&priv->iowait)) {
 680                         struct qib_ibport *ibp;
 681
 682                         ibp = &ppd->ibport_data;
 683                         ibp->rvp.n_dmawait++;
 684                         qp->s_flags |= RVT_S_WAIT_DMA_DESC;
 685                         list_add_tail(&priv->iowait, &dev->dmawait);
 686                 }
 687                 spin_unlock(&dev->rdi.pending_lock);
 688                 qp->s_flags &= ~RVT_S_BUSY;
 689                 spin_unlock(&qp->s_lock);
 690                 ret = -EBUSY;
 691         } else {
 692                 spin_unlock(&qp->s_lock);
 693                 qib_put_txreq(tx);
 694         }
 695 unlock:
 696         spin_unlock_irqrestore(&ppd->sdma_lock, flags);
 697         return ret;
 698 }
 699
 700 /*
 701  * sdma_lock should be acquired before calling this routine
 702  */
 703 void dump_sdma_state(struct qib_pportdata *ppd)
 704 {
 705         struct qib_sdma_desc *descq;
 706         struct qib_sdma_txreq *txp, *txpnext;
 707         __le64 *descqp;
 708         u64 desc[2];
 709         u64 addr;
 710         u16 gen, dwlen, dwoffset;
 711         u16 head, tail, cnt;
 712
 713         head = ppd->sdma_descq_head;
 714         tail = ppd->sdma_descq_tail;
 715         cnt = qib_sdma_descq_freecnt(ppd);
 716         descq = ppd->sdma_descq;
 717
 718         qib_dev_porterr(ppd->dd, ppd->port,
 719                 "SDMA ppd->sdma_descq_head: %u\n", head);
 720         qib_dev_porterr(ppd->dd, ppd->port,
 721                 "SDMA ppd->sdma_descq_tail: %u\n", tail);
 722         qib_dev_porterr(ppd->dd, ppd->port,
 723                 "SDMA sdma_descq_freecnt: %u\n", cnt);
 724
 725         /* print info for each entry in the descriptor queue */
 726         while (head != tail) {
 727                 char flags[6] = { 'x', 'x', 'x', 'x', 'x', 0 };
 728
 729                 descqp = &descq[head].qw[0];
 730                 desc[0] = le64_to_cpu(descqp[0]);
 731                 desc[1] = le64_to_cpu(descqp[1]);
 732                 flags[0] = (desc[0] & 1<<15) ? 'I' : '-';
 733                 flags[1] = (desc[0] & 1<<14) ? 'L' : 'S';
 734                 flags[2] = (desc[0] & 1<<13) ? 'H' : '-';
 735                 flags[3] = (desc[0] & 1<<12) ? 'F' : '-';
 736                 flags[4] = (desc[0] & 1<<11) ? 'L' : '-';
 737                 addr = (desc[1] << 32) | ((desc[0] >> 32) & 0xfffffffcULL);
 738                 gen = (desc[0] >> 30) & 3ULL;
 739                 dwlen = (desc[0] >> 14) & (0x7ffULL << 2);
 740                 dwoffset = (desc[0] & 0x7ffULL) << 2;
 741                 qib_dev_porterr(ppd->dd, ppd->port,
 742                         "SDMA sdmadesc[%u]: flags:%s addr:0x%016llx gen:%u len:%u bytes offset:%u bytes\n",
 743                          head, flags, addr, gen, dwlen, dwoffset);
 744                 if (++head == ppd->sdma_descq_cnt)
 745                         head = 0;
 746         }
 747
 748         /* print dma descriptor indices from the TX requests */
 749         list_for_each_entry_safe(txp, txpnext, &ppd->sdma_activelist,
 750                                  list)
 751                 qib_dev_porterr(ppd->dd, ppd->port,
 752                         "SDMA txp->start_idx: %u txp->next_descq_idx: %u\n",
 753                         txp->start_idx, txp->next_descq_idx);
 754 }
 755
 756 void qib_sdma_process_event(struct qib_pportdata *ppd,
 757         enum qib_sdma_events event)
 758 {
 759         unsigned long flags;
 760
 761         spin_lock_irqsave(&ppd->sdma_lock, flags);
 762
 763         __qib_sdma_process_event(ppd, event);
 764
 765         if (ppd->sdma_state.current_state == qib_sdma_state_s99_running)
 766                 qib_verbs_sdma_desc_avail(ppd, qib_sdma_descq_freecnt(ppd));
 767
 768         spin_unlock_irqrestore(&ppd->sdma_lock, flags);
 769 }
 770
 771 void __qib_sdma_process_event(struct qib_pportdata *ppd,
 772         enum qib_sdma_events event)
 773 {
 774         struct qib_sdma_state *ss = &ppd->sdma_state;
 775
 776         switch (ss->current_state) {
 777         case qib_sdma_state_s00_hw_down:
 778                 switch (event) {
 779                 case qib_sdma_event_e00_go_hw_down:
 780                         break;
 781                 case qib_sdma_event_e30_go_running:
 782                         /*
 783                          * If down, but running requested (usually result
 784                          * of link up, then we need to start up.
 785                          * This can happen when hw down is requested while
 786                          * bringing the link up with traffic active on
 787                          * 7220, e.g. */
 788                         ss->go_s99_running = 1;
 789                         /* fall through -- and start dma engine */
 790                 case qib_sdma_event_e10_go_hw_start:
 791                         /* This reference means the state machine is started */
 792                         sdma_get(&ppd->sdma_state);
 793                         sdma_set_state(ppd,
 794                                        qib_sdma_state_s10_hw_start_up_wait);
 795                         break;
 796                 case qib_sdma_event_e20_hw_started:
 797                         break;
 798                 case qib_sdma_event_e40_sw_cleaned:
 799                         sdma_sw_tear_down(ppd);
 800                         break;
 801                 case qib_sdma_event_e50_hw_cleaned:
 802                         break;
 803                 case qib_sdma_event_e60_hw_halted:
 804                         break;
 805                 case qib_sdma_event_e70_go_idle:
 806                         break;
 807                 case qib_sdma_event_e7220_err_halted:
 808                         break;
 809                 case qib_sdma_event_e7322_err_halted:
 810                         break;
 811                 case qib_sdma_event_e90_timer_tick:
 812                         break;
 813                 }
 814                 break;
 815
 816         case qib_sdma_state_s10_hw_start_up_wait:
 817                 switch (event) {
 818                 case qib_sdma_event_e00_go_hw_down:
 819                         sdma_set_state(ppd, qib_sdma_state_s00_hw_down);
 820                         sdma_sw_tear_down(ppd);
 821                         break;
 822                 case qib_sdma_event_e10_go_hw_start:
 823                         break;
 824                 case qib_sdma_event_e20_hw_started:
 825                         sdma_set_state(ppd, ss->go_s99_running ?
 826                                        qib_sdma_state_s99_running :
 827                                        qib_sdma_state_s20_idle);
 828                         break;
 829                 case qib_sdma_event_e30_go_running:
 830                         ss->go_s99_running = 1;
 831                         break;
 832                 case qib_sdma_event_e40_sw_cleaned:
 833                         break;
 834                 case qib_sdma_event_e50_hw_cleaned:
 835                         break;
 836                 case qib_sdma_event_e60_hw_halted:
 837                         break;
 838                 case qib_sdma_event_e70_go_idle:
 839                         ss->go_s99_running = 0;
 840                         break;
 841                 case qib_sdma_event_e7220_err_halted:
 842                         break;
 843                 case qib_sdma_event_e7322_err_halted:
 844                         break;
 845                 case qib_sdma_event_e90_timer_tick:
 846                         break;
 847                 }
 848                 break;
 849
 850         case qib_sdma_state_s20_idle:
 851                 switch (event) {
 852                 case qib_sdma_event_e00_go_hw_down:
 853                         sdma_set_state(ppd, qib_sdma_state_s00_hw_down);
 854                         sdma_sw_tear_down(ppd);
 855                         break;
 856                 case qib_sdma_event_e10_go_hw_start:
 857                         break;
 858                 case qib_sdma_event_e20_hw_started:
 859                         break;
 860                 case qib_sdma_event_e30_go_running:
 861                         sdma_set_state(ppd, qib_sdma_state_s99_running);
 862                         ss->go_s99_running = 1;
 863                         break;
 864                 case qib_sdma_event_e40_sw_cleaned:
 865                         break;
 866                 case qib_sdma_event_e50_hw_cleaned:
 867                         break;
 868                 case qib_sdma_event_e60_hw_halted:
 869                         break;
 870                 case qib_sdma_event_e70_go_idle:
 871                         break;
 872                 case qib_sdma_event_e7220_err_halted:
 873                         break;
 874                 case qib_sdma_event_e7322_err_halted:
 875                         break;
 876                 case qib_sdma_event_e90_timer_tick:
 877                         break;
 878                 }
 879                 break;
 880
 881         case qib_sdma_state_s30_sw_clean_up_wait:
 882                 switch (event) {
 883                 case qib_sdma_event_e00_go_hw_down:
 884                         sdma_set_state(ppd, qib_sdma_state_s00_hw_down);
 885                         break;
 886                 case qib_sdma_event_e10_go_hw_start:
 887                         break;
 888                 case qib_sdma_event_e20_hw_started:
 889                         break;
 890                 case qib_sdma_event_e30_go_running:
 891                         ss->go_s99_running = 1;
 892                         break;
 893                 case qib_sdma_event_e40_sw_cleaned:
 894                         sdma_set_state(ppd,
 895                                        qib_sdma_state_s10_hw_start_up_wait);
 896                         sdma_hw_start_up(ppd);
 897                         break;
 898                 case qib_sdma_event_e50_hw_cleaned:
 899                         break;
 900                 case qib_sdma_event_e60_hw_halted:
 901                         break;
 902                 case qib_sdma_event_e70_go_idle:
 903                         ss->go_s99_running = 0;
 904                         break;
 905                 case qib_sdma_event_e7220_err_halted:
 906                         break;
 907                 case qib_sdma_event_e7322_err_halted:
 908                         break;
 909                 case qib_sdma_event_e90_timer_tick:
 910                         break;
 911                 }
 912                 break;
 913
 914         case qib_sdma_state_s40_hw_clean_up_wait:
 915                 switch (event) {
 916                 case qib_sdma_event_e00_go_hw_down:
 917                         sdma_set_state(ppd, qib_sdma_state_s00_hw_down);
 918                         sdma_start_sw_clean_up(ppd);
 919                         break;
 920                 case qib_sdma_event_e10_go_hw_start:
 921                         break;
 922                 case qib_sdma_event_e20_hw_started:
 923                         break;
 924                 case qib_sdma_event_e30_go_running:
 925                         ss->go_s99_running = 1;
 926                         break;
 927                 case qib_sdma_event_e40_sw_cleaned:
 928                         break;
 929                 case qib_sdma_event_e50_hw_cleaned:
 930                         sdma_set_state(ppd,
 931                                        qib_sdma_state_s30_sw_clean_up_wait);
 932                         sdma_start_sw_clean_up(ppd);
 933                         break;
 934                 case qib_sdma_event_e60_hw_halted:
 935                         break;
 936                 case qib_sdma_event_e70_go_idle:
 937                         ss->go_s99_running = 0;
 938                         break;
 939                 case qib_sdma_event_e7220_err_halted:
 940                         break;
 941                 case qib_sdma_event_e7322_err_halted:
 942                         break;
 943                 case qib_sdma_event_e90_timer_tick:
 944                         break;
 945                 }
 946                 break;
 947
 948         case qib_sdma_state_s50_hw_halt_wait:
 949                 switch (event) {
 950                 case qib_sdma_event_e00_go_hw_down:
 951                         sdma_set_state(ppd, qib_sdma_state_s00_hw_down);
 952                         sdma_start_sw_clean_up(ppd);
 953                         break;
 954                 case qib_sdma_event_e10_go_hw_start:
 955                         break;
 956                 case qib_sdma_event_e20_hw_started:
 957                         break;
 958                 case qib_sdma_event_e30_go_running:
 959                         ss->go_s99_running = 1;
 960                         break;
 961                 case qib_sdma_event_e40_sw_cleaned:
 962                         break;
 963                 case qib_sdma_event_e50_hw_cleaned:
 964                         break;
 965                 case qib_sdma_event_e60_hw_halted:
 966                         sdma_set_state(ppd,
 967                                        qib_sdma_state_s40_hw_clean_up_wait);
 968                         ppd->dd->f_sdma_hw_clean_up(ppd);
 969                         break;
 970                 case qib_sdma_event_e70_go_idle:
 971                         ss->go_s99_running = 0;
 972                         break;
 973                 case qib_sdma_event_e7220_err_halted:
 974                         break;
 975                 case qib_sdma_event_e7322_err_halted:
 976                         break;
 977                 case qib_sdma_event_e90_timer_tick:
 978                         break;
 979                 }
 980                 break;
 981
 982         case qib_sdma_state_s99_running:
 983                 switch (event) {
 984                 case qib_sdma_event_e00_go_hw_down:
 985                         sdma_set_state(ppd, qib_sdma_state_s00_hw_down);
 986                         sdma_start_sw_clean_up(ppd);
 987                         break;
 988                 case qib_sdma_event_e10_go_hw_start:
 989                         break;
 990                 case qib_sdma_event_e20_hw_started:
 991                         break;
 992                 case qib_sdma_event_e30_go_running:
 993                         break;
 994                 case qib_sdma_event_e40_sw_cleaned:
 995                         break;
 996                 case qib_sdma_event_e50_hw_cleaned:
 997                         break;
 998                 case qib_sdma_event_e60_hw_halted:
 999                         sdma_set_state(ppd,
1000                                        qib_sdma_state_s30_sw_clean_up_wait);
1001                         sdma_start_sw_clean_up(ppd);
1002                         break;
1003                 case qib_sdma_event_e70_go_idle:
1004                         sdma_set_state(ppd, qib_sdma_state_s50_hw_halt_wait);
1005                         ss->go_s99_running = 0;
1006                         break;
1007                 case qib_sdma_event_e7220_err_halted:
1008                         sdma_set_state(ppd,
1009                                        qib_sdma_state_s30_sw_clean_up_wait);
1010                         sdma_start_sw_clean_up(ppd);
1011                         break;
1012                 case qib_sdma_event_e7322_err_halted:
1013                         sdma_set_state(ppd, qib_sdma_state_s50_hw_halt_wait);
1014                         break;
1015                 case qib_sdma_event_e90_timer_tick:
1016                         break;
1017                 }
1018                 break;
1019         }
1020
1021         ss->last_event = event;
1022 }