drivers/char/xillybus/xillybus_core.c

   1 /*
   2  * linux/drivers/misc/xillybus_core.c
   3  *
   4  * Copyright 2011 Xillybus Ltd, http://xillybus.com
   5  *
   6  * Driver for the Xillybus FPGA/host framework.
   7  *
   8  * This driver interfaces with a special IP core in an FPGA, setting up
   9  * a pipe between a hardware FIFO in the programmable logic and a device
  10  * file in the host. The number of such pipes and their attributes are
  11  * set up on the logic. This driver detects these automatically and
  12  * creates the device files accordingly.
  13  *
  14  * This program is free software; you can redistribute it and/or modify
  15  * it under the smems of the GNU General Public License as published by
  16  * the Free Software Foundation; version 2 of the License.
  17  */
  18
  19 #include <linux/list.h>
  20 #include <linux/device.h>
  21 #include <linux/module.h>
  22 #include <linux/io.h>
  23 #include <linux/dma-mapping.h>
  24 #include <linux/interrupt.h>
  25 #include <linux/sched.h>
  26 #include <linux/fs.h>
  27 #include <linux/cdev.h>
  28 #include <linux/spinlock.h>
  29 #include <linux/mutex.h>
  30 #include <linux/crc32.h>
  31 #include <linux/poll.h>
  32 #include <linux/delay.h>
  33 #include <linux/slab.h>
  34 #include <linux/workqueue.h>
  35 #include "xillybus.h"
  36
  37 MODULE_DESCRIPTION("Xillybus core functions");
  38 MODULE_AUTHOR("Eli Billauer, Xillybus Ltd.");
  39 MODULE_VERSION("1.07");
  40 MODULE_ALIAS("xillybus_core");
  41 MODULE_LICENSE("GPL v2");
  42
  43 /* General timeout is 100 ms, rx timeout is 10 ms */
  44 #define XILLY_RX_TIMEOUT (10*HZ/1000)
  45 #define XILLY_TIMEOUT (100*HZ/1000)
  46
  47 #define fpga_msg_ctrl_reg              0x0008
  48 #define fpga_dma_control_reg           0x0020
  49 #define fpga_dma_bufno_reg             0x0024
  50 #define fpga_dma_bufaddr_lowaddr_reg   0x0028
  51 #define fpga_dma_bufaddr_highaddr_reg  0x002c
  52 #define fpga_buf_ctrl_reg              0x0030
  53 #define fpga_buf_offset_reg            0x0034
  54 #define fpga_endian_reg                0x0040
  55
  56 #define XILLYMSG_OPCODE_RELEASEBUF 1
  57 #define XILLYMSG_OPCODE_QUIESCEACK 2
  58 #define XILLYMSG_OPCODE_FIFOEOF 3
  59 #define XILLYMSG_OPCODE_FATAL_ERROR 4
  60 #define XILLYMSG_OPCODE_NONEMPTY 5
  61
  62 static const char xillyname[] = "xillybus";
  63
  64 static struct class *xillybus_class;
  65
  66 /*
  67  * ep_list_lock is the last lock to be taken; No other lock requests are
  68  * allowed while holding it. It merely protects list_of_endpoints, and not
  69  * the endpoints listed in it.
  70  */
  71
  72 static LIST_HEAD(list_of_endpoints);
  73 static struct mutex ep_list_lock;
  74 static struct workqueue_struct *xillybus_wq;
  75
  76 /*
  77  * Locking scheme: Mutexes protect invocations of character device methods.
  78  * If both locks are taken, wr_mutex is taken first, rd_mutex second.
  79  *
  80  * wr_spinlock protects wr_*_buf_idx, wr_empty, wr_sleepy, wr_ready and the
  81  * buffers' end_offset fields against changes made by IRQ handler (and in
  82  * theory, other file request handlers, but the mutex handles that). Nothing
  83  * else.
  84  * They are held for short direct memory manipulations. Needless to say,
  85  * no mutex locking is allowed when a spinlock is held.
  86  *
  87  * rd_spinlock does the same with rd_*_buf_idx, rd_empty and end_offset.
  88  *
  89  * register_mutex is endpoint-specific, and is held when non-atomic
  90  * register operations are performed. wr_mutex and rd_mutex may be
  91  * held when register_mutex is taken, but none of the spinlocks. Note that
  92  * register_mutex doesn't protect against sporadic buf_ctrl_reg writes
  93  * which are unrelated to buf_offset_reg, since they are harmless.
  94  *
  95  * Blocking on the wait queues is allowed with mutexes held, but not with
  96  * spinlocks.
  97  *
  98  * Only interruptible blocking is allowed on mutexes and wait queues.
  99  *
 100  * All in all, the locking order goes (with skips allowed, of course):
 101  * wr_mutex -> rd_mutex -> register_mutex -> wr_spinlock -> rd_spinlock
 102  */
 103
 104 static void malformed_message(struct xilly_endpoint *endpoint, u32 *buf)
 105 {
 106         int opcode;
 107         int msg_channel, msg_bufno, msg_data, msg_dir;
 108
 109         opcode = (buf[0] >> 24) & 0xff;
 110         msg_dir = buf[0] & 1;
 111         msg_channel = (buf[0] >> 1) & 0x7ff;
 112         msg_bufno = (buf[0] >> 12) & 0x3ff;
 113         msg_data = buf[1] & 0xfffffff;
 114
 115         dev_warn(endpoint->dev,
 116                  "Malformed message (skipping): opcode=%d, channel=%03x, dir=%d, bufno=%03x, data=%07x\n",
 117                  opcode, msg_channel, msg_dir, msg_bufno, msg_data);
 118 }
 119
 120 /*
 121  * xillybus_isr assumes the interrupt is allocated exclusively to it,
 122  * which is the natural case MSI and several other hardware-oriented
 123  * interrupts. Sharing is not allowed.
 124  */
 125
 126 irqreturn_t xillybus_isr(int irq, void *data)
 127 {
 128         struct xilly_endpoint *ep = data;
 129         u32 *buf;
 130         unsigned int buf_size;
 131         int i;
 132         int opcode;
 133         unsigned int msg_channel, msg_bufno, msg_data, msg_dir;
 134         struct xilly_channel *channel;
 135
 136         buf = ep->msgbuf_addr;
 137         buf_size = ep->msg_buf_size/sizeof(u32);
 138
 139         ep->ephw->hw_sync_sgl_for_cpu(ep,
 140                                       ep->msgbuf_dma_addr,
 141                                       ep->msg_buf_size,
 142                                       DMA_FROM_DEVICE);
 143
 144         for (i = 0; i < buf_size; i += 2) {
 145                 if (((buf[i+1] >> 28) & 0xf) != ep->msg_counter) {
 146                         malformed_message(ep, &buf[i]);
 147                         dev_warn(ep->dev,
 148                                  "Sending a NACK on counter %x (instead of %x) on entry %d\n",
 149                                  ((buf[i+1] >> 28) & 0xf),
 150                                  ep->msg_counter,
 151                                  i/2);
 152
 153                         if (++ep->failed_messages > 10) {
 154                                 dev_err(ep->dev,
 155                                         "Lost sync with interrupt messages. Stopping.\n");
 156                         } else {
 157                                 ep->ephw->hw_sync_sgl_for_device(
 158                                         ep,
 159                                         ep->msgbuf_dma_addr,
 160                                         ep->msg_buf_size,
 161                                         DMA_FROM_DEVICE);
 162
 163                                 iowrite32(0x01,  /* Message NACK */
 164                                           ep->registers + fpga_msg_ctrl_reg);
 165                         }
 166                         return IRQ_HANDLED;
 167                 } else if (buf[i] & (1 << 22)) /* Last message */
 168                         break;
 169         }
 170
 171         if (i >= buf_size) {
 172                 dev_err(ep->dev, "Bad interrupt message. Stopping.\n");
 173                 return IRQ_HANDLED;
 174         }
 175
 176         buf_size = i + 2;
 177
 178         for (i = 0; i < buf_size; i += 2) { /* Scan through messages */
 179                 opcode = (buf[i] >> 24) & 0xff;
 180
 181                 msg_dir = buf[i] & 1;
 182                 msg_channel = (buf[i] >> 1) & 0x7ff;
 183                 msg_bufno = (buf[i] >> 12) & 0x3ff;
 184                 msg_data = buf[i+1] & 0xfffffff;
 185
 186                 switch (opcode) {
 187                 case XILLYMSG_OPCODE_RELEASEBUF:
 188                         if ((msg_channel > ep->num_channels) ||
 189                             (msg_channel == 0)) {
 190                                 malformed_message(ep, &buf[i]);
 191                                 break;
 192                         }
 193
 194                         channel = ep->channels[msg_channel];
 195
 196                         if (msg_dir) { /* Write channel */
 197                                 if (msg_bufno >= channel->num_wr_buffers) {
 198                                         malformed_message(ep, &buf[i]);
 199                                         break;
 200                                 }
 201                                 spin_lock(&channel->wr_spinlock);
 202                                 channel->wr_buffers[msg_bufno]->end_offset =
 203                                         msg_data;
 204                                 channel->wr_fpga_buf_idx = msg_bufno;
 205                                 channel->wr_empty = 0;
 206                                 channel->wr_sleepy = 0;
 207                                 spin_unlock(&channel->wr_spinlock);
 208
 209                                 wake_up_interruptible(&channel->wr_wait);
 210
 211                         } else {
 212                                 /* Read channel */
 213
 214                                 if (msg_bufno >= channel->num_rd_buffers) {
 215                                         malformed_message(ep, &buf[i]);
 216                                         break;
 217                                 }
 218
 219                                 spin_lock(&channel->rd_spinlock);
 220                                 channel->rd_fpga_buf_idx = msg_bufno;
 221                                 channel->rd_full = 0;
 222                                 spin_unlock(&channel->rd_spinlock);
 223
 224                                 wake_up_interruptible(&channel->rd_wait);
 225                                 if (!channel->rd_synchronous)
 226                                         queue_delayed_work(
 227                                                 xillybus_wq,
 228                                                 &channel->rd_workitem,
 229                                                 XILLY_RX_TIMEOUT);
 230                         }
 231
 232                         break;
 233                 case XILLYMSG_OPCODE_NONEMPTY:
 234                         if ((msg_channel > ep->num_channels) ||
 235                             (msg_channel == 0) || (!msg_dir) ||
 236                             !ep->channels[msg_channel]->wr_supports_nonempty) {
 237                                 malformed_message(ep, &buf[i]);
 238                                 break;
 239                         }
 240
 241                         channel = ep->channels[msg_channel];
 242
 243                         if (msg_bufno >= channel->num_wr_buffers) {
 244                                 malformed_message(ep, &buf[i]);
 245                                 break;
 246                         }
 247                         spin_lock(&channel->wr_spinlock);
 248                         if (msg_bufno == channel->wr_host_buf_idx)
 249                                 channel->wr_ready = 1;
 250                         spin_unlock(&channel->wr_spinlock);
 251
 252                         wake_up_interruptible(&channel->wr_ready_wait);
 253
 254                         break;
 255                 case XILLYMSG_OPCODE_QUIESCEACK:
 256                         ep->idtlen = msg_data;
 257                         wake_up_interruptible(&ep->ep_wait);
 258
 259                         break;
 260                 case XILLYMSG_OPCODE_FIFOEOF:
 261                         if ((msg_channel > ep->num_channels) ||
 262                             (msg_channel == 0) || (!msg_dir) ||
 263                             !ep->channels[msg_channel]->num_wr_buffers) {
 264                                 malformed_message(ep, &buf[i]);
 265                                 break;
 266                         }
 267                         channel = ep->channels[msg_channel];
 268                         spin_lock(&channel->wr_spinlock);
 269                         channel->wr_eof = msg_bufno;
 270                         channel->wr_sleepy = 0;
 271
 272                         channel->wr_hangup = channel->wr_empty &&
 273                                 (channel->wr_host_buf_idx == msg_bufno);
 274
 275                         spin_unlock(&channel->wr_spinlock);
 276
 277                         wake_up_interruptible(&channel->wr_wait);
 278
 279                         break;
 280                 case XILLYMSG_OPCODE_FATAL_ERROR:
 281                         ep->fatal_error = 1;
 282                         wake_up_interruptible(&ep->ep_wait); /* For select() */
 283                         dev_err(ep->dev,
 284                                 "FPGA reported a fatal error. This means that the low-level communication with the device has failed. This hardware problem is most likely unrelated to Xillybus (neither kernel module nor FPGA core), but reports are still welcome. All I/O is aborted.\n");
 285                         break;
 286                 default:
 287                         malformed_message(ep, &buf[i]);
 288                         break;
 289                 }
 290         }
 291
 292         ep->ephw->hw_sync_sgl_for_device(ep,
 293                                          ep->msgbuf_dma_addr,
 294                                          ep->msg_buf_size,
 295                                          DMA_FROM_DEVICE);
 296
 297         ep->msg_counter = (ep->msg_counter + 1) & 0xf;
 298         ep->failed_messages = 0;
 299         iowrite32(0x03, ep->registers + fpga_msg_ctrl_reg); /* Message ACK */
 300
 301         return IRQ_HANDLED;
 302 }
 303 EXPORT_SYMBOL(xillybus_isr);
 304
 305 /*
 306  * A few trivial memory management functions.
 307  * NOTE: These functions are used only on probe and remove, and therefore
 308  * no locks are applied!
 309  */
 310
 311 static void xillybus_autoflush(struct work_struct *work);
 312
 313 struct xilly_alloc_state {
 314         void *salami;
 315         int left_of_salami;
 316         int nbuffer;
 317         enum dma_data_direction direction;
 318         u32 regdirection;
 319 };
 320
 321 static int xilly_get_dma_buffers(struct xilly_endpoint *ep,
 322                                  struct xilly_alloc_state *s,
 323                                  struct xilly_buffer **buffers,
 324                                  int bufnum, int bytebufsize)
 325 {
 326         int i, rc;
 327         dma_addr_t dma_addr;
 328         struct device *dev = ep->dev;
 329         struct xilly_buffer *this_buffer = NULL; /* Init to silence warning */
 330
 331         if (buffers) { /* Not the message buffer */
 332                 this_buffer = devm_kcalloc(dev, bufnum,
 333                                            sizeof(struct xilly_buffer),
 334                                            GFP_KERNEL);
 335                 if (!this_buffer)
 336                         return -ENOMEM;
 337         }
 338
 339         for (i = 0; i < bufnum; i++) {
 340                 /*
 341                  * Buffers are expected in descending size order, so there
 342                  * is either enough space for this buffer or none at all.
 343                  */
 344
 345                 if ((s->left_of_salami < bytebufsize) &&
 346                     (s->left_of_salami > 0)) {
 347                         dev_err(ep->dev,
 348                                 "Corrupt buffer allocation in IDT. Aborting.\n");
 349                         return -ENODEV;
 350                 }
 351
 352                 if (s->left_of_salami == 0) {
 353                         int allocorder, allocsize;
 354
 355                         allocsize = PAGE_SIZE;
 356                         allocorder = 0;
 357                         while (bytebufsize > allocsize) {
 358                                 allocsize *= 2;
 359                                 allocorder++;
 360                         }
 361
 362                         s->salami = (void *) devm_get_free_pages(
 363                                 dev,
 364                                 GFP_KERNEL | __GFP_DMA32 | __GFP_ZERO,
 365                                 allocorder);
 366                         if (!s->salami)
 367                                 return -ENOMEM;
 368
 369                         s->left_of_salami = allocsize;
 370                 }
 371
 372                 rc = ep->ephw->map_single(ep, s->salami,
 373                                           bytebufsize, s->direction,
 374                                           &dma_addr);
 375                 if (rc)
 376                         return rc;
 377
 378                 iowrite32((u32) (dma_addr & 0xffffffff),
 379                           ep->registers + fpga_dma_bufaddr_lowaddr_reg);
 380                 iowrite32(((u32) ((((u64) dma_addr) >> 32) & 0xffffffff)),
 381                           ep->registers + fpga_dma_bufaddr_highaddr_reg);
 382
 383                 if (buffers) { /* Not the message buffer */
 384                         this_buffer->addr = s->salami;
 385                         this_buffer->dma_addr = dma_addr;
 386                         buffers[i] = this_buffer++;
 387
 388                         iowrite32(s->regdirection | s->nbuffer++,
 389                                   ep->registers + fpga_dma_bufno_reg);
 390                 } else {
 391                         ep->msgbuf_addr = s->salami;
 392                         ep->msgbuf_dma_addr = dma_addr;
 393                         ep->msg_buf_size = bytebufsize;
 394
 395                         iowrite32(s->regdirection,
 396                                   ep->registers + fpga_dma_bufno_reg);
 397                 }
 398
 399                 s->left_of_salami -= bytebufsize;
 400                 s->salami += bytebufsize;
 401         }
 402         return 0;
 403 }
 404
 405 static int xilly_setupchannels(struct xilly_endpoint *ep,
 406                                unsigned char *chandesc,
 407                                int entries)
 408 {
 409         struct device *dev = ep->dev;
 410         int i, entry, rc;
 411         struct xilly_channel *channel;
 412         int channelnum, bufnum, bufsize, format, is_writebuf;
 413         int bytebufsize;
 414         int synchronous, allowpartial, exclusive_open, seekable;
 415         int supports_nonempty;
 416         int msg_buf_done = 0;
 417
 418         struct xilly_alloc_state rd_alloc = {
 419                 .salami = NULL,
 420                 .left_of_salami = 0,
 421                 .nbuffer = 1,
 422                 .direction = DMA_TO_DEVICE,
 423                 .regdirection = 0,
 424         };
 425
 426         struct xilly_alloc_state wr_alloc = {
 427                 .salami = NULL,
 428                 .left_of_salami = 0,
 429                 .nbuffer = 1,
 430                 .direction = DMA_FROM_DEVICE,
 431                 .regdirection = 0x80000000,
 432         };
 433
 434         channel = devm_kcalloc(dev, ep->num_channels,
 435                                sizeof(struct xilly_channel), GFP_KERNEL);
 436         if (!channel)
 437                 return -ENOMEM;
 438
 439         ep->channels = devm_kcalloc(dev, ep->num_channels + 1,
 440                                     sizeof(struct xilly_channel *),
 441                                     GFP_KERNEL);
 442         if (!ep->channels)
 443                 return -ENOMEM;
 444
 445         ep->channels[0] = NULL; /* Channel 0 is message buf. */
 446
 447         /* Initialize all channels with defaults */
 448
 449         for (i = 1; i <= ep->num_channels; i++) {
 450                 channel->wr_buffers = NULL;
 451                 channel->rd_buffers = NULL;
 452                 channel->num_wr_buffers = 0;
 453                 channel->num_rd_buffers = 0;
 454                 channel->wr_fpga_buf_idx = -1;
 455                 channel->wr_host_buf_idx = 0;
 456                 channel->wr_host_buf_pos = 0;
 457                 channel->wr_empty = 1;
 458                 channel->wr_ready = 0;
 459                 channel->wr_sleepy = 1;
 460                 channel->rd_fpga_buf_idx = 0;
 461                 channel->rd_host_buf_idx = 0;
 462                 channel->rd_host_buf_pos = 0;
 463                 channel->rd_full = 0;
 464                 channel->wr_ref_count = 0;
 465                 channel->rd_ref_count = 0;
 466
 467                 spin_lock_init(&channel->wr_spinlock);
 468                 spin_lock_init(&channel->rd_spinlock);
 469                 mutex_init(&channel->wr_mutex);
 470                 mutex_init(&channel->rd_mutex);
 471                 init_waitqueue_head(&channel->rd_wait);
 472                 init_waitqueue_head(&channel->wr_wait);
 473                 init_waitqueue_head(&channel->wr_ready_wait);
 474
 475                 INIT_DELAYED_WORK(&channel->rd_workitem, xillybus_autoflush);
 476
 477                 channel->endpoint = ep;
 478                 channel->chan_num = i;
 479
 480                 channel->log2_element_size = 0;
 481
 482                 ep->channels[i] = channel++;
 483         }
 484
 485         for (entry = 0; entry < entries; entry++, chandesc += 4) {
 486                 struct xilly_buffer **buffers = NULL;
 487
 488                 is_writebuf = chandesc[0] & 0x01;
 489                 channelnum = (chandesc[0] >> 1) | ((chandesc[1] & 0x0f) << 7);
 490                 format = (chandesc[1] >> 4) & 0x03;
 491                 allowpartial = (chandesc[1] >> 6) & 0x01;
 492                 synchronous = (chandesc[1] >> 7) & 0x01;
 493                 bufsize = 1 << (chandesc[2] & 0x1f);
 494                 bufnum = 1 << (chandesc[3] & 0x0f);
 495                 exclusive_open = (chandesc[2] >> 7) & 0x01;
 496                 seekable = (chandesc[2] >> 6) & 0x01;
 497                 supports_nonempty = (chandesc[2] >> 5) & 0x01;
 498
 499                 if ((channelnum > ep->num_channels) ||
 500                     ((channelnum == 0) && !is_writebuf)) {
 501                         dev_err(ep->dev,
 502                                 "IDT requests channel out of range. Aborting.\n");
 503                         return -ENODEV;
 504                 }
 505
 506                 channel = ep->channels[channelnum]; /* NULL for msg channel */
 507
 508                 if (!is_writebuf || channelnum > 0) {
 509                         channel->log2_element_size = ((format > 2) ?
 510                                                       2 : format);
 511
 512                         bytebufsize = bufsize *
 513                                 (1 << channel->log2_element_size);
 514
 515                         buffers = devm_kcalloc(dev, bufnum,
 516                                                sizeof(struct xilly_buffer *),
 517                                                GFP_KERNEL);
 518                         if (!buffers)
 519                                 return -ENOMEM;
 520                 } else {
 521                         bytebufsize = bufsize << 2;
 522                 }
 523
 524                 if (!is_writebuf) {
 525                         channel->num_rd_buffers = bufnum;
 526                         channel->rd_buf_size = bytebufsize;
 527                         channel->rd_allow_partial = allowpartial;
 528                         channel->rd_synchronous = synchronous;
 529                         channel->rd_exclusive_open = exclusive_open;
 530                         channel->seekable = seekable;
 531
 532                         channel->rd_buffers = buffers;
 533                         rc = xilly_get_dma_buffers(ep, &rd_alloc, buffers,
 534                                                    bufnum, bytebufsize);
 535                 } else if (channelnum > 0) {
 536                         channel->num_wr_buffers = bufnum;
 537                         channel->wr_buf_size = bytebufsize;
 538
 539                         channel->seekable = seekable;
 540                         channel->wr_supports_nonempty = supports_nonempty;
 541
 542                         channel->wr_allow_partial = allowpartial;
 543                         channel->wr_synchronous = synchronous;
 544                         channel->wr_exclusive_open = exclusive_open;
 545
 546                         channel->wr_buffers = buffers;
 547                         rc = xilly_get_dma_buffers(ep, &wr_alloc, buffers,
 548                                                    bufnum, bytebufsize);
 549                 } else {
 550                         rc = xilly_get_dma_buffers(ep, &wr_alloc, NULL,
 551                                                    bufnum, bytebufsize);
 552                         msg_buf_done++;
 553                 }
 554
 555                 if (rc)
 556                         return -ENOMEM;
 557         }
 558
 559         if (!msg_buf_done) {
 560                 dev_err(ep->dev,
 561                         "Corrupt IDT: No message buffer. Aborting.\n");
 562                 return -ENODEV;
 563         }
 564         return 0;
 565 }
 566
 567 static int xilly_scan_idt(struct xilly_endpoint *endpoint,
 568                           struct xilly_idt_handle *idt_handle)
 569 {
 570         int count = 0;
 571         unsigned char *idt = endpoint->channels[1]->wr_buffers[0]->addr;
 572         unsigned char *end_of_idt = idt + endpoint->idtlen - 4;
 573         unsigned char *scan;
 574         int len;
 575
 576         scan = idt;
 577         idt_handle->idt = idt;
 578
 579         scan++; /* Skip version number */
 580
 581         while ((scan <= end_of_idt) && *scan) {
 582                 while ((scan <= end_of_idt) && *scan++)
 583                         /* Do nothing, just scan thru string */;
 584                 count++;
 585         }
 586
 587         scan++;
 588
 589         if (scan > end_of_idt) {
 590                 dev_err(endpoint->dev,
 591                         "IDT device name list overflow. Aborting.\n");
 592                 return -ENODEV;
 593         }
 594         idt_handle->chandesc = scan;
 595
 596         len = endpoint->idtlen - (3 + ((int) (scan - idt)));
 597
 598         if (len & 0x03) {
 599                 dev_err(endpoint->dev,
 600                         "Corrupt IDT device name list. Aborting.\n");
 601                 return -ENODEV;
 602         }
 603
 604         idt_handle->entries = len >> 2;
 605         endpoint->num_channels = count;
 606
 607         return 0;
 608 }
 609
 610 static int xilly_obtain_idt(struct xilly_endpoint *endpoint)
 611 {
 612         struct xilly_channel *channel;
 613         unsigned char *version;
 614         long t;
 615
 616         channel = endpoint->channels[1]; /* This should be generated ad-hoc */
 617
 618         channel->wr_sleepy = 1;
 619
 620         iowrite32(1 |
 621                   (3 << 24), /* Opcode 3 for channel 0 = Send IDT */
 622                   endpoint->registers + fpga_buf_ctrl_reg);
 623
 624         t = wait_event_interruptible_timeout(channel->wr_wait,
 625                                              (!channel->wr_sleepy),
 626                                              XILLY_TIMEOUT);
 627
 628         if (t <= 0) {
 629                 dev_err(endpoint->dev, "Failed to obtain IDT. Aborting.\n");
 630
 631                 if (endpoint->fatal_error)
 632                         return -EIO;
 633
 634                 return -ENODEV;
 635         }
 636
 637         endpoint->ephw->hw_sync_sgl_for_cpu(
 638                 channel->endpoint,
 639                 channel->wr_buffers[0]->dma_addr,
 640                 channel->wr_buf_size,
 641                 DMA_FROM_DEVICE);
 642
 643         if (channel->wr_buffers[0]->end_offset != endpoint->idtlen) {
 644                 dev_err(endpoint->dev,
 645                         "IDT length mismatch (%d != %d). Aborting.\n",
 646                         channel->wr_buffers[0]->end_offset, endpoint->idtlen);
 647                 return -ENODEV;
 648         }
 649
 650         if (crc32_le(~0, channel->wr_buffers[0]->addr,
 651                      endpoint->idtlen+1) != 0) {
 652                 dev_err(endpoint->dev, "IDT failed CRC check. Aborting.\n");
 653                 return -ENODEV;
 654         }
 655
 656         version = channel->wr_buffers[0]->addr;
 657
 658         /* Check version number. Reject anything above 0x82. */
 659         if (*version > 0x82) {
 660                 dev_err(endpoint->dev,
 661                         "No support for IDT version 0x%02x. Maybe the xillybus driver needs an upgrade. Aborting.\n",
 662                         *version);
 663                 return -ENODEV;
 664         }
 665
 666         return 0;
 667 }
 668
 669 static ssize_t xillybus_read(struct file *filp, char __user *userbuf,
 670                              size_t count, loff_t *f_pos)
 671 {
 672         ssize_t rc;
 673         unsigned long flags;
 674         int bytes_done = 0;
 675         int no_time_left = 0;
 676         long deadline, left_to_sleep;
 677         struct xilly_channel *channel = filp->private_data;
 678
 679         int empty, reached_eof, exhausted, ready;
 680         /* Initializations are there only to silence warnings */
 681
 682         int howmany = 0, bufpos = 0, bufidx = 0, bufferdone = 0;
 683         int waiting_bufidx;
 684
 685         if (channel->endpoint->fatal_error)
 686                 return -EIO;
 687
 688         deadline = jiffies + 1 + XILLY_RX_TIMEOUT;
 689
 690         rc = mutex_lock_interruptible(&channel->wr_mutex);
 691         if (rc)
 692                 return rc;
 693
 694         while (1) { /* Note that we may drop mutex within this loop */
 695                 int bytes_to_do = count - bytes_done;
 696
 697                 spin_lock_irqsave(&channel->wr_spinlock, flags);
 698
 699                 empty = channel->wr_empty;
 700                 ready = !empty || channel->wr_ready;
 701
 702                 if (!empty) {
 703                         bufidx = channel->wr_host_buf_idx;
 704                         bufpos = channel->wr_host_buf_pos;
 705                         howmany = ((channel->wr_buffers[bufidx]->end_offset
 706                                     + 1) << channel->log2_element_size)
 707                                 - bufpos;
 708
 709                         /* Update wr_host_* to its post-operation state */
 710                         if (howmany > bytes_to_do) {
 711                                 bufferdone = 0;
 712
 713                                 howmany = bytes_to_do;
 714                                 channel->wr_host_buf_pos += howmany;
 715                         } else {
 716                                 bufferdone = 1;
 717
 718                                 channel->wr_host_buf_pos = 0;
 719
 720                                 if (bufidx == channel->wr_fpga_buf_idx) {
 721                                         channel->wr_empty = 1;
 722                                         channel->wr_sleepy = 1;
 723                                         channel->wr_ready = 0;
 724                                 }
 725
 726                                 if (bufidx >= (channel->num_wr_buffers - 1))
 727                                         channel->wr_host_buf_idx = 0;
 728                                 else
 729                                         channel->wr_host_buf_idx++;
 730                         }
 731                 }
 732
 733                 /*
 734                  * Marking our situation after the possible changes above,
 735                  * for use after releasing the spinlock.
 736                  *
 737                  * empty = empty before change
 738                  * exhasted = empty after possible change
 739                  */
 740
 741                 reached_eof = channel->wr_empty &&
 742                         (channel->wr_host_buf_idx == channel->wr_eof);
 743                 channel->wr_hangup = reached_eof;
 744                 exhausted = channel->wr_empty;
 745                 waiting_bufidx = channel->wr_host_buf_idx;
 746
 747                 spin_unlock_irqrestore(&channel->wr_spinlock, flags);
 748
 749                 if (!empty) { /* Go on, now without the spinlock */
 750
 751                         if (bufpos == 0) /* Position zero means it's virgin */
 752                                 channel->endpoint->ephw->hw_sync_sgl_for_cpu(
 753                                         channel->endpoint,
 754                                         channel->wr_buffers[bufidx]->dma_addr,
 755                                         channel->wr_buf_size,
 756                                         DMA_FROM_DEVICE);
 757
 758                         if (copy_to_user(
 759                                     userbuf,
 760                                     channel->wr_buffers[bufidx]->addr
 761                                     + bufpos, howmany))
 762                                 rc = -EFAULT;
 763
 764                         userbuf += howmany;
 765                         bytes_done += howmany;
 766
 767                         if (bufferdone) {
 768                                 channel->endpoint->ephw->hw_sync_sgl_for_device(
 769                                         channel->endpoint,
 770                                         channel->wr_buffers[bufidx]->dma_addr,
 771                                         channel->wr_buf_size,
 772                                         DMA_FROM_DEVICE);
 773
 774                                 /*
 775                                  * Tell FPGA the buffer is done with. It's an
 776                                  * atomic operation to the FPGA, so what
 777                                  * happens with other channels doesn't matter,
 778                                  * and the certain channel is protected with
 779                                  * the channel-specific mutex.
 780                                  */
 781
 782                                 iowrite32(1 | (channel->chan_num << 1) |
 783                                           (bufidx << 12),
 784                                           channel->endpoint->registers +
 785                                           fpga_buf_ctrl_reg);
 786                         }
 787
 788                         if (rc) {
 789                                 mutex_unlock(&channel->wr_mutex);
 790                                 return rc;
 791                         }
 792                 }
 793
 794                 /* This includes a zero-count return = EOF */
 795                 if ((bytes_done >= count) || reached_eof)
 796                         break;
 797
 798                 if (!exhausted)
 799                         continue; /* More in RAM buffer(s)? Just go on. */
 800
 801                 if ((bytes_done > 0) &&
 802                     (no_time_left ||
 803                      (channel->wr_synchronous && channel->wr_allow_partial)))
 804                         break;
 805
 806                 /*
 807                  * Nonblocking read: The "ready" flag tells us that the FPGA
 808                  * has data to send. In non-blocking mode, if it isn't on,
 809                  * just return. But if there is, we jump directly to the point
 810                  * where we ask for the FPGA to send all it has, and wait
 811                  * until that data arrives. So in a sense, we *do* block in
 812                  * nonblocking mode, but only for a very short time.
 813                  */
 814
 815                 if (!no_time_left && (filp->f_flags & O_NONBLOCK)) {
 816                         if (bytes_done > 0)
 817                                 break;
 818
 819                         if (ready)
 820                                 goto desperate;
 821
 822                         rc = -EAGAIN;
 823                         break;
 824                 }
 825
 826                 if (!no_time_left || (bytes_done > 0)) {
 827                         /*
 828                          * Note that in case of an element-misaligned read
 829                          * request, offsetlimit will include the last element,
 830                          * which will be partially read from.
 831                          */
 832                         int offsetlimit = ((count - bytes_done) - 1) >>
 833                                 channel->log2_element_size;
 834                         int buf_elements = channel->wr_buf_size >>
 835                                 channel->log2_element_size;
 836
 837                         /*
 838                          * In synchronous mode, always send an offset limit.
 839                          * Just don't send a value too big.
 840                          */
 841
 842                         if (channel->wr_synchronous) {
 843                                 /* Don't request more than one buffer */
 844                                 if (channel->wr_allow_partial &&
 845                                     (offsetlimit >= buf_elements))
 846                                         offsetlimit = buf_elements - 1;
 847
 848                                 /* Don't request more than all buffers */
 849                                 if (!channel->wr_allow_partial &&
 850                                     (offsetlimit >=
 851                                      (buf_elements * channel->num_wr_buffers)))
 852                                         offsetlimit = buf_elements *
 853                                                 channel->num_wr_buffers - 1;
 854                         }
 855
 856                         /*
 857                          * In asynchronous mode, force early flush of a buffer
 858                          * only if that will allow returning a full count. The
 859                          * "offsetlimit < ( ... )" rather than "<=" excludes
 860                          * requesting a full buffer, which would obviously
 861                          * cause a buffer transmission anyhow
 862                          */
 863
 864                         if (channel->wr_synchronous ||
 865                             (offsetlimit < (buf_elements - 1))) {
 866                                 mutex_lock(&channel->endpoint->register_mutex);
 867
 868                                 iowrite32(offsetlimit,
 869                                           channel->endpoint->registers +
 870                                           fpga_buf_offset_reg);
 871
 872                                 iowrite32(1 | (channel->chan_num << 1) |
 873                                           (2 << 24) |  /* 2 = offset limit */
 874                                           (waiting_bufidx << 12),
 875                                           channel->endpoint->registers +
 876                                           fpga_buf_ctrl_reg);
 877
 878                                 mutex_unlock(&channel->endpoint->
 879                                              register_mutex);
 880                         }
 881                 }
 882
 883                 /*
 884                  * If partial completion is disallowed, there is no point in
 885                  * timeout sleeping. Neither if no_time_left is set and
 886                  * there's no data.
 887                  */
 888
 889                 if (!channel->wr_allow_partial ||
 890                     (no_time_left && (bytes_done == 0))) {
 891                         /*
 892                          * This do-loop will run more than once if another
 893                          * thread reasserted wr_sleepy before we got the mutex
 894                          * back, so we try again.
 895                          */
 896
 897                         do {
 898                                 mutex_unlock(&channel->wr_mutex);
 899
 900                                 if (wait_event_interruptible(
 901                                             channel->wr_wait,
 902                                             (!channel->wr_sleepy)))
 903                                         goto interrupted;
 904
 905                                 if (mutex_lock_interruptible(
 906                                             &channel->wr_mutex))
 907                                         goto interrupted;
 908                         } while (channel->wr_sleepy);
 909
 910                         continue;
 911
 912 interrupted: /* Mutex is not held if got here */
 913                         if (channel->endpoint->fatal_error)
 914                                 return -EIO;
 915                         if (bytes_done)
 916                                 return bytes_done;
 917                         if (filp->f_flags & O_NONBLOCK)
 918                                 return -EAGAIN; /* Don't admit snoozing */
 919                         return -EINTR;
 920                 }
 921
 922                 left_to_sleep = deadline - ((long) jiffies);
 923
 924                 /*
 925                  * If our time is out, skip the waiting. We may miss wr_sleepy
 926                  * being deasserted but hey, almost missing the train is like
 927                  * missing it.
 928                  */
 929
 930                 if (left_to_sleep > 0) {
 931                         left_to_sleep =
 932                                 wait_event_interruptible_timeout(
 933                                         channel->wr_wait,
 934                                         (!channel->wr_sleepy),
 935                                         left_to_sleep);
 936
 937                         if (left_to_sleep > 0) /* wr_sleepy deasserted */
 938                                 continue;
 939
 940                         if (left_to_sleep < 0) { /* Interrupt */
 941                                 mutex_unlock(&channel->wr_mutex);
 942                                 if (channel->endpoint->fatal_error)
 943                                         return -EIO;
 944                                 if (bytes_done)
 945                                         return bytes_done;
 946                                 return -EINTR;
 947                         }
 948                 }
 949
 950 desperate:
 951                 no_time_left = 1; /* We're out of sleeping time. Desperate! */
 952
 953                 if (bytes_done == 0) {
 954                         /*
 955                          * Reaching here means that we allow partial return,
 956                          * that we've run out of time, and that we have
 957                          * nothing to return.
 958                          * So tell the FPGA to send anything it has or gets.
 959                          */
 960
 961                         iowrite32(1 | (channel->chan_num << 1) |
 962                                   (3 << 24) |  /* Opcode 3, flush it all! */
 963                                   (waiting_bufidx << 12),
 964                                   channel->endpoint->registers +
 965                                   fpga_buf_ctrl_reg);
 966                 }
 967
 968                 /*
 969                  * Reaching here means that we *do* have data in the buffer,
 970                  * but the "partial" flag disallows returning less than
 971                  * required. And we don't have as much. So loop again,
 972                  * which is likely to end up blocking indefinitely until
 973                  * enough data has arrived.
 974                  */
 975         }
 976
 977         mutex_unlock(&channel->wr_mutex);
 978
 979         if (channel->endpoint->fatal_error)
 980                 return -EIO;
 981
 982         if (rc)
 983                 return rc;
 984
 985         return bytes_done;
 986 }
 987
 988 /*
 989  * The timeout argument takes values as follows:
 990  *  >0 : Flush with timeout
 991  * ==0 : Flush, and wait idefinitely for the flush to complete
 992  *  <0 : Autoflush: Flush only if there's a single buffer occupied
 993  */
 994
 995 static int xillybus_myflush(struct xilly_channel *channel, long timeout)
 996 {
 997         int rc;
 998         unsigned long flags;
 999
1000         int end_offset_plus1;
1001         int bufidx, bufidx_minus1;
1002         int i;
1003         int empty;
1004         int new_rd_host_buf_pos;
1005
1006         if (channel->endpoint->fatal_error)
1007                 return -EIO;
1008         rc = mutex_lock_interruptible(&channel->rd_mutex);
1009         if (rc)
1010                 return rc;
1011
1012         /*
1013          * Don't flush a closed channel. This can happen when the work queued
1014          * autoflush thread fires off after the file has closed. This is not
1015          * an error, just something to dismiss.
1016          */
1017
1018         if (!channel->rd_ref_count)
1019                 goto done;
1020
1021         bufidx = channel->rd_host_buf_idx;
1022
1023         bufidx_minus1 = (bufidx == 0) ?
1024                 channel->num_rd_buffers - 1 :
1025                 bufidx - 1;
1026
1027         end_offset_plus1 = channel->rd_host_buf_pos >>
1028                 channel->log2_element_size;
1029
1030         new_rd_host_buf_pos = channel->rd_host_buf_pos -
1031                 (end_offset_plus1 << channel->log2_element_size);
1032
1033         /* Submit the current buffer if it's nonempty */
1034         if (end_offset_plus1) {
1035                 unsigned char *tail = channel->rd_buffers[bufidx]->addr +
1036                         (end_offset_plus1 << channel->log2_element_size);
1037
1038                 /* Copy  unflushed data, so we can put it in next buffer */
1039                 for (i = 0; i < new_rd_host_buf_pos; i++)
1040                         channel->rd_leftovers[i] = *tail++;
1041
1042                 spin_lock_irqsave(&channel->rd_spinlock, flags);
1043
1044                 /* Autoflush only if a single buffer is occupied */
1045
1046                 if ((timeout < 0) &&
1047                     (channel->rd_full ||
1048                      (bufidx_minus1 != channel->rd_fpga_buf_idx))) {
1049                         spin_unlock_irqrestore(&channel->rd_spinlock, flags);
1050                         /*
1051                          * A new work item may be queued by the ISR exactly
1052                          * now, since the execution of a work item allows the
1053                          * queuing of a new one while it's running.
1054                          */
1055                         goto done;
1056                 }
1057
1058                 /* The 4th element is never needed for data, so it's a flag */
1059                 channel->rd_leftovers[3] = (new_rd_host_buf_pos != 0);
1060
1061                 /* Set up rd_full to reflect a certain moment's state */
1062
1063                 if (bufidx == channel->rd_fpga_buf_idx)
1064                         channel->rd_full = 1;
1065                 spin_unlock_irqrestore(&channel->rd_spinlock, flags);
1066
1067                 if (bufidx >= (channel->num_rd_buffers - 1))
1068                         channel->rd_host_buf_idx = 0;
1069                 else
1070                         channel->rd_host_buf_idx++;
1071
1072                 channel->endpoint->ephw->hw_sync_sgl_for_device(
1073                         channel->endpoint,
1074                         channel->rd_buffers[bufidx]->dma_addr,
1075                         channel->rd_buf_size,
1076                         DMA_TO_DEVICE);
1077
1078                 mutex_lock(&channel->endpoint->register_mutex);
1079
1080                 iowrite32(end_offset_plus1 - 1,
1081                           channel->endpoint->registers + fpga_buf_offset_reg);
1082
1083                 iowrite32((channel->chan_num << 1) | /* Channel ID */
1084                           (2 << 24) |  /* Opcode 2, submit buffer */
1085                           (bufidx << 12),
1086                           channel->endpoint->registers + fpga_buf_ctrl_reg);
1087
1088                 mutex_unlock(&channel->endpoint->register_mutex);
1089         } else if (bufidx == 0) {
1090                 bufidx = channel->num_rd_buffers - 1;
1091         } else {
1092                 bufidx--;
1093         }
1094
1095         channel->rd_host_buf_pos = new_rd_host_buf_pos;
1096
1097         if (timeout < 0)
1098                 goto done; /* Autoflush */
1099
1100         /*
1101          * bufidx is now the last buffer written to (or equal to
1102          * rd_fpga_buf_idx if buffer was never written to), and
1103          * channel->rd_host_buf_idx the one after it.
1104          *
1105          * If bufidx == channel->rd_fpga_buf_idx we're either empty or full.
1106          */
1107
1108         while (1) { /* Loop waiting for draining of buffers */
1109                 spin_lock_irqsave(&channel->rd_spinlock, flags);
1110
1111                 if (bufidx != channel->rd_fpga_buf_idx)
1112                         channel->rd_full = 1; /*
1113                                                * Not really full,
1114                                                * but needs waiting.
1115                                                */
1116
1117                 empty = !channel->rd_full;
1118
1119                 spin_unlock_irqrestore(&channel->rd_spinlock, flags);
1120
1121                 if (empty)
1122                         break;
1123
1124                 /*
1125                  * Indefinite sleep with mutex taken. With data waiting for
1126                  * flushing user should not be surprised if open() for write
1127                  * sleeps.
1128                  */
1129                 if (timeout == 0)
1130                         wait_event_interruptible(channel->rd_wait,
1131                                                  (!channel->rd_full));
1132
1133                 else if (wait_event_interruptible_timeout(
1134                                  channel->rd_wait,
1135                                  (!channel->rd_full),
1136                                  timeout) == 0) {
1137                         dev_warn(channel->endpoint->dev,
1138                                  "Timed out while flushing. Output data may be lost.\n");
1139
1140                         rc = -ETIMEDOUT;
1141                         break;
1142                 }
1143
1144                 if (channel->rd_full) {
1145                         rc = -EINTR;
1146                         break;
1147                 }
1148         }
1149
1150 done:
1151         mutex_unlock(&channel->rd_mutex);
1152
1153         if (channel->endpoint->fatal_error)
1154                 return -EIO;
1155
1156         return rc;
1157 }
1158
1159 static int xillybus_flush(struct file *filp, fl_owner_t id)
1160 {
1161         if (!(filp->f_mode & FMODE_WRITE))
1162                 return 0;
1163
1164         return xillybus_myflush(filp->private_data, HZ); /* 1 second timeout */
1165 }
1166
1167 static void xillybus_autoflush(struct work_struct *work)
1168 {
1169         struct delayed_work *workitem = container_of(
1170                 work, struct delayed_work, work);
1171         struct xilly_channel *channel = container_of(
1172                 workitem, struct xilly_channel, rd_workitem);
1173         int rc;
1174
1175         rc = xillybus_myflush(channel, -1);
1176         if (rc == -EINTR)
1177                 dev_warn(channel->endpoint->dev,
1178                          "Autoflush failed because work queue thread got a signal.\n");
1179         else if (rc)
1180                 dev_err(channel->endpoint->dev,
1181                         "Autoflush failed under weird circumstances.\n");
1182 }
1183
1184 static ssize_t xillybus_write(struct file *filp, const char __user *userbuf,
1185                               size_t count, loff_t *f_pos)
1186 {
1187         ssize_t rc;
1188         unsigned long flags;
1189         int bytes_done = 0;
1190         struct xilly_channel *channel = filp->private_data;
1191
1192         int full, exhausted;
1193         /* Initializations are there only to silence warnings */
1194
1195         int howmany = 0, bufpos = 0, bufidx = 0, bufferdone = 0;
1196         int end_offset_plus1 = 0;
1197
1198         if (channel->endpoint->fatal_error)
1199                 return -EIO;
1200
1201         rc = mutex_lock_interruptible(&channel->rd_mutex);
1202         if (rc)
1203                 return rc;
1204
1205         while (1) {
1206                 int bytes_to_do = count - bytes_done;
1207
1208                 spin_lock_irqsave(&channel->rd_spinlock, flags);
1209
1210                 full = channel->rd_full;
1211
1212                 if (!full) {
1213                         bufidx = channel->rd_host_buf_idx;
1214                         bufpos = channel->rd_host_buf_pos;
1215                         howmany = channel->rd_buf_size - bufpos;
1216
1217                         /*
1218                          * Update rd_host_* to its state after this operation.
1219                          * count=0 means committing the buffer immediately,
1220                          * which is like flushing, but not necessarily block.
1221                          */
1222
1223                         if ((howmany > bytes_to_do) &&
1224                             (count ||
1225                              ((bufpos >> channel->log2_element_size) == 0))) {
1226                                 bufferdone = 0;
1227
1228                                 howmany = bytes_to_do;
1229                                 channel->rd_host_buf_pos += howmany;
1230                         } else {
1231                                 bufferdone = 1;
1232
1233                                 if (count) {
1234                                         end_offset_plus1 =
1235                                                 channel->rd_buf_size >>
1236                                                 channel->log2_element_size;
1237                                         channel->rd_host_buf_pos = 0;
1238                                 } else {
1239                                         unsigned char *tail;
1240                                         int i;
1241
1242                                         howmany = 0;
1243
1244                                         end_offset_plus1 = bufpos >>
1245                                                 channel->log2_element_size;
1246
1247                                         channel->rd_host_buf_pos -=
1248                                                 end_offset_plus1 <<
1249                                                 channel->log2_element_size;
1250
1251                                         tail = channel->
1252                                                 rd_buffers[bufidx]->addr +
1253                                                 (end_offset_plus1 <<
1254                                                  channel->log2_element_size);
1255
1256                                         for (i = 0;
1257                                              i < channel->rd_host_buf_pos;
1258                                              i++)
1259                                                 channel->rd_leftovers[i] =
1260                                                         *tail++;
1261                                 }
1262
1263                                 if (bufidx == channel->rd_fpga_buf_idx)
1264                                         channel->rd_full = 1;
1265
1266                                 if (bufidx >= (channel->num_rd_buffers - 1))
1267                                         channel->rd_host_buf_idx = 0;
1268                                 else
1269                                         channel->rd_host_buf_idx++;
1270                         }
1271                 }
1272
1273                 /*
1274                  * Marking our situation after the possible changes above,
1275                  * for use  after releasing the spinlock.
1276                  *
1277                  * full = full before change
1278                  * exhasted = full after possible change
1279                  */
1280
1281                 exhausted = channel->rd_full;
1282
1283                 spin_unlock_irqrestore(&channel->rd_spinlock, flags);
1284
1285                 if (!full) { /* Go on, now without the spinlock */
1286                         unsigned char *head =
1287                                 channel->rd_buffers[bufidx]->addr;
1288                         int i;
1289
1290                         if ((bufpos == 0) || /* Zero means it's virgin */
1291                             (channel->rd_leftovers[3] != 0)) {
1292                                 channel->endpoint->ephw->hw_sync_sgl_for_cpu(
1293                                         channel->endpoint,
1294                                         channel->rd_buffers[bufidx]->dma_addr,
1295                                         channel->rd_buf_size,
1296                                         DMA_TO_DEVICE);
1297
1298                                 /* Virgin, but leftovers are due */
1299                                 for (i = 0; i < bufpos; i++)
1300                                         *head++ = channel->rd_leftovers[i];
1301
1302                                 channel->rd_leftovers[3] = 0; /* Clear flag */
1303                         }
1304
1305                         if (copy_from_user(
1306                                     channel->rd_buffers[bufidx]->addr + bufpos,
1307                                     userbuf, howmany))
1308                                 rc = -EFAULT;
1309
1310                         userbuf += howmany;
1311                         bytes_done += howmany;
1312
1313                         if (bufferdone) {
1314                                 channel->endpoint->ephw->hw_sync_sgl_for_device(
1315                                         channel->endpoint,
1316                                         channel->rd_buffers[bufidx]->dma_addr,
1317                                         channel->rd_buf_size,
1318                                         DMA_TO_DEVICE);
1319
1320                                 mutex_lock(&channel->endpoint->register_mutex);
1321
1322                                 iowrite32(end_offset_plus1 - 1,
1323                                           channel->endpoint->registers +
1324                                           fpga_buf_offset_reg);
1325
1326                                 iowrite32((channel->chan_num << 1) |
1327                                           (2 << 24) |  /* 2 = submit buffer */
1328                                           (bufidx << 12),
1329                                           channel->endpoint->registers +
1330                                           fpga_buf_ctrl_reg);
1331
1332                                 mutex_unlock(&channel->endpoint->
1333                                              register_mutex);
1334
1335                                 channel->rd_leftovers[3] =
1336                                         (channel->rd_host_buf_pos != 0);
1337                         }
1338
1339                         if (rc) {
1340                                 mutex_unlock(&channel->rd_mutex);
1341
1342                                 if (channel->endpoint->fatal_error)
1343                                         return -EIO;
1344
1345                                 if (!channel->rd_synchronous)
1346                                         queue_delayed_work(
1347                                                 xillybus_wq,
1348                                                 &channel->rd_workitem,
1349                                                 XILLY_RX_TIMEOUT);
1350
1351                                 return rc;
1352                         }
1353                 }
1354
1355                 if (bytes_done >= count)
1356                         break;
1357
1358                 if (!exhausted)
1359                         continue; /* If there's more space, just go on */
1360
1361                 if ((bytes_done > 0) && channel->rd_allow_partial)
1362                         break;
1363
1364                 /*
1365                  * Indefinite sleep with mutex taken. With data waiting for
1366                  * flushing, user should not be surprised if open() for write
1367                  * sleeps.
1368                  */
1369
1370                 if (filp->f_flags & O_NONBLOCK) {
1371                         rc = -EAGAIN;
1372                         break;
1373                 }
1374
1375                 if (wait_event_interruptible(channel->rd_wait,
1376                                              (!channel->rd_full))) {
1377                         mutex_unlock(&channel->rd_mutex);
1378
1379                         if (channel->endpoint->fatal_error)
1380                                 return -EIO;
1381
1382                         if (bytes_done)
1383                                 return bytes_done;
1384                         return -EINTR;
1385                 }
1386         }
1387
1388         mutex_unlock(&channel->rd_mutex);
1389
1390         if (!channel->rd_synchronous)
1391                 queue_delayed_work(xillybus_wq,
1392                                    &channel->rd_workitem,
1393                                    XILLY_RX_TIMEOUT);
1394
1395         if (channel->endpoint->fatal_error)
1396                 return -EIO;
1397
1398         if (rc)
1399                 return rc;
1400
1401         if ((channel->rd_synchronous) && (bytes_done > 0)) {
1402                 rc = xillybus_myflush(filp->private_data, 0); /* No timeout */
1403
1404                 if (rc && (rc != -EINTR))
1405                         return rc;
1406         }
1407
1408         return bytes_done;
1409 }
1410
1411 static int xillybus_open(struct inode *inode, struct file *filp)
1412 {
1413         int rc = 0;
1414         unsigned long flags;
1415         int minor = iminor(inode);
1416         int major = imajor(inode);
1417         struct xilly_endpoint *ep_iter, *endpoint = NULL;
1418         struct xilly_channel *channel;
1419
1420         mutex_lock(&ep_list_lock);
1421
1422         list_for_each_entry(ep_iter, &list_of_endpoints, ep_list) {
1423                 if ((ep_iter->major == major) &&
1424                     (minor >= ep_iter->lowest_minor) &&
1425                     (minor < (ep_iter->lowest_minor +
1426                               ep_iter->num_channels))) {
1427                         endpoint = ep_iter;
1428                         break;
1429                 }
1430         }
1431         mutex_unlock(&ep_list_lock);
1432
1433         if (!endpoint) {
1434                 pr_err("xillybus: open() failed to find a device for major=%d and minor=%d\n",
1435                        major, minor);
1436                 return -ENODEV;
1437         }
1438
1439         if (endpoint->fatal_error)
1440                 return -EIO;
1441
1442         channel = endpoint->channels[1 + minor - endpoint->lowest_minor];
1443         filp->private_data = channel;
1444
1445         /*
1446          * It gets complicated because:
1447          * 1. We don't want to take a mutex we don't have to
1448          * 2. We don't want to open one direction if the other will fail.
1449          */
1450
1451         if ((filp->f_mode & FMODE_READ) && (!channel->num_wr_buffers))
1452                 return -ENODEV;
1453
1454         if ((filp->f_mode & FMODE_WRITE) && (!channel->num_rd_buffers))
1455                 return -ENODEV;
1456
1457         if ((filp->f_mode & FMODE_READ) && (filp->f_flags & O_NONBLOCK) &&
1458             (channel->wr_synchronous || !channel->wr_allow_partial ||
1459              !channel->wr_supports_nonempty)) {
1460                 dev_err(endpoint->dev,
1461                         "open() failed: O_NONBLOCK not allowed for read on this device\n");
1462                 return -ENODEV;
1463         }
1464
1465         if ((filp->f_mode & FMODE_WRITE) && (filp->f_flags & O_NONBLOCK) &&
1466             (channel->rd_synchronous || !channel->rd_allow_partial)) {
1467                 dev_err(endpoint->dev,
1468                         "open() failed: O_NONBLOCK not allowed for write on this device\n");
1469                 return -ENODEV;
1470         }
1471
1472         /*
1473          * Note: open() may block on getting mutexes despite O_NONBLOCK.
1474          * This shouldn't occur normally, since multiple open of the same
1475          * file descriptor is almost always prohibited anyhow
1476          * (*_exclusive_open is normally set in real-life systems).
1477          */
1478
1479         if (filp->f_mode & FMODE_READ) {
1480                 rc = mutex_lock_interruptible(&channel->wr_mutex);
1481                 if (rc)
1482                         return rc;
1483         }
1484
1485         if (filp->f_mode & FMODE_WRITE) {
1486                 rc = mutex_lock_interruptible(&channel->rd_mutex);
1487                 if (rc)
1488                         goto unlock_wr;
1489         }
1490
1491         if ((filp->f_mode & FMODE_READ) &&
1492             (channel->wr_ref_count != 0) &&
1493             (channel->wr_exclusive_open)) {
1494                 rc = -EBUSY;
1495                 goto unlock;
1496         }
1497
1498         if ((filp->f_mode & FMODE_WRITE) &&
1499             (channel->rd_ref_count != 0) &&
1500             (channel->rd_exclusive_open)) {
1501                 rc = -EBUSY;
1502                 goto unlock;
1503         }
1504
1505         if (filp->f_mode & FMODE_READ) {
1506                 if (channel->wr_ref_count == 0) { /* First open of file */
1507                         /* Move the host to first buffer */
1508                         spin_lock_irqsave(&channel->wr_spinlock, flags);
1509                         channel->wr_host_buf_idx = 0;
1510                         channel->wr_host_buf_pos = 0;
1511                         channel->wr_fpga_buf_idx = -1;
1512                         channel->wr_empty = 1;
1513                         channel->wr_ready = 0;
1514                         channel->wr_sleepy = 1;
1515                         channel->wr_eof = -1;
1516                         channel->wr_hangup = 0;
1517
1518                         spin_unlock_irqrestore(&channel->wr_spinlock, flags);
1519
1520                         iowrite32(1 | (channel->chan_num << 1) |
1521                                   (4 << 24) |  /* Opcode 4, open channel */
1522                                   ((channel->wr_synchronous & 1) << 23),
1523                                   channel->endpoint->registers +
1524                                   fpga_buf_ctrl_reg);
1525                 }
1526
1527                 channel->wr_ref_count++;
1528         }
1529
1530         if (filp->f_mode & FMODE_WRITE) {
1531                 if (channel->rd_ref_count == 0) { /* First open of file */
1532                         /* Move the host to first buffer */
1533                         spin_lock_irqsave(&channel->rd_spinlock, flags);
1534                         channel->rd_host_buf_idx = 0;
1535                         channel->rd_host_buf_pos = 0;
1536                         channel->rd_leftovers[3] = 0; /* No leftovers. */
1537                         channel->rd_fpga_buf_idx = channel->num_rd_buffers - 1;
1538                         channel->rd_full = 0;
1539
1540                         spin_unlock_irqrestore(&channel->rd_spinlock, flags);
1541
1542                         iowrite32((channel->chan_num << 1) |
1543                                   (4 << 24),   /* Opcode 4, open channel */
1544                                   channel->endpoint->registers +
1545                                   fpga_buf_ctrl_reg);
1546                 }
1547
1548                 channel->rd_ref_count++;
1549         }
1550
1551 unlock:
1552         if (filp->f_mode & FMODE_WRITE)
1553                 mutex_unlock(&channel->rd_mutex);
1554 unlock_wr:
1555         if (filp->f_mode & FMODE_READ)
1556                 mutex_unlock(&channel->wr_mutex);
1557
1558         if (!rc && (!channel->seekable))
1559                 return nonseekable_open(inode, filp);
1560
1561         return rc;
1562 }
1563
1564 static int xillybus_release(struct inode *inode, struct file *filp)
1565 {
1566         unsigned long flags;
1567         struct xilly_channel *channel = filp->private_data;
1568
1569         int buf_idx;
1570         int eof;
1571
1572         if (channel->endpoint->fatal_error)
1573                 return -EIO;
1574
1575         if (filp->f_mode & FMODE_WRITE) {
1576                 mutex_lock(&channel->rd_mutex);
1577
1578                 channel->rd_ref_count--;
1579
1580                 if (channel->rd_ref_count == 0) {
1581                         /*
1582                          * We rely on the kernel calling flush()
1583                          * before we get here.
1584                          */
1585
1586                         iowrite32((channel->chan_num << 1) | /* Channel ID */
1587                                   (5 << 24),  /* Opcode 5, close channel */
1588                                   channel->endpoint->registers +
1589                                   fpga_buf_ctrl_reg);
1590                 }
1591                 mutex_unlock(&channel->rd_mutex);
1592         }
1593
1594         if (filp->f_mode & FMODE_READ) {
1595                 mutex_lock(&channel->wr_mutex);
1596
1597                 channel->wr_ref_count--;
1598
1599                 if (channel->wr_ref_count == 0) {
1600                         iowrite32(1 | (channel->chan_num << 1) |
1601                                   (5 << 24),  /* Opcode 5, close channel */
1602                                   channel->endpoint->registers +
1603                                   fpga_buf_ctrl_reg);
1604
1605                         /*
1606                          * This is crazily cautious: We make sure that not
1607                          * only that we got an EOF (be it because we closed
1608                          * the channel or because of a user's EOF), but verify
1609                          * that it's one beyond the last buffer arrived, so
1610                          * we have no leftover buffers pending before wrapping
1611                          * up (which can only happen in asynchronous channels,
1612                          * BTW)
1613                          */
1614
1615                         while (1) {
1616                                 spin_lock_irqsave(&channel->wr_spinlock,
1617                                                   flags);
1618                                 buf_idx = channel->wr_fpga_buf_idx;
1619                                 eof = channel->wr_eof;
1620                                 channel->wr_sleepy = 1;
1621                                 spin_unlock_irqrestore(&channel->wr_spinlock,
1622                                                        flags);
1623
1624                                 /*
1625                                  * Check if eof points at the buffer after
1626                                  * the last one the FPGA submitted. Note that
1627                                  * no EOF is marked by negative eof.
1628                                  */
1629
1630                                 buf_idx++;
1631                                 if (buf_idx == channel->num_wr_buffers)
1632                                         buf_idx = 0;
1633
1634                                 if (buf_idx == eof)
1635                                         break;
1636
1637                                 /*
1638                                  * Steal extra 100 ms if awaken by interrupt.
1639                                  * This is a simple workaround for an
1640                                  * interrupt pending when entering, which would
1641                                  * otherwise result in declaring the hardware
1642                                  * non-responsive.
1643                                  */
1644
1645                                 if (wait_event_interruptible(
1646                                             channel->wr_wait,
1647                                             (!channel->wr_sleepy)))
1648                                         msleep(100);
1649
1650                                 if (channel->wr_sleepy) {
1651                                         mutex_unlock(&channel->wr_mutex);
1652                                         dev_warn(channel->endpoint->dev,
1653                                                  "Hardware failed to respond to close command, therefore left in messy state.\n");
1654                                         return -EINTR;
1655                                 }
1656                         }
1657                 }
1658
1659                 mutex_unlock(&channel->wr_mutex);
1660         }
1661
1662         return 0;
1663 }
1664
1665 static loff_t xillybus_llseek(struct file *filp, loff_t offset, int whence)
1666 {
1667         struct xilly_channel *channel = filp->private_data;
1668         loff_t pos = filp->f_pos;
1669         int rc = 0;
1670
1671         /*
1672          * Take both mutexes not allowing interrupts, since it seems like
1673          * common applications don't expect an -EINTR here. Besides, multiple
1674          * access to a single file descriptor on seekable devices is a mess
1675          * anyhow.
1676          */
1677
1678         if (channel->endpoint->fatal_error)
1679                 return -EIO;
1680
1681         mutex_lock(&channel->wr_mutex);
1682         mutex_lock(&channel->rd_mutex);
1683
1684         switch (whence) {
1685         case SEEK_SET:
1686                 pos = offset;
1687                 break;
1688         case SEEK_CUR:
1689                 pos += offset;
1690                 break;
1691         case SEEK_END:
1692                 pos = offset; /* Going to the end => to the beginning */
1693                 break;
1694         default:
1695                 rc = -EINVAL;
1696                 goto end;
1697         }
1698
1699         /* In any case, we must finish on an element boundary */
1700         if (pos & ((1 << channel->log2_element_size) - 1)) {
1701                 rc = -EINVAL;
1702                 goto end;
1703         }
1704
1705         mutex_lock(&channel->endpoint->register_mutex);
1706
1707         iowrite32(pos >> channel->log2_element_size,
1708                   channel->endpoint->registers + fpga_buf_offset_reg);
1709
1710         iowrite32((channel->chan_num << 1) |
1711                   (6 << 24),  /* Opcode 6, set address */
1712                   channel->endpoint->registers + fpga_buf_ctrl_reg);
1713
1714         mutex_unlock(&channel->endpoint->register_mutex);
1715
1716 end:
1717         mutex_unlock(&channel->rd_mutex);
1718         mutex_unlock(&channel->wr_mutex);
1719
1720         if (rc) /* Return error after releasing mutexes */
1721                 return rc;
1722
1723         filp->f_pos = pos;
1724
1725         /*
1726          * Since seekable devices are allowed only when the channel is
1727          * synchronous, we assume that there is no data pending in either
1728          * direction (which holds true as long as no concurrent access on the
1729          * file descriptor takes place).
1730          * The only thing we may need to throw away is leftovers from partial
1731          * write() flush.
1732          */
1733
1734         channel->rd_leftovers[3] = 0;
1735
1736         return pos;
1737 }
1738
1739 static unsigned int xillybus_poll(struct file *filp, poll_table *wait)
1740 {
1741         struct xilly_channel *channel = filp->private_data;
1742         unsigned int mask = 0;
1743         unsigned long flags;
1744
1745         poll_wait(filp, &channel->endpoint->ep_wait, wait);
1746
1747         /*
1748          * poll() won't play ball regarding read() channels which
1749          * aren't asynchronous and support the nonempty message. Allowing
1750          * that will create situations where data has been delivered at
1751          * the FPGA, and users expecting select() to wake up, which it may
1752          * not.
1753          */
1754
1755         if (!channel->wr_synchronous && channel->wr_supports_nonempty) {
1756                 poll_wait(filp, &channel->wr_wait, wait);
1757                 poll_wait(filp, &channel->wr_ready_wait, wait);
1758
1759                 spin_lock_irqsave(&channel->wr_spinlock, flags);
1760                 if (!channel->wr_empty || channel->wr_ready)
1761                         mask |= POLLIN | POLLRDNORM;
1762
1763                 if (channel->wr_hangup)
1764                         /*
1765                          * Not POLLHUP, because its behavior is in the
1766                          * mist, and POLLIN does what we want: Wake up
1767                          * the read file descriptor so it sees EOF.
1768                          */
1769                         mask |=  POLLIN | POLLRDNORM;
1770                 spin_unlock_irqrestore(&channel->wr_spinlock, flags);
1771         }
1772
1773         /*
1774          * If partial data write is disallowed on a write() channel,
1775          * it's pointless to ever signal OK to write, because is could
1776          * block despite some space being available.
1777          */
1778
1779         if (channel->rd_allow_partial) {
1780                 poll_wait(filp, &channel->rd_wait, wait);
1781
1782                 spin_lock_irqsave(&channel->rd_spinlock, flags);
1783                 if (!channel->rd_full)
1784                         mask |= POLLOUT | POLLWRNORM;
1785                 spin_unlock_irqrestore(&channel->rd_spinlock, flags);
1786         }
1787
1788         if (channel->endpoint->fatal_error)
1789                 mask |= POLLERR;
1790
1791         return mask;
1792 }
1793
1794 static const struct file_operations xillybus_fops = {
1795         .owner      = THIS_MODULE,
1796         .read       = xillybus_read,
1797         .write      = xillybus_write,
1798         .open       = xillybus_open,
1799         .flush      = xillybus_flush,
1800         .release    = xillybus_release,
1801         .llseek     = xillybus_llseek,
1802         .poll       = xillybus_poll,
1803 };
1804
1805 static int xillybus_init_chrdev(struct xilly_endpoint *endpoint,
1806                                 const unsigned char *idt)
1807 {
1808         int rc;
1809         dev_t dev;
1810         int devnum, i, minor, major;
1811         char devname[48];
1812         struct device *device;
1813
1814         rc = alloc_chrdev_region(&dev, 0, /* minor start */
1815                                  endpoint->num_channels,
1816                                  xillyname);
1817         if (rc) {
1818                 dev_warn(endpoint->dev, "Failed to obtain major/minors");
1819                 return rc;
1820         }
1821
1822         endpoint->major = major = MAJOR(dev);
1823         endpoint->lowest_minor = minor = MINOR(dev);
1824
1825         cdev_init(&endpoint->cdev, &xillybus_fops);
1826         endpoint->cdev.owner = endpoint->ephw->owner;
1827         rc = cdev_add(&endpoint->cdev, MKDEV(major, minor),
1828                       endpoint->num_channels);
1829         if (rc) {
1830                 dev_warn(endpoint->dev, "Failed to add cdev. Aborting.\n");
1831                 goto unregister_chrdev;
1832         }
1833
1834         idt++;
1835
1836         for (i = minor, devnum = 0;
1837              devnum < endpoint->num_channels;
1838              devnum++, i++) {
1839                 snprintf(devname, sizeof(devname)-1, "xillybus_%s", idt);
1840
1841                 devname[sizeof(devname)-1] = 0; /* Should never matter */
1842
1843                 while (*idt++)
1844                         /* Skip to next */;
1845
1846                 device = device_create(xillybus_class,
1847                                        NULL,
1848                                        MKDEV(major, i),
1849                                        NULL,
1850                                        "%s", devname);
1851
1852                 if (IS_ERR(device)) {
1853                         dev_warn(endpoint->dev,
1854                                  "Failed to create %s device. Aborting.\n",
1855                                  devname);
1856                         rc = -ENODEV;
1857                         goto unroll_device_create;
1858                 }
1859         }
1860
1861         dev_info(endpoint->dev, "Created %d device files.\n",
1862                  endpoint->num_channels);
1863         return 0; /* succeed */
1864
1865 unroll_device_create:
1866         devnum--; i--;
1867         for (; devnum >= 0; devnum--, i--)
1868                 device_destroy(xillybus_class, MKDEV(major, i));
1869
1870         cdev_del(&endpoint->cdev);
1871 unregister_chrdev:
1872         unregister_chrdev_region(MKDEV(major, minor), endpoint->num_channels);
1873
1874         return rc;
1875 }
1876
1877 static void xillybus_cleanup_chrdev(struct xilly_endpoint *endpoint)
1878 {
1879         int minor;
1880
1881         for (minor = endpoint->lowest_minor;
1882              minor < (endpoint->lowest_minor + endpoint->num_channels);
1883              minor++)
1884                 device_destroy(xillybus_class, MKDEV(endpoint->major, minor));
1885         cdev_del(&endpoint->cdev);
1886         unregister_chrdev_region(MKDEV(endpoint->major,
1887                                        endpoint->lowest_minor),
1888                                  endpoint->num_channels);
1889
1890         dev_info(endpoint->dev, "Removed %d device files.\n",
1891                  endpoint->num_channels);
1892 }
1893
1894 struct xilly_endpoint *xillybus_init_endpoint(struct pci_dev *pdev,
1895                                               struct device *dev,
1896                                               struct xilly_endpoint_hardware
1897                                               *ephw)
1898 {
1899         struct xilly_endpoint *endpoint;
1900
1901         endpoint = devm_kzalloc(dev, sizeof(*endpoint), GFP_KERNEL);
1902         if (!endpoint)
1903                 return NULL;
1904
1905         endpoint->pdev = pdev;
1906         endpoint->dev = dev;
1907         endpoint->ephw = ephw;
1908         endpoint->msg_counter = 0x0b;
1909         endpoint->failed_messages = 0;
1910         endpoint->fatal_error = 0;
1911
1912         init_waitqueue_head(&endpoint->ep_wait);
1913         mutex_init(&endpoint->register_mutex);
1914
1915         return endpoint;
1916 }
1917 EXPORT_SYMBOL(xillybus_init_endpoint);
1918
1919 static int xilly_quiesce(struct xilly_endpoint *endpoint)
1920 {
1921         long t;
1922
1923         endpoint->idtlen = -1;
1924
1925         iowrite32((u32) (endpoint->dma_using_dac & 0x0001),
1926                   endpoint->registers + fpga_dma_control_reg);
1927
1928         t = wait_event_interruptible_timeout(endpoint->ep_wait,
1929                                              (endpoint->idtlen >= 0),
1930                                              XILLY_TIMEOUT);
1931         if (t <= 0) {
1932                 dev_err(endpoint->dev,
1933                         "Failed to quiesce the device on exit.\n");
1934                 return -ENODEV;
1935         }
1936         return 0;
1937 }
1938
1939 int xillybus_endpoint_discovery(struct xilly_endpoint *endpoint)
1940 {
1941         int rc;
1942         long t;
1943
1944         void *bootstrap_resources;
1945         int idtbuffersize = (1 << PAGE_SHIFT);
1946         struct device *dev = endpoint->dev;
1947
1948         /*
1949          * The bogus IDT is used during bootstrap for allocating the initial
1950          * message buffer, and then the message buffer and space for the IDT
1951          * itself. The initial message buffer is of a single page's size, but
1952          * it's soon replaced with a more modest one (and memory is freed).
1953          */
1954
1955         unsigned char bogus_idt[8] = { 1, 224, (PAGE_SHIFT)-2, 0,
1956                                        3, 192, PAGE_SHIFT, 0 };
1957         struct xilly_idt_handle idt_handle;
1958
1959         /*
1960          * Writing the value 0x00000001 to Endianness register signals which
1961          * endianness this processor is using, so the FPGA can swap words as
1962          * necessary.
1963          */
1964
1965         iowrite32(1, endpoint->registers + fpga_endian_reg);
1966
1967         /* Bootstrap phase I: Allocate temporary message buffer */
1968
1969         bootstrap_resources = devres_open_group(dev, NULL, GFP_KERNEL);
1970         if (!bootstrap_resources)
1971                 return -ENOMEM;
1972
1973         endpoint->num_channels = 0;
1974
1975         rc = xilly_setupchannels(endpoint, bogus_idt, 1);
1976         if (rc)
1977                 return rc;
1978
1979         /* Clear the message subsystem (and counter in particular) */
1980         iowrite32(0x04, endpoint->registers + fpga_msg_ctrl_reg);
1981
1982         endpoint->idtlen = -1;
1983
1984         /*
1985          * Set DMA 32/64 bit mode, quiesce the device (?!) and get IDT
1986          * buffer size.
1987          */
1988         iowrite32((u32) (endpoint->dma_using_dac & 0x0001),
1989                   endpoint->registers + fpga_dma_control_reg);
1990
1991         t = wait_event_interruptible_timeout(endpoint->ep_wait,
1992                                              (endpoint->idtlen >= 0),
1993                                              XILLY_TIMEOUT);
1994         if (t <= 0) {
1995                 dev_err(endpoint->dev, "No response from FPGA. Aborting.\n");
1996                 return -ENODEV;
1997         }
1998
1999         /* Enable DMA */
2000         iowrite32((u32) (0x0002 | (endpoint->dma_using_dac & 0x0001)),
2001                   endpoint->registers + fpga_dma_control_reg);
2002
2003         /* Bootstrap phase II: Allocate buffer for IDT and obtain it */
2004         while (endpoint->idtlen >= idtbuffersize) {
2005                 idtbuffersize *= 2;
2006                 bogus_idt[6]++;
2007         }
2008
2009         endpoint->num_channels = 1;
2010
2011         rc = xilly_setupchannels(endpoint, bogus_idt, 2);
2012         if (rc)
2013                 goto failed_idt;
2014
2015         rc = xilly_obtain_idt(endpoint);
2016         if (rc)
2017                 goto failed_idt;
2018
2019         rc = xilly_scan_idt(endpoint, &idt_handle);
2020         if (rc)
2021                 goto failed_idt;
2022
2023         devres_close_group(dev, bootstrap_resources);
2024
2025         /* Bootstrap phase III: Allocate buffers according to IDT */
2026
2027         rc = xilly_setupchannels(endpoint,
2028                                  idt_handle.chandesc,
2029                                  idt_handle.entries);
2030         if (rc)
2031                 goto failed_idt;
2032
2033         /*
2034          * endpoint is now completely configured. We put it on the list
2035          * available to open() before registering the char device(s)
2036          */
2037
2038         mutex_lock(&ep_list_lock);
2039         list_add_tail(&endpoint->ep_list, &list_of_endpoints);
2040         mutex_unlock(&ep_list_lock);
2041
2042         rc = xillybus_init_chrdev(endpoint, idt_handle.idt);
2043         if (rc)
2044                 goto failed_chrdevs;
2045
2046         devres_release_group(dev, bootstrap_resources);
2047
2048         return 0;
2049
2050 failed_chrdevs:
2051         mutex_lock(&ep_list_lock);
2052         list_del(&endpoint->ep_list);
2053         mutex_unlock(&ep_list_lock);
2054
2055 failed_idt:
2056         xilly_quiesce(endpoint);
2057         flush_workqueue(xillybus_wq);
2058
2059         return rc;
2060 }
2061 EXPORT_SYMBOL(xillybus_endpoint_discovery);
2062
2063 void xillybus_endpoint_remove(struct xilly_endpoint *endpoint)
2064 {
2065         xillybus_cleanup_chrdev(endpoint);
2066
2067         mutex_lock(&ep_list_lock);
2068         list_del(&endpoint->ep_list);
2069         mutex_unlock(&ep_list_lock);
2070
2071         xilly_quiesce(endpoint);
2072
2073         /*
2074          * Flushing is done upon endpoint release to prevent access to memory
2075          * just about to be released. This makes the quiesce complete.
2076          */
2077         flush_workqueue(xillybus_wq);
2078 }
2079 EXPORT_SYMBOL(xillybus_endpoint_remove);
2080
2081 static int __init xillybus_init(void)
2082 {
2083         mutex_init(&ep_list_lock);
2084
2085         xillybus_class = class_create(THIS_MODULE, xillyname);
2086         if (IS_ERR(xillybus_class))
2087                 return PTR_ERR(xillybus_class);
2088
2089         xillybus_wq = alloc_workqueue(xillyname, 0, 0);
2090         if (!xillybus_wq) {
2091                 class_destroy(xillybus_class);
2092                 return -ENOMEM;
2093         }
2094
2095         return 0;
2096 }
2097
2098 static void __exit xillybus_exit(void)
2099 {
2100         /* flush_workqueue() was called for each endpoint released */
2101         destroy_workqueue(xillybus_wq);
2102
2103         class_destroy(xillybus_class);
2104 }
2105
2106 module_init(xillybus_init);
2107 module_exit(xillybus_exit);