drivers/spi/spi-imx.c

   1 // SPDX-License-Identifier: GPL-2.0+
   2 // Copyright 2004-2007 Freescale Semiconductor, Inc. All Rights Reserved.
   3 // Copyright (C) 2008 Juergen Beisert
   4
   5 #include <linux/clk.h>
   6 #include <linux/completion.h>
   7 #include <linux/delay.h>
   8 #include <linux/dmaengine.h>
   9 #include <linux/dma-mapping.h>
  10 #include <linux/err.h>
  11 #include <linux/gpio.h>
  12 #include <linux/interrupt.h>
  13 #include <linux/io.h>
  14 #include <linux/irq.h>
  15 #include <linux/kernel.h>
  16 #include <linux/module.h>
  17 #include <linux/platform_device.h>
  18 #include <linux/slab.h>
  19 #include <linux/spi/spi.h>
  20 #include <linux/spi/spi_bitbang.h>
  21 #include <linux/types.h>
  22 #include <linux/of.h>
  23 #include <linux/of_device.h>
  24 #include <linux/of_gpio.h>
  25
  26 #include <linux/platform_data/dma-imx.h>
  27 #include <linux/platform_data/spi-imx.h>
  28
  29 #define DRIVER_NAME "spi_imx"
  30
  31 #define MXC_CSPIRXDATA          0x00
  32 #define MXC_CSPITXDATA          0x04
  33 #define MXC_CSPICTRL            0x08
  34 #define MXC_CSPIINT             0x0c
  35 #define MXC_RESET               0x1c
  36
  37 /* generic defines to abstract from the different register layouts */
  38 #define MXC_INT_RR      (1 << 0) /* Receive data ready interrupt */
  39 #define MXC_INT_TE      (1 << 1) /* Transmit FIFO empty interrupt */
  40 #define MXC_INT_RDR     BIT(4) /* Receive date threshold interrupt */
  41
  42 /* The maximum  bytes that a sdma BD can transfer.*/
  43 #define MAX_SDMA_BD_BYTES  (1 << 15)
  44 #define MX51_ECSPI_CTRL_MAX_BURST       512
  45 /* The maximum bytes that IMX53_ECSPI can transfer in slave mode.*/
  46 #define MX53_MAX_TRANSFER_BYTES         512
  47
  48 enum spi_imx_devtype {
  49         IMX1_CSPI,
  50         IMX21_CSPI,
  51         IMX27_CSPI,
  52         IMX31_CSPI,
  53         IMX35_CSPI,     /* CSPI on all i.mx except above */
  54         IMX51_ECSPI,    /* ECSPI on i.mx51 */
  55         IMX53_ECSPI,    /* ECSPI on i.mx53 and later */
  56 };
  57
  58 struct spi_imx_data;
  59
  60 struct spi_imx_devtype_data {
  61         void (*intctrl)(struct spi_imx_data *, int);
  62         int (*config)(struct spi_device *);
  63         void (*trigger)(struct spi_imx_data *);
  64         int (*rx_available)(struct spi_imx_data *);
  65         void (*reset)(struct spi_imx_data *);
  66         void (*disable)(struct spi_imx_data *);
  67         bool has_dmamode;
  68         bool has_slavemode;
  69         unsigned int fifo_size;
  70         bool dynamic_burst;
  71         enum spi_imx_devtype devtype;
  72 };
  73
  74 struct spi_imx_data {
  75         struct spi_bitbang bitbang;
  76         struct device *dev;
  77
  78         struct completion xfer_done;
  79         void __iomem *base;
  80         unsigned long base_phys;
  81
  82         struct clk *clk_per;
  83         struct clk *clk_ipg;
  84         unsigned long spi_clk;
  85         unsigned int spi_bus_clk;
  86
  87         unsigned int speed_hz;
  88         unsigned int bits_per_word;
  89         unsigned int spi_drctl;
  90
  91         unsigned int count, remainder;
  92         void (*tx)(struct spi_imx_data *);
  93         void (*rx)(struct spi_imx_data *);
  94         void *rx_buf;
  95         const void *tx_buf;
  96         unsigned int txfifo; /* number of words pushed in tx FIFO */
  97         unsigned int dynamic_burst;
  98
  99         /* Slave mode */
 100         bool slave_mode;
 101         bool slave_aborted;
 102         unsigned int slave_burst;
 103
 104         /* DMA */
 105         bool usedma;
 106         u32 wml;
 107         struct completion dma_rx_completion;
 108         struct completion dma_tx_completion;
 109
 110         const struct spi_imx_devtype_data *devtype_data;
 111 };
 112
 113 static inline int is_imx27_cspi(struct spi_imx_data *d)
 114 {
 115         return d->devtype_data->devtype == IMX27_CSPI;
 116 }
 117
 118 static inline int is_imx35_cspi(struct spi_imx_data *d)
 119 {
 120         return d->devtype_data->devtype == IMX35_CSPI;
 121 }
 122
 123 static inline int is_imx51_ecspi(struct spi_imx_data *d)
 124 {
 125         return d->devtype_data->devtype == IMX51_ECSPI;
 126 }
 127
 128 static inline int is_imx53_ecspi(struct spi_imx_data *d)
 129 {
 130         return d->devtype_data->devtype == IMX53_ECSPI;
 131 }
 132
 133 #define MXC_SPI_BUF_RX(type)                                            \
 134 static void spi_imx_buf_rx_##type(struct spi_imx_data *spi_imx)         \
 135 {                                                                       \
 136         unsigned int val = readl(spi_imx->base + MXC_CSPIRXDATA);       \
 137                                                                         \
 138         if (spi_imx->rx_buf) {                                          \
 139                 *(type *)spi_imx->rx_buf = val;                         \
 140                 spi_imx->rx_buf += sizeof(type);                        \
 141         }                                                               \
 142                                                                         \
 143         spi_imx->remainder -= sizeof(type);                             \
 144 }
 145
 146 #define MXC_SPI_BUF_TX(type)                                            \
 147 static void spi_imx_buf_tx_##type(struct spi_imx_data *spi_imx)         \
 148 {                                                                       \
 149         type val = 0;                                                   \
 150                                                                         \
 151         if (spi_imx->tx_buf) {                                          \
 152                 val = *(type *)spi_imx->tx_buf;                         \
 153                 spi_imx->tx_buf += sizeof(type);                        \
 154         }                                                               \
 155                                                                         \
 156         spi_imx->count -= sizeof(type);                                 \
 157                                                                         \
 158         writel(val, spi_imx->base + MXC_CSPITXDATA);                    \
 159 }
 160
 161 MXC_SPI_BUF_RX(u8)
 162 MXC_SPI_BUF_TX(u8)
 163 MXC_SPI_BUF_RX(u16)
 164 MXC_SPI_BUF_TX(u16)
 165 MXC_SPI_BUF_RX(u32)
 166 MXC_SPI_BUF_TX(u32)
 167
 168 /* First entry is reserved, second entry is valid only if SDHC_SPIEN is set
 169  * (which is currently not the case in this driver)
 170  */
 171 static int mxc_clkdivs[] = {0, 3, 4, 6, 8, 12, 16, 24, 32, 48, 64, 96, 128, 192,
 172         256, 384, 512, 768, 1024};
 173
 174 /* MX21, MX27 */
 175 static unsigned int spi_imx_clkdiv_1(unsigned int fin,
 176                 unsigned int fspi, unsigned int max, unsigned int *fres)
 177 {
 178         int i;
 179
 180         for (i = 2; i < max; i++)
 181                 if (fspi * mxc_clkdivs[i] >= fin)
 182                         break;
 183
 184         *fres = fin / mxc_clkdivs[i];
 185         return i;
 186 }
 187
 188 /* MX1, MX31, MX35, MX51 CSPI */
 189 static unsigned int spi_imx_clkdiv_2(unsigned int fin,
 190                 unsigned int fspi, unsigned int *fres)
 191 {
 192         int i, div = 4;
 193
 194         for (i = 0; i < 7; i++) {
 195                 if (fspi * div >= fin)
 196                         goto out;
 197                 div <<= 1;
 198         }
 199
 200 out:
 201         *fres = fin / div;
 202         return i;
 203 }
 204
 205 static int spi_imx_bytes_per_word(const int bits_per_word)
 206 {
 207         if (bits_per_word <= 8)
 208                 return 1;
 209         else if (bits_per_word <= 16)
 210                 return 2;
 211         else
 212                 return 4;
 213 }
 214
 215 static bool spi_imx_can_dma(struct spi_master *master, struct spi_device *spi,
 216                          struct spi_transfer *transfer)
 217 {
 218         struct spi_imx_data *spi_imx = spi_master_get_devdata(master);
 219         unsigned int bytes_per_word, i;
 220
 221         if (!master->dma_rx)
 222                 return false;
 223
 224         if (spi_imx->slave_mode)
 225                 return false;
 226
 227         bytes_per_word = spi_imx_bytes_per_word(transfer->bits_per_word);
 228
 229         for (i = spi_imx->devtype_data->fifo_size / 2; i > 0; i--) {
 230                 if (!(transfer->len % (i * bytes_per_word)))
 231                         break;
 232         }
 233
 234         spi_imx->wml = i;
 235         spi_imx->dynamic_burst = 0;
 236
 237         return true;
 238 }
 239
 240 /*
 241  * Note the number of natively supported chip selects for MX51 is 4. Some
 242  * devices may have less actual SS pins but the register map supports 4. When
 243  * using gpio chip selects the cs values passed into the macros below can go
 244  * outside the range 0 - 3. We therefore need to limit the cs value to avoid
 245  * corrupting bits outside the allocated locations.
 246  *
 247  * The simplest way to do this is to just mask the cs bits to 2 bits. This
 248  * still allows all 4 native chip selects to work as well as gpio chip selects
 249  * (which can use any of the 4 chip select configurations).
 250  */
 251
 252 #define MX51_ECSPI_CTRL         0x08
 253 #define MX51_ECSPI_CTRL_ENABLE          (1 <<  0)
 254 #define MX51_ECSPI_CTRL_XCH             (1 <<  2)
 255 #define MX51_ECSPI_CTRL_SMC             (1 << 3)
 256 #define MX51_ECSPI_CTRL_MODE_MASK       (0xf << 4)
 257 #define MX51_ECSPI_CTRL_DRCTL(drctl)    ((drctl) << 16)
 258 #define MX51_ECSPI_CTRL_POSTDIV_OFFSET  8
 259 #define MX51_ECSPI_CTRL_PREDIV_OFFSET   12
 260 #define MX51_ECSPI_CTRL_CS(cs)          ((cs & 3) << 18)
 261 #define MX51_ECSPI_CTRL_BL_OFFSET       20
 262 #define MX51_ECSPI_CTRL_BL_MASK         (0xfff << 20)
 263
 264 #define MX51_ECSPI_CONFIG       0x0c
 265 #define MX51_ECSPI_CONFIG_SCLKPHA(cs)   (1 << ((cs & 3) +  0))
 266 #define MX51_ECSPI_CONFIG_SCLKPOL(cs)   (1 << ((cs & 3) +  4))
 267 #define MX51_ECSPI_CONFIG_SBBCTRL(cs)   (1 << ((cs & 3) +  8))
 268 #define MX51_ECSPI_CONFIG_SSBPOL(cs)    (1 << ((cs & 3) + 12))
 269 #define MX51_ECSPI_CONFIG_SCLKCTL(cs)   (1 << ((cs & 3) + 20))
 270
 271 #define MX51_ECSPI_INT          0x10
 272 #define MX51_ECSPI_INT_TEEN             (1 <<  0)
 273 #define MX51_ECSPI_INT_RREN             (1 <<  3)
 274 #define MX51_ECSPI_INT_RDREN            (1 <<  4)
 275
 276 #define MX51_ECSPI_DMA      0x14
 277 #define MX51_ECSPI_DMA_TX_WML(wml)      ((wml) & 0x3f)
 278 #define MX51_ECSPI_DMA_RX_WML(wml)      (((wml) & 0x3f) << 16)
 279 #define MX51_ECSPI_DMA_RXT_WML(wml)     (((wml) & 0x3f) << 24)
 280
 281 #define MX51_ECSPI_DMA_TEDEN            (1 << 7)
 282 #define MX51_ECSPI_DMA_RXDEN            (1 << 23)
 283 #define MX51_ECSPI_DMA_RXTDEN           (1 << 31)
 284
 285 #define MX51_ECSPI_STAT         0x18
 286 #define MX51_ECSPI_STAT_RR              (1 <<  3)
 287
 288 #define MX51_ECSPI_TESTREG      0x20
 289 #define MX51_ECSPI_TESTREG_LBC  BIT(31)
 290
 291 static void spi_imx_buf_rx_swap_u32(struct spi_imx_data *spi_imx)
 292 {
 293         unsigned int val = readl(spi_imx->base + MXC_CSPIRXDATA);
 294 #ifdef __LITTLE_ENDIAN
 295         unsigned int bytes_per_word;
 296 #endif
 297
 298         if (spi_imx->rx_buf) {
 299 #ifdef __LITTLE_ENDIAN
 300                 bytes_per_word = spi_imx_bytes_per_word(spi_imx->bits_per_word);
 301                 if (bytes_per_word == 1)
 302                         val = cpu_to_be32(val);
 303                 else if (bytes_per_word == 2)
 304                         val = (val << 16) | (val >> 16);
 305 #endif
 306                 *(u32 *)spi_imx->rx_buf = val;
 307                 spi_imx->rx_buf += sizeof(u32);
 308         }
 309
 310         spi_imx->remainder -= sizeof(u32);
 311 }
 312
 313 static void spi_imx_buf_rx_swap(struct spi_imx_data *spi_imx)
 314 {
 315         int unaligned;
 316         u32 val;
 317
 318         unaligned = spi_imx->remainder % 4;
 319
 320         if (!unaligned) {
 321                 spi_imx_buf_rx_swap_u32(spi_imx);
 322                 return;
 323         }
 324
 325         if (spi_imx_bytes_per_word(spi_imx->bits_per_word) == 2) {
 326                 spi_imx_buf_rx_u16(spi_imx);
 327                 return;
 328         }
 329
 330         val = readl(spi_imx->base + MXC_CSPIRXDATA);
 331
 332         while (unaligned--) {
 333                 if (spi_imx->rx_buf) {
 334                         *(u8 *)spi_imx->rx_buf = (val >> (8 * unaligned)) & 0xff;
 335                         spi_imx->rx_buf++;
 336                 }
 337                 spi_imx->remainder--;
 338         }
 339 }
 340
 341 static void spi_imx_buf_tx_swap_u32(struct spi_imx_data *spi_imx)
 342 {
 343         u32 val = 0;
 344 #ifdef __LITTLE_ENDIAN
 345         unsigned int bytes_per_word;
 346 #endif
 347
 348         if (spi_imx->tx_buf) {
 349                 val = *(u32 *)spi_imx->tx_buf;
 350                 spi_imx->tx_buf += sizeof(u32);
 351         }
 352
 353         spi_imx->count -= sizeof(u32);
 354 #ifdef __LITTLE_ENDIAN
 355         bytes_per_word = spi_imx_bytes_per_word(spi_imx->bits_per_word);
 356
 357         if (bytes_per_word == 1)
 358                 val = cpu_to_be32(val);
 359         else if (bytes_per_word == 2)
 360                 val = (val << 16) | (val >> 16);
 361 #endif
 362         writel(val, spi_imx->base + MXC_CSPITXDATA);
 363 }
 364
 365 static void spi_imx_buf_tx_swap(struct spi_imx_data *spi_imx)
 366 {
 367         int unaligned;
 368         u32 val = 0;
 369
 370         unaligned = spi_imx->count % 4;
 371
 372         if (!unaligned) {
 373                 spi_imx_buf_tx_swap_u32(spi_imx);
 374                 return;
 375         }
 376
 377         if (spi_imx_bytes_per_word(spi_imx->bits_per_word) == 2) {
 378                 spi_imx_buf_tx_u16(spi_imx);
 379                 return;
 380         }
 381
 382         while (unaligned--) {
 383                 if (spi_imx->tx_buf) {
 384                         val |= *(u8 *)spi_imx->tx_buf << (8 * unaligned);
 385                         spi_imx->tx_buf++;
 386                 }
 387                 spi_imx->count--;
 388         }
 389
 390         writel(val, spi_imx->base + MXC_CSPITXDATA);
 391 }
 392
 393 static void mx53_ecspi_rx_slave(struct spi_imx_data *spi_imx)
 394 {
 395         u32 val = be32_to_cpu(readl(spi_imx->base + MXC_CSPIRXDATA));
 396
 397         if (spi_imx->rx_buf) {
 398                 int n_bytes = spi_imx->slave_burst % sizeof(val);
 399
 400                 if (!n_bytes)
 401                         n_bytes = sizeof(val);
 402
 403                 memcpy(spi_imx->rx_buf,
 404                        ((u8 *)&val) + sizeof(val) - n_bytes, n_bytes);
 405
 406                 spi_imx->rx_buf += n_bytes;
 407                 spi_imx->slave_burst -= n_bytes;
 408         }
 409
 410         spi_imx->remainder -= sizeof(u32);
 411 }
 412
 413 static void mx53_ecspi_tx_slave(struct spi_imx_data *spi_imx)
 414 {
 415         u32 val = 0;
 416         int n_bytes = spi_imx->count % sizeof(val);
 417
 418         if (!n_bytes)
 419                 n_bytes = sizeof(val);
 420
 421         if (spi_imx->tx_buf) {
 422                 memcpy(((u8 *)&val) + sizeof(val) - n_bytes,
 423                        spi_imx->tx_buf, n_bytes);
 424                 val = cpu_to_be32(val);
 425                 spi_imx->tx_buf += n_bytes;
 426         }
 427
 428         spi_imx->count -= n_bytes;
 429
 430         writel(val, spi_imx->base + MXC_CSPITXDATA);
 431 }
 432
 433 /* MX51 eCSPI */
 434 static unsigned int mx51_ecspi_clkdiv(struct spi_imx_data *spi_imx,
 435                                       unsigned int fspi, unsigned int *fres)
 436 {
 437         /*
 438          * there are two 4-bit dividers, the pre-divider divides by
 439          * $pre, the post-divider by 2^$post
 440          */
 441         unsigned int pre, post;
 442         unsigned int fin = spi_imx->spi_clk;
 443
 444         fspi = min(fspi, fin);
 445
 446         post = fls(fin) - fls(fspi);
 447         if (fin > fspi << post)
 448                 post++;
 449
 450         /* now we have: (fin <= fspi << post) with post being minimal */
 451
 452         post = max(4U, post) - 4;
 453         if (unlikely(post > 0xf)) {
 454                 dev_err(spi_imx->dev, "cannot set clock freq: %u (base freq: %u)\n",
 455                                 fspi, fin);
 456                 return 0xff;
 457         }
 458
 459         pre = DIV_ROUND_UP(fin, fspi << post) - 1;
 460
 461         dev_dbg(spi_imx->dev, "%s: fin: %u, fspi: %u, post: %u, pre: %u\n",
 462                         __func__, fin, fspi, post, pre);
 463
 464         /* Resulting frequency for the SCLK line. */
 465         *fres = (fin / (pre + 1)) >> post;
 466
 467         return (pre << MX51_ECSPI_CTRL_PREDIV_OFFSET) |
 468                 (post << MX51_ECSPI_CTRL_POSTDIV_OFFSET);
 469 }
 470
 471 static void mx51_ecspi_intctrl(struct spi_imx_data *spi_imx, int enable)
 472 {
 473         unsigned val = 0;
 474
 475         if (enable & MXC_INT_TE)
 476                 val |= MX51_ECSPI_INT_TEEN;
 477
 478         if (enable & MXC_INT_RR)
 479                 val |= MX51_ECSPI_INT_RREN;
 480
 481         if (enable & MXC_INT_RDR)
 482                 val |= MX51_ECSPI_INT_RDREN;
 483
 484         writel(val, spi_imx->base + MX51_ECSPI_INT);
 485 }
 486
 487 static void mx51_ecspi_trigger(struct spi_imx_data *spi_imx)
 488 {
 489         u32 reg;
 490
 491         reg = readl(spi_imx->base + MX51_ECSPI_CTRL);
 492         reg |= MX51_ECSPI_CTRL_XCH;
 493         writel(reg, spi_imx->base + MX51_ECSPI_CTRL);
 494 }
 495
 496 static void mx51_ecspi_disable(struct spi_imx_data *spi_imx)
 497 {
 498         u32 ctrl;
 499
 500         ctrl = readl(spi_imx->base + MX51_ECSPI_CTRL);
 501         ctrl &= ~MX51_ECSPI_CTRL_ENABLE;
 502         writel(ctrl, spi_imx->base + MX51_ECSPI_CTRL);
 503 }
 504
 505 static int mx51_ecspi_config(struct spi_device *spi)
 506 {
 507         struct spi_imx_data *spi_imx = spi_master_get_devdata(spi->master);
 508         u32 ctrl = MX51_ECSPI_CTRL_ENABLE;
 509         u32 clk = spi_imx->speed_hz, delay, reg;
 510         u32 cfg = readl(spi_imx->base + MX51_ECSPI_CONFIG);
 511
 512         /* set Master or Slave mode */
 513         if (spi_imx->slave_mode)
 514                 ctrl &= ~MX51_ECSPI_CTRL_MODE_MASK;
 515         else
 516                 ctrl |= MX51_ECSPI_CTRL_MODE_MASK;
 517
 518         /*
 519          * Enable SPI_RDY handling (falling edge/level triggered).
 520          */
 521         if (spi->mode & SPI_READY)
 522                 ctrl |= MX51_ECSPI_CTRL_DRCTL(spi_imx->spi_drctl);
 523
 524         /* set clock speed */
 525         ctrl |= mx51_ecspi_clkdiv(spi_imx, spi_imx->speed_hz, &clk);
 526         spi_imx->spi_bus_clk = clk;
 527
 528         /* set chip select to use */
 529         ctrl |= MX51_ECSPI_CTRL_CS(spi->chip_select);
 530
 531         if (spi_imx->slave_mode && is_imx53_ecspi(spi_imx))
 532                 ctrl |= (spi_imx->slave_burst * 8 - 1)
 533                         << MX51_ECSPI_CTRL_BL_OFFSET;
 534         else
 535                 ctrl |= (spi_imx->bits_per_word - 1)
 536                         << MX51_ECSPI_CTRL_BL_OFFSET;
 537
 538         /*
 539          * eCSPI burst completion by Chip Select signal in Slave mode
 540          * is not functional for imx53 Soc, config SPI burst completed when
 541          * BURST_LENGTH + 1 bits are received
 542          */
 543         if (spi_imx->slave_mode && is_imx53_ecspi(spi_imx))
 544                 cfg &= ~MX51_ECSPI_CONFIG_SBBCTRL(spi->chip_select);
 545         else
 546                 cfg |= MX51_ECSPI_CONFIG_SBBCTRL(spi->chip_select);
 547
 548         if (spi->mode & SPI_CPHA)
 549                 cfg |= MX51_ECSPI_CONFIG_SCLKPHA(spi->chip_select);
 550         else
 551                 cfg &= ~MX51_ECSPI_CONFIG_SCLKPHA(spi->chip_select);
 552
 553         if (spi->mode & SPI_CPOL) {
 554                 cfg |= MX51_ECSPI_CONFIG_SCLKPOL(spi->chip_select);
 555                 cfg |= MX51_ECSPI_CONFIG_SCLKCTL(spi->chip_select);
 556         } else {
 557                 cfg &= ~MX51_ECSPI_CONFIG_SCLKPOL(spi->chip_select);
 558                 cfg &= ~MX51_ECSPI_CONFIG_SCLKCTL(spi->chip_select);
 559         }
 560         if (spi->mode & SPI_CS_HIGH)
 561                 cfg |= MX51_ECSPI_CONFIG_SSBPOL(spi->chip_select);
 562         else
 563                 cfg &= ~MX51_ECSPI_CONFIG_SSBPOL(spi->chip_select);
 564
 565         if (spi_imx->usedma)
 566                 ctrl |= MX51_ECSPI_CTRL_SMC;
 567
 568         /* CTRL register always go first to bring out controller from reset */
 569         writel(ctrl, spi_imx->base + MX51_ECSPI_CTRL);
 570
 571         reg = readl(spi_imx->base + MX51_ECSPI_TESTREG);
 572         if (spi->mode & SPI_LOOP)
 573                 reg |= MX51_ECSPI_TESTREG_LBC;
 574         else
 575                 reg &= ~MX51_ECSPI_TESTREG_LBC;
 576         writel(reg, spi_imx->base + MX51_ECSPI_TESTREG);
 577
 578         writel(cfg, spi_imx->base + MX51_ECSPI_CONFIG);
 579
 580         /*
 581          * Wait until the changes in the configuration register CONFIGREG
 582          * propagate into the hardware. It takes exactly one tick of the
 583          * SCLK clock, but we will wait two SCLK clock just to be sure. The
 584          * effect of the delay it takes for the hardware to apply changes
 585          * is noticable if the SCLK clock run very slow. In such a case, if
 586          * the polarity of SCLK should be inverted, the GPIO ChipSelect might
 587          * be asserted before the SCLK polarity changes, which would disrupt
 588          * the SPI communication as the device on the other end would consider
 589          * the change of SCLK polarity as a clock tick already.
 590          */
 591         delay = (2 * 1000000) / clk;
 592         if (likely(delay < 10)) /* SCLK is faster than 100 kHz */
 593                 udelay(delay);
 594         else                    /* SCLK is _very_ slow */
 595                 usleep_range(delay, delay + 10);
 596
 597         /*
 598          * Configure the DMA register: setup the watermark
 599          * and enable DMA request.
 600          */
 601
 602         writel(MX51_ECSPI_DMA_RX_WML(spi_imx->wml) |
 603                 MX51_ECSPI_DMA_TX_WML(spi_imx->wml) |
 604                 MX51_ECSPI_DMA_RXT_WML(spi_imx->wml) |
 605                 MX51_ECSPI_DMA_TEDEN | MX51_ECSPI_DMA_RXDEN |
 606                 MX51_ECSPI_DMA_RXTDEN, spi_imx->base + MX51_ECSPI_DMA);
 607
 608         return 0;
 609 }
 610
 611 static int mx51_ecspi_rx_available(struct spi_imx_data *spi_imx)
 612 {
 613         return readl(spi_imx->base + MX51_ECSPI_STAT) & MX51_ECSPI_STAT_RR;
 614 }
 615
 616 static void mx51_ecspi_reset(struct spi_imx_data *spi_imx)
 617 {
 618         /* drain receive buffer */
 619         while (mx51_ecspi_rx_available(spi_imx))
 620                 readl(spi_imx->base + MXC_CSPIRXDATA);
 621 }
 622
 623 #define MX31_INTREG_TEEN        (1 << 0)
 624 #define MX31_INTREG_RREN        (1 << 3)
 625
 626 #define MX31_CSPICTRL_ENABLE    (1 << 0)
 627 #define MX31_CSPICTRL_MASTER    (1 << 1)
 628 #define MX31_CSPICTRL_XCH       (1 << 2)
 629 #define MX31_CSPICTRL_SMC       (1 << 3)
 630 #define MX31_CSPICTRL_POL       (1 << 4)
 631 #define MX31_CSPICTRL_PHA       (1 << 5)
 632 #define MX31_CSPICTRL_SSCTL     (1 << 6)
 633 #define MX31_CSPICTRL_SSPOL     (1 << 7)
 634 #define MX31_CSPICTRL_BC_SHIFT  8
 635 #define MX35_CSPICTRL_BL_SHIFT  20
 636 #define MX31_CSPICTRL_CS_SHIFT  24
 637 #define MX35_CSPICTRL_CS_SHIFT  12
 638 #define MX31_CSPICTRL_DR_SHIFT  16
 639
 640 #define MX31_CSPI_DMAREG        0x10
 641 #define MX31_DMAREG_RH_DEN      (1<<4)
 642 #define MX31_DMAREG_TH_DEN      (1<<1)
 643
 644 #define MX31_CSPISTATUS         0x14
 645 #define MX31_STATUS_RR          (1 << 3)
 646
 647 #define MX31_CSPI_TESTREG       0x1C
 648 #define MX31_TEST_LBC           (1 << 14)
 649
 650 /* These functions also work for the i.MX35, but be aware that
 651  * the i.MX35 has a slightly different register layout for bits
 652  * we do not use here.
 653  */
 654 static void mx31_intctrl(struct spi_imx_data *spi_imx, int enable)
 655 {
 656         unsigned int val = 0;
 657
 658         if (enable & MXC_INT_TE)
 659                 val |= MX31_INTREG_TEEN;
 660         if (enable & MXC_INT_RR)
 661                 val |= MX31_INTREG_RREN;
 662
 663         writel(val, spi_imx->base + MXC_CSPIINT);
 664 }
 665
 666 static void mx31_trigger(struct spi_imx_data *spi_imx)
 667 {
 668         unsigned int reg;
 669
 670         reg = readl(spi_imx->base + MXC_CSPICTRL);
 671         reg |= MX31_CSPICTRL_XCH;
 672         writel(reg, spi_imx->base + MXC_CSPICTRL);
 673 }
 674
 675 static int mx31_config(struct spi_device *spi)
 676 {
 677         struct spi_imx_data *spi_imx = spi_master_get_devdata(spi->master);
 678         unsigned int reg = MX31_CSPICTRL_ENABLE | MX31_CSPICTRL_MASTER;
 679         unsigned int clk;
 680
 681         reg |= spi_imx_clkdiv_2(spi_imx->spi_clk, spi_imx->speed_hz, &clk) <<
 682                 MX31_CSPICTRL_DR_SHIFT;
 683         spi_imx->spi_bus_clk = clk;
 684
 685         if (is_imx35_cspi(spi_imx)) {
 686                 reg |= (spi_imx->bits_per_word - 1) << MX35_CSPICTRL_BL_SHIFT;
 687                 reg |= MX31_CSPICTRL_SSCTL;
 688         } else {
 689                 reg |= (spi_imx->bits_per_word - 1) << MX31_CSPICTRL_BC_SHIFT;
 690         }
 691
 692         if (spi->mode & SPI_CPHA)
 693                 reg |= MX31_CSPICTRL_PHA;
 694         if (spi->mode & SPI_CPOL)
 695                 reg |= MX31_CSPICTRL_POL;
 696         if (spi->mode & SPI_CS_HIGH)
 697                 reg |= MX31_CSPICTRL_SSPOL;
 698         if (!gpio_is_valid(spi->cs_gpio))
 699                 reg |= (spi->chip_select) <<
 700                         (is_imx35_cspi(spi_imx) ? MX35_CSPICTRL_CS_SHIFT :
 701                                                   MX31_CSPICTRL_CS_SHIFT);
 702
 703         if (spi_imx->usedma)
 704                 reg |= MX31_CSPICTRL_SMC;
 705
 706         writel(reg, spi_imx->base + MXC_CSPICTRL);
 707
 708         reg = readl(spi_imx->base + MX31_CSPI_TESTREG);
 709         if (spi->mode & SPI_LOOP)
 710                 reg |= MX31_TEST_LBC;
 711         else
 712                 reg &= ~MX31_TEST_LBC;
 713         writel(reg, spi_imx->base + MX31_CSPI_TESTREG);
 714
 715         if (spi_imx->usedma) {
 716                 /* configure DMA requests when RXFIFO is half full and
 717                    when TXFIFO is half empty */
 718                 writel(MX31_DMAREG_RH_DEN | MX31_DMAREG_TH_DEN,
 719                         spi_imx->base + MX31_CSPI_DMAREG);
 720         }
 721
 722         return 0;
 723 }
 724
 725 static int mx31_rx_available(struct spi_imx_data *spi_imx)
 726 {
 727         return readl(spi_imx->base + MX31_CSPISTATUS) & MX31_STATUS_RR;
 728 }
 729
 730 static void mx31_reset(struct spi_imx_data *spi_imx)
 731 {
 732         /* drain receive buffer */
 733         while (readl(spi_imx->base + MX31_CSPISTATUS) & MX31_STATUS_RR)
 734                 readl(spi_imx->base + MXC_CSPIRXDATA);
 735 }
 736
 737 #define MX21_INTREG_RR          (1 << 4)
 738 #define MX21_INTREG_TEEN        (1 << 9)
 739 #define MX21_INTREG_RREN        (1 << 13)
 740
 741 #define MX21_CSPICTRL_POL       (1 << 5)
 742 #define MX21_CSPICTRL_PHA       (1 << 6)
 743 #define MX21_CSPICTRL_SSPOL     (1 << 8)
 744 #define MX21_CSPICTRL_XCH       (1 << 9)
 745 #define MX21_CSPICTRL_ENABLE    (1 << 10)
 746 #define MX21_CSPICTRL_MASTER    (1 << 11)
 747 #define MX21_CSPICTRL_DR_SHIFT  14
 748 #define MX21_CSPICTRL_CS_SHIFT  19
 749
 750 static void mx21_intctrl(struct spi_imx_data *spi_imx, int enable)
 751 {
 752         unsigned int val = 0;
 753
 754         if (enable & MXC_INT_TE)
 755                 val |= MX21_INTREG_TEEN;
 756         if (enable & MXC_INT_RR)
 757                 val |= MX21_INTREG_RREN;
 758
 759         writel(val, spi_imx->base + MXC_CSPIINT);
 760 }
 761
 762 static void mx21_trigger(struct spi_imx_data *spi_imx)
 763 {
 764         unsigned int reg;
 765
 766         reg = readl(spi_imx->base + MXC_CSPICTRL);
 767         reg |= MX21_CSPICTRL_XCH;
 768         writel(reg, spi_imx->base + MXC_CSPICTRL);
 769 }
 770
 771 static int mx21_config(struct spi_device *spi)
 772 {
 773         struct spi_imx_data *spi_imx = spi_master_get_devdata(spi->master);
 774         unsigned int reg = MX21_CSPICTRL_ENABLE | MX21_CSPICTRL_MASTER;
 775         unsigned int max = is_imx27_cspi(spi_imx) ? 16 : 18;
 776         unsigned int clk;
 777
 778         reg |= spi_imx_clkdiv_1(spi_imx->spi_clk, spi_imx->speed_hz, max, &clk)
 779                 << MX21_CSPICTRL_DR_SHIFT;
 780         spi_imx->spi_bus_clk = clk;
 781
 782         reg |= spi_imx->bits_per_word - 1;
 783
 784         if (spi->mode & SPI_CPHA)
 785                 reg |= MX21_CSPICTRL_PHA;
 786         if (spi->mode & SPI_CPOL)
 787                 reg |= MX21_CSPICTRL_POL;
 788         if (spi->mode & SPI_CS_HIGH)
 789                 reg |= MX21_CSPICTRL_SSPOL;
 790         if (!gpio_is_valid(spi->cs_gpio))
 791                 reg |= spi->chip_select << MX21_CSPICTRL_CS_SHIFT;
 792
 793         writel(reg, spi_imx->base + MXC_CSPICTRL);
 794
 795         return 0;
 796 }
 797
 798 static int mx21_rx_available(struct spi_imx_data *spi_imx)
 799 {
 800         return readl(spi_imx->base + MXC_CSPIINT) & MX21_INTREG_RR;
 801 }
 802
 803 static void mx21_reset(struct spi_imx_data *spi_imx)
 804 {
 805         writel(1, spi_imx->base + MXC_RESET);
 806 }
 807
 808 #define MX1_INTREG_RR           (1 << 3)
 809 #define MX1_INTREG_TEEN         (1 << 8)
 810 #define MX1_INTREG_RREN         (1 << 11)
 811
 812 #define MX1_CSPICTRL_POL        (1 << 4)
 813 #define MX1_CSPICTRL_PHA        (1 << 5)
 814 #define MX1_CSPICTRL_XCH        (1 << 8)
 815 #define MX1_CSPICTRL_ENABLE     (1 << 9)
 816 #define MX1_CSPICTRL_MASTER     (1 << 10)
 817 #define MX1_CSPICTRL_DR_SHIFT   13
 818
 819 static void mx1_intctrl(struct spi_imx_data *spi_imx, int enable)
 820 {
 821         unsigned int val = 0;
 822
 823         if (enable & MXC_INT_TE)
 824                 val |= MX1_INTREG_TEEN;
 825         if (enable & MXC_INT_RR)
 826                 val |= MX1_INTREG_RREN;
 827
 828         writel(val, spi_imx->base + MXC_CSPIINT);
 829 }
 830
 831 static void mx1_trigger(struct spi_imx_data *spi_imx)
 832 {
 833         unsigned int reg;
 834
 835         reg = readl(spi_imx->base + MXC_CSPICTRL);
 836         reg |= MX1_CSPICTRL_XCH;
 837         writel(reg, spi_imx->base + MXC_CSPICTRL);
 838 }
 839
 840 static int mx1_config(struct spi_device *spi)
 841 {
 842         struct spi_imx_data *spi_imx = spi_master_get_devdata(spi->master);
 843         unsigned int reg = MX1_CSPICTRL_ENABLE | MX1_CSPICTRL_MASTER;
 844         unsigned int clk;
 845
 846         reg |= spi_imx_clkdiv_2(spi_imx->spi_clk, spi_imx->speed_hz, &clk) <<
 847                 MX1_CSPICTRL_DR_SHIFT;
 848         spi_imx->spi_bus_clk = clk;
 849
 850         reg |= spi_imx->bits_per_word - 1;
 851
 852         if (spi->mode & SPI_CPHA)
 853                 reg |= MX1_CSPICTRL_PHA;
 854         if (spi->mode & SPI_CPOL)
 855                 reg |= MX1_CSPICTRL_POL;
 856
 857         writel(reg, spi_imx->base + MXC_CSPICTRL);
 858
 859         return 0;
 860 }
 861
 862 static int mx1_rx_available(struct spi_imx_data *spi_imx)
 863 {
 864         return readl(spi_imx->base + MXC_CSPIINT) & MX1_INTREG_RR;
 865 }
 866
 867 static void mx1_reset(struct spi_imx_data *spi_imx)
 868 {
 869         writel(1, spi_imx->base + MXC_RESET);
 870 }
 871
 872 static struct spi_imx_devtype_data imx1_cspi_devtype_data = {
 873         .intctrl = mx1_intctrl,
 874         .config = mx1_config,
 875         .trigger = mx1_trigger,
 876         .rx_available = mx1_rx_available,
 877         .reset = mx1_reset,
 878         .fifo_size = 8,
 879         .has_dmamode = false,
 880         .dynamic_burst = false,
 881         .has_slavemode = false,
 882         .devtype = IMX1_CSPI,
 883 };
 884
 885 static struct spi_imx_devtype_data imx21_cspi_devtype_data = {
 886         .intctrl = mx21_intctrl,
 887         .config = mx21_config,
 888         .trigger = mx21_trigger,
 889         .rx_available = mx21_rx_available,
 890         .reset = mx21_reset,
 891         .fifo_size = 8,
 892         .has_dmamode = false,
 893         .dynamic_burst = false,
 894         .has_slavemode = false,
 895         .devtype = IMX21_CSPI,
 896 };
 897
 898 static struct spi_imx_devtype_data imx27_cspi_devtype_data = {
 899         /* i.mx27 cspi shares the functions with i.mx21 one */
 900         .intctrl = mx21_intctrl,
 901         .config = mx21_config,
 902         .trigger = mx21_trigger,
 903         .rx_available = mx21_rx_available,
 904         .reset = mx21_reset,
 905         .fifo_size = 8,
 906         .has_dmamode = false,
 907         .dynamic_burst = false,
 908         .has_slavemode = false,
 909         .devtype = IMX27_CSPI,
 910 };
 911
 912 static struct spi_imx_devtype_data imx31_cspi_devtype_data = {
 913         .intctrl = mx31_intctrl,
 914         .config = mx31_config,
 915         .trigger = mx31_trigger,
 916         .rx_available = mx31_rx_available,
 917         .reset = mx31_reset,
 918         .fifo_size = 8,
 919         .has_dmamode = false,
 920         .dynamic_burst = false,
 921         .has_slavemode = false,
 922         .devtype = IMX31_CSPI,
 923 };
 924
 925 static struct spi_imx_devtype_data imx35_cspi_devtype_data = {
 926         /* i.mx35 and later cspi shares the functions with i.mx31 one */
 927         .intctrl = mx31_intctrl,
 928         .config = mx31_config,
 929         .trigger = mx31_trigger,
 930         .rx_available = mx31_rx_available,
 931         .reset = mx31_reset,
 932         .fifo_size = 8,
 933         .has_dmamode = true,
 934         .dynamic_burst = false,
 935         .has_slavemode = false,
 936         .devtype = IMX35_CSPI,
 937 };
 938
 939 static struct spi_imx_devtype_data imx51_ecspi_devtype_data = {
 940         .intctrl = mx51_ecspi_intctrl,
 941         .config = mx51_ecspi_config,
 942         .trigger = mx51_ecspi_trigger,
 943         .rx_available = mx51_ecspi_rx_available,
 944         .reset = mx51_ecspi_reset,
 945         .fifo_size = 64,
 946         .has_dmamode = true,
 947         .dynamic_burst = true,
 948         .has_slavemode = true,
 949         .disable = mx51_ecspi_disable,
 950         .devtype = IMX51_ECSPI,
 951 };
 952
 953 static struct spi_imx_devtype_data imx53_ecspi_devtype_data = {
 954         .intctrl = mx51_ecspi_intctrl,
 955         .config = mx51_ecspi_config,
 956         .trigger = mx51_ecspi_trigger,
 957         .rx_available = mx51_ecspi_rx_available,
 958         .reset = mx51_ecspi_reset,
 959         .fifo_size = 64,
 960         .has_dmamode = true,
 961         .has_slavemode = true,
 962         .disable = mx51_ecspi_disable,
 963         .devtype = IMX53_ECSPI,
 964 };
 965
 966 static const struct platform_device_id spi_imx_devtype[] = {
 967         {
 968                 .name = "imx1-cspi",
 969                 .driver_data = (kernel_ulong_t) &imx1_cspi_devtype_data,
 970         }, {
 971                 .name = "imx21-cspi",
 972                 .driver_data = (kernel_ulong_t) &imx21_cspi_devtype_data,
 973         }, {
 974                 .name = "imx27-cspi",
 975                 .driver_data = (kernel_ulong_t) &imx27_cspi_devtype_data,
 976         }, {
 977                 .name = "imx31-cspi",
 978                 .driver_data = (kernel_ulong_t) &imx31_cspi_devtype_data,
 979         }, {
 980                 .name = "imx35-cspi",
 981                 .driver_data = (kernel_ulong_t) &imx35_cspi_devtype_data,
 982         }, {
 983                 .name = "imx51-ecspi",
 984                 .driver_data = (kernel_ulong_t) &imx51_ecspi_devtype_data,
 985         }, {
 986                 .name = "imx53-ecspi",
 987                 .driver_data = (kernel_ulong_t) &imx53_ecspi_devtype_data,
 988         }, {
 989                 /* sentinel */
 990         }
 991 };
 992
 993 static const struct of_device_id spi_imx_dt_ids[] = {
 994         { .compatible = "fsl,imx1-cspi", .data = &imx1_cspi_devtype_data, },
 995         { .compatible = "fsl,imx21-cspi", .data = &imx21_cspi_devtype_data, },
 996         { .compatible = "fsl,imx27-cspi", .data = &imx27_cspi_devtype_data, },
 997         { .compatible = "fsl,imx31-cspi", .data = &imx31_cspi_devtype_data, },
 998         { .compatible = "fsl,imx35-cspi", .data = &imx35_cspi_devtype_data, },
 999         { .compatible = "fsl,imx51-ecspi", .data = &imx51_ecspi_devtype_data, },
1000         { .compatible = "fsl,imx53-ecspi", .data = &imx53_ecspi_devtype_data, },
1001         { /* sentinel */ }
1002 };
1003 MODULE_DEVICE_TABLE(of, spi_imx_dt_ids);
1004
1005 static void spi_imx_chipselect(struct spi_device *spi, int is_active)
1006 {
1007         int active = is_active != BITBANG_CS_INACTIVE;
1008         int dev_is_lowactive = !(spi->mode & SPI_CS_HIGH);
1009
1010         if (spi->mode & SPI_NO_CS)
1011                 return;
1012
1013         if (!gpio_is_valid(spi->cs_gpio))
1014                 return;
1015
1016         gpio_set_value(spi->cs_gpio, dev_is_lowactive ^ active);
1017 }
1018
1019 static void spi_imx_set_burst_len(struct spi_imx_data *spi_imx, int n_bits)
1020 {
1021         u32 ctrl;
1022
1023         ctrl = readl(spi_imx->base + MX51_ECSPI_CTRL);
1024         ctrl &= ~MX51_ECSPI_CTRL_BL_MASK;
1025         ctrl |= ((n_bits - 1) << MX51_ECSPI_CTRL_BL_OFFSET);
1026         writel(ctrl, spi_imx->base + MX51_ECSPI_CTRL);
1027 }
1028
1029 static void spi_imx_push(struct spi_imx_data *spi_imx)
1030 {
1031         unsigned int burst_len, fifo_words;
1032
1033         if (spi_imx->dynamic_burst)
1034                 fifo_words = 4;
1035         else
1036                 fifo_words = spi_imx_bytes_per_word(spi_imx->bits_per_word);
1037         /*
1038          * Reload the FIFO when the remaining bytes to be transferred in the
1039          * current burst is 0. This only applies when bits_per_word is a
1040          * multiple of 8.
1041          */
1042         if (!spi_imx->remainder) {
1043                 if (spi_imx->dynamic_burst) {
1044
1045                         /* We need to deal unaligned data first */
1046                         burst_len = spi_imx->count % MX51_ECSPI_CTRL_MAX_BURST;
1047
1048                         if (!burst_len)
1049                                 burst_len = MX51_ECSPI_CTRL_MAX_BURST;
1050
1051                         spi_imx_set_burst_len(spi_imx, burst_len * 8);
1052
1053                         spi_imx->remainder = burst_len;
1054                 } else {
1055                         spi_imx->remainder = fifo_words;
1056                 }
1057         }
1058
1059         while (spi_imx->txfifo < spi_imx->devtype_data->fifo_size) {
1060                 if (!spi_imx->count)
1061                         break;
1062                 if (spi_imx->dynamic_burst &&
1063                     spi_imx->txfifo >=  DIV_ROUND_UP(spi_imx->remainder,
1064                                                      fifo_words))
1065                         break;
1066                 spi_imx->tx(spi_imx);
1067                 spi_imx->txfifo++;
1068         }
1069
1070         if (!spi_imx->slave_mode)
1071                 spi_imx->devtype_data->trigger(spi_imx);
1072 }
1073
1074 static irqreturn_t spi_imx_isr(int irq, void *dev_id)
1075 {
1076         struct spi_imx_data *spi_imx = dev_id;
1077
1078         while (spi_imx->txfifo &&
1079                spi_imx->devtype_data->rx_available(spi_imx)) {
1080                 spi_imx->rx(spi_imx);
1081                 spi_imx->txfifo--;
1082         }
1083
1084         if (spi_imx->count) {
1085                 spi_imx_push(spi_imx);
1086                 return IRQ_HANDLED;
1087         }
1088
1089         if (spi_imx->txfifo) {
1090                 /* No data left to push, but still waiting for rx data,
1091                  * enable receive data available interrupt.
1092                  */
1093                 spi_imx->devtype_data->intctrl(
1094                                 spi_imx, MXC_INT_RR);
1095                 return IRQ_HANDLED;
1096         }
1097
1098         spi_imx->devtype_data->intctrl(spi_imx, 0);
1099         complete(&spi_imx->xfer_done);
1100
1101         return IRQ_HANDLED;
1102 }
1103
1104 static int spi_imx_dma_configure(struct spi_master *master)
1105 {
1106         int ret;
1107         enum dma_slave_buswidth buswidth;
1108         struct dma_slave_config rx = {}, tx = {};
1109         struct spi_imx_data *spi_imx = spi_master_get_devdata(master);
1110
1111         switch (spi_imx_bytes_per_word(spi_imx->bits_per_word)) {
1112         case 4:
1113                 buswidth = DMA_SLAVE_BUSWIDTH_4_BYTES;
1114                 break;
1115         case 2:
1116                 buswidth = DMA_SLAVE_BUSWIDTH_2_BYTES;
1117                 break;
1118         case 1:
1119                 buswidth = DMA_SLAVE_BUSWIDTH_1_BYTE;
1120                 break;
1121         default:
1122                 return -EINVAL;
1123         }
1124
1125         tx.direction = DMA_MEM_TO_DEV;
1126         tx.dst_addr = spi_imx->base_phys + MXC_CSPITXDATA;
1127         tx.dst_addr_width = buswidth;
1128         tx.dst_maxburst = spi_imx->wml;
1129         ret = dmaengine_slave_config(master->dma_tx, &tx);
1130         if (ret) {
1131                 dev_err(spi_imx->dev, "TX dma configuration failed with %d\n", ret);
1132                 return ret;
1133         }
1134
1135         rx.direction = DMA_DEV_TO_MEM;
1136         rx.src_addr = spi_imx->base_phys + MXC_CSPIRXDATA;
1137         rx.src_addr_width = buswidth;
1138         rx.src_maxburst = spi_imx->wml;
1139         ret = dmaengine_slave_config(master->dma_rx, &rx);
1140         if (ret) {
1141                 dev_err(spi_imx->dev, "RX dma configuration failed with %d\n", ret);
1142                 return ret;
1143         }
1144
1145         return 0;
1146 }
1147
1148 static int spi_imx_setupxfer(struct spi_device *spi,
1149                                  struct spi_transfer *t)
1150 {
1151         struct spi_imx_data *spi_imx = spi_master_get_devdata(spi->master);
1152         int ret;
1153
1154         if (!t)
1155                 return 0;
1156
1157         spi_imx->bits_per_word = t->bits_per_word;
1158         spi_imx->speed_hz  = t->speed_hz;
1159
1160         /*
1161          * Initialize the functions for transfer. To transfer non byte-aligned
1162          * words, we have to use multiple word-size bursts, we can't use
1163          * dynamic_burst in that case.
1164          */
1165         if (spi_imx->devtype_data->dynamic_burst && !spi_imx->slave_mode &&
1166             (spi_imx->bits_per_word == 8 ||
1167             spi_imx->bits_per_word == 16 ||
1168             spi_imx->bits_per_word == 32)) {
1169
1170                 spi_imx->rx = spi_imx_buf_rx_swap;
1171                 spi_imx->tx = spi_imx_buf_tx_swap;
1172                 spi_imx->dynamic_burst = 1;
1173
1174         } else {
1175                 if (spi_imx->bits_per_word <= 8) {
1176                         spi_imx->rx = spi_imx_buf_rx_u8;
1177                         spi_imx->tx = spi_imx_buf_tx_u8;
1178                 } else if (spi_imx->bits_per_word <= 16) {
1179                         spi_imx->rx = spi_imx_buf_rx_u16;
1180                         spi_imx->tx = spi_imx_buf_tx_u16;
1181                 } else {
1182                         spi_imx->rx = spi_imx_buf_rx_u32;
1183                         spi_imx->tx = spi_imx_buf_tx_u32;
1184                 }
1185                 spi_imx->dynamic_burst = 0;
1186         }
1187
1188         if (spi_imx_can_dma(spi_imx->bitbang.master, spi, t))
1189                 spi_imx->usedma = 1;
1190         else
1191                 spi_imx->usedma = 0;
1192
1193         if (spi_imx->usedma) {
1194                 ret = spi_imx_dma_configure(spi->master);
1195                 if (ret)
1196                         return ret;
1197         }
1198
1199         if (is_imx53_ecspi(spi_imx) && spi_imx->slave_mode) {
1200                 spi_imx->rx = mx53_ecspi_rx_slave;
1201                 spi_imx->tx = mx53_ecspi_tx_slave;
1202                 spi_imx->slave_burst = t->len;
1203         }
1204
1205         spi_imx->devtype_data->config(spi);
1206
1207         return 0;
1208 }
1209
1210 static void spi_imx_sdma_exit(struct spi_imx_data *spi_imx)
1211 {
1212         struct spi_master *master = spi_imx->bitbang.master;
1213
1214         if (master->dma_rx) {
1215                 dma_release_channel(master->dma_rx);
1216                 master->dma_rx = NULL;
1217         }
1218
1219         if (master->dma_tx) {
1220                 dma_release_channel(master->dma_tx);
1221                 master->dma_tx = NULL;
1222         }
1223 }
1224
1225 static int spi_imx_sdma_init(struct device *dev, struct spi_imx_data *spi_imx,
1226                              struct spi_master *master)
1227 {
1228         int ret;
1229
1230         /* use pio mode for i.mx6dl chip TKT238285 */
1231         if (of_machine_is_compatible("fsl,imx6dl"))
1232                 return 0;
1233
1234         spi_imx->wml = spi_imx->devtype_data->fifo_size / 2;
1235
1236         /* Prepare for TX DMA: */
1237         master->dma_tx = dma_request_slave_channel_reason(dev, "tx");
1238         if (IS_ERR(master->dma_tx)) {
1239                 ret = PTR_ERR(master->dma_tx);
1240                 dev_dbg(dev, "can't get the TX DMA channel, error %d!\n", ret);
1241                 master->dma_tx = NULL;
1242                 goto err;
1243         }
1244
1245         /* Prepare for RX : */
1246         master->dma_rx = dma_request_slave_channel_reason(dev, "rx");
1247         if (IS_ERR(master->dma_rx)) {
1248                 ret = PTR_ERR(master->dma_rx);
1249                 dev_dbg(dev, "can't get the RX DMA channel, error %d\n", ret);
1250                 master->dma_rx = NULL;
1251                 goto err;
1252         }
1253
1254         init_completion(&spi_imx->dma_rx_completion);
1255         init_completion(&spi_imx->dma_tx_completion);
1256         master->can_dma = spi_imx_can_dma;
1257         master->max_dma_len = MAX_SDMA_BD_BYTES;
1258         spi_imx->bitbang.master->flags = SPI_MASTER_MUST_RX |
1259                                          SPI_MASTER_MUST_TX;
1260
1261         return 0;
1262 err:
1263         spi_imx_sdma_exit(spi_imx);
1264         return ret;
1265 }
1266
1267 static void spi_imx_dma_rx_callback(void *cookie)
1268 {
1269         struct spi_imx_data *spi_imx = (struct spi_imx_data *)cookie;
1270
1271         complete(&spi_imx->dma_rx_completion);
1272 }
1273
1274 static void spi_imx_dma_tx_callback(void *cookie)
1275 {
1276         struct spi_imx_data *spi_imx = (struct spi_imx_data *)cookie;
1277
1278         complete(&spi_imx->dma_tx_completion);
1279 }
1280
1281 static int spi_imx_calculate_timeout(struct spi_imx_data *spi_imx, int size)
1282 {
1283         unsigned long timeout = 0;
1284
1285         /* Time with actual data transfer and CS change delay related to HW */
1286         timeout = (8 + 4) * size / spi_imx->spi_bus_clk;
1287
1288         /* Add extra second for scheduler related activities */
1289         timeout += 1;
1290
1291         /* Double calculated timeout */
1292         return msecs_to_jiffies(2 * timeout * MSEC_PER_SEC);
1293 }
1294
1295 static int spi_imx_dma_transfer(struct spi_imx_data *spi_imx,
1296                                 struct spi_transfer *transfer)
1297 {
1298         struct dma_async_tx_descriptor *desc_tx, *desc_rx;
1299         unsigned long transfer_timeout;
1300         unsigned long timeout;
1301         struct spi_master *master = spi_imx->bitbang.master;
1302         struct sg_table *tx = &transfer->tx_sg, *rx = &transfer->rx_sg;
1303
1304         /*
1305          * The TX DMA setup starts the transfer, so make sure RX is configured
1306          * before TX.
1307          */
1308         desc_rx = dmaengine_prep_slave_sg(master->dma_rx,
1309                                 rx->sgl, rx->nents, DMA_DEV_TO_MEM,
1310                                 DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
1311         if (!desc_rx)
1312                 return -EINVAL;
1313
1314         desc_rx->callback = spi_imx_dma_rx_callback;
1315         desc_rx->callback_param = (void *)spi_imx;
1316         dmaengine_submit(desc_rx);
1317         reinit_completion(&spi_imx->dma_rx_completion);
1318         dma_async_issue_pending(master->dma_rx);
1319
1320         desc_tx = dmaengine_prep_slave_sg(master->dma_tx,
1321                                 tx->sgl, tx->nents, DMA_MEM_TO_DEV,
1322                                 DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
1323         if (!desc_tx) {
1324                 dmaengine_terminate_all(master->dma_tx);
1325                 return -EINVAL;
1326         }
1327
1328         desc_tx->callback = spi_imx_dma_tx_callback;
1329         desc_tx->callback_param = (void *)spi_imx;
1330         dmaengine_submit(desc_tx);
1331         reinit_completion(&spi_imx->dma_tx_completion);
1332         dma_async_issue_pending(master->dma_tx);
1333
1334         transfer_timeout = spi_imx_calculate_timeout(spi_imx, transfer->len);
1335
1336         /* Wait SDMA to finish the data transfer.*/
1337         timeout = wait_for_completion_timeout(&spi_imx->dma_tx_completion,
1338                                                 transfer_timeout);
1339         if (!timeout) {
1340                 dev_err(spi_imx->dev, "I/O Error in DMA TX\n");
1341                 dmaengine_terminate_all(master->dma_tx);
1342                 dmaengine_terminate_all(master->dma_rx);
1343                 return -ETIMEDOUT;
1344         }
1345
1346         timeout = wait_for_completion_timeout(&spi_imx->dma_rx_completion,
1347                                               transfer_timeout);
1348         if (!timeout) {
1349                 dev_err(&master->dev, "I/O Error in DMA RX\n");
1350                 spi_imx->devtype_data->reset(spi_imx);
1351                 dmaengine_terminate_all(master->dma_rx);
1352                 return -ETIMEDOUT;
1353         }
1354
1355         return transfer->len;
1356 }
1357
1358 static int spi_imx_pio_transfer(struct spi_device *spi,
1359                                 struct spi_transfer *transfer)
1360 {
1361         struct spi_imx_data *spi_imx = spi_master_get_devdata(spi->master);
1362         unsigned long transfer_timeout;
1363         unsigned long timeout;
1364
1365         spi_imx->tx_buf = transfer->tx_buf;
1366         spi_imx->rx_buf = transfer->rx_buf;
1367         spi_imx->count = transfer->len;
1368         spi_imx->txfifo = 0;
1369         spi_imx->remainder = 0;
1370
1371         reinit_completion(&spi_imx->xfer_done);
1372
1373         spi_imx_push(spi_imx);
1374
1375         spi_imx->devtype_data->intctrl(spi_imx, MXC_INT_TE);
1376
1377         transfer_timeout = spi_imx_calculate_timeout(spi_imx, transfer->len);
1378
1379         timeout = wait_for_completion_timeout(&spi_imx->xfer_done,
1380                                               transfer_timeout);
1381         if (!timeout) {
1382                 dev_err(&spi->dev, "I/O Error in PIO\n");
1383                 spi_imx->devtype_data->reset(spi_imx);
1384                 return -ETIMEDOUT;
1385         }
1386
1387         return transfer->len;
1388 }
1389
1390 static int spi_imx_pio_transfer_slave(struct spi_device *spi,
1391                                       struct spi_transfer *transfer)
1392 {
1393         struct spi_imx_data *spi_imx = spi_master_get_devdata(spi->master);
1394         int ret = transfer->len;
1395
1396         if (is_imx53_ecspi(spi_imx) &&
1397             transfer->len > MX53_MAX_TRANSFER_BYTES) {
1398                 dev_err(&spi->dev, "Transaction too big, max size is %d bytes\n",
1399                         MX53_MAX_TRANSFER_BYTES);
1400                 return -EMSGSIZE;
1401         }
1402
1403         spi_imx->tx_buf = transfer->tx_buf;
1404         spi_imx->rx_buf = transfer->rx_buf;
1405         spi_imx->count = transfer->len;
1406         spi_imx->txfifo = 0;
1407         spi_imx->remainder = 0;
1408
1409         reinit_completion(&spi_imx->xfer_done);
1410         spi_imx->slave_aborted = false;
1411
1412         spi_imx_push(spi_imx);
1413
1414         spi_imx->devtype_data->intctrl(spi_imx, MXC_INT_TE | MXC_INT_RDR);
1415
1416         if (wait_for_completion_interruptible(&spi_imx->xfer_done) ||
1417             spi_imx->slave_aborted) {
1418                 dev_dbg(&spi->dev, "interrupted\n");
1419                 ret = -EINTR;
1420         }
1421
1422         /* ecspi has a HW issue when works in Slave mode,
1423          * after 64 words writtern to TXFIFO, even TXFIFO becomes empty,
1424          * ECSPI_TXDATA keeps shift out the last word data,
1425          * so we have to disable ECSPI when in slave mode after the
1426          * transfer completes
1427          */
1428         if (spi_imx->devtype_data->disable)
1429                 spi_imx->devtype_data->disable(spi_imx);
1430
1431         return ret;
1432 }
1433
1434 static int spi_imx_transfer(struct spi_device *spi,
1435                                 struct spi_transfer *transfer)
1436 {
1437         struct spi_imx_data *spi_imx = spi_master_get_devdata(spi->master);
1438
1439         /* flush rxfifo before transfer */
1440         while (spi_imx->devtype_data->rx_available(spi_imx))
1441                 readl(spi_imx->base + MXC_CSPIRXDATA);
1442
1443         if (spi_imx->slave_mode)
1444                 return spi_imx_pio_transfer_slave(spi, transfer);
1445
1446         if (spi_imx->usedma)
1447                 return spi_imx_dma_transfer(spi_imx, transfer);
1448         else
1449                 return spi_imx_pio_transfer(spi, transfer);
1450 }
1451
1452 static int spi_imx_setup(struct spi_device *spi)
1453 {
1454         dev_dbg(&spi->dev, "%s: mode %d, %u bpw, %d hz\n", __func__,
1455                  spi->mode, spi->bits_per_word, spi->max_speed_hz);
1456
1457         if (spi->mode & SPI_NO_CS)
1458                 return 0;
1459
1460         if (gpio_is_valid(spi->cs_gpio))
1461                 gpio_direction_output(spi->cs_gpio,
1462                                       spi->mode & SPI_CS_HIGH ? 0 : 1);
1463
1464         spi_imx_chipselect(spi, BITBANG_CS_INACTIVE);
1465
1466         return 0;
1467 }
1468
1469 static void spi_imx_cleanup(struct spi_device *spi)
1470 {
1471 }
1472
1473 static int
1474 spi_imx_prepare_message(struct spi_master *master, struct spi_message *msg)
1475 {
1476         struct spi_imx_data *spi_imx = spi_master_get_devdata(master);
1477         int ret;
1478
1479         ret = clk_enable(spi_imx->clk_per);
1480         if (ret)
1481                 return ret;
1482
1483         ret = clk_enable(spi_imx->clk_ipg);
1484         if (ret) {
1485                 clk_disable(spi_imx->clk_per);
1486                 return ret;
1487         }
1488
1489         return 0;
1490 }
1491
1492 static int
1493 spi_imx_unprepare_message(struct spi_master *master, struct spi_message *msg)
1494 {
1495         struct spi_imx_data *spi_imx = spi_master_get_devdata(master);
1496
1497         clk_disable(spi_imx->clk_ipg);
1498         clk_disable(spi_imx->clk_per);
1499         return 0;
1500 }
1501
1502 static int spi_imx_slave_abort(struct spi_master *master)
1503 {
1504         struct spi_imx_data *spi_imx = spi_master_get_devdata(master);
1505
1506         spi_imx->slave_aborted = true;
1507         complete(&spi_imx->xfer_done);
1508
1509         return 0;
1510 }
1511
1512 static int spi_imx_probe(struct platform_device *pdev)
1513 {
1514         struct device_node *np = pdev->dev.of_node;
1515         const struct of_device_id *of_id =
1516                         of_match_device(spi_imx_dt_ids, &pdev->dev);
1517         struct spi_imx_master *mxc_platform_info =
1518                         dev_get_platdata(&pdev->dev);
1519         struct spi_master *master;
1520         struct spi_imx_data *spi_imx;
1521         struct resource *res;
1522         int i, ret, irq, spi_drctl;
1523         const struct spi_imx_devtype_data *devtype_data = of_id ? of_id->data :
1524                 (struct spi_imx_devtype_data *)pdev->id_entry->driver_data;
1525         bool slave_mode;
1526
1527         if (!np && !mxc_platform_info) {
1528                 dev_err(&pdev->dev, "can't get the platform data\n");
1529                 return -EINVAL;
1530         }
1531
1532         slave_mode = devtype_data->has_slavemode &&
1533                         of_property_read_bool(np, "spi-slave");
1534         if (slave_mode)
1535                 master = spi_alloc_slave(&pdev->dev,
1536                                          sizeof(struct spi_imx_data));
1537         else
1538                 master = spi_alloc_master(&pdev->dev,
1539                                           sizeof(struct spi_imx_data));
1540         if (!master)
1541                 return -ENOMEM;
1542
1543         ret = of_property_read_u32(np, "fsl,spi-rdy-drctl", &spi_drctl);
1544         if ((ret < 0) || (spi_drctl >= 0x3)) {
1545                 /* '11' is reserved */
1546                 spi_drctl = 0;
1547         }
1548
1549         platform_set_drvdata(pdev, master);
1550
1551         master->bits_per_word_mask = SPI_BPW_RANGE_MASK(1, 32);
1552         master->bus_num = np ? -1 : pdev->id;
1553
1554         spi_imx = spi_master_get_devdata(master);
1555         spi_imx->bitbang.master = master;
1556         spi_imx->dev = &pdev->dev;
1557         spi_imx->slave_mode = slave_mode;
1558
1559         spi_imx->devtype_data = devtype_data;
1560
1561         /* Get number of chip selects, either platform data or OF */
1562         if (mxc_platform_info) {
1563                 master->num_chipselect = mxc_platform_info->num_chipselect;
1564                 if (mxc_platform_info->chipselect) {
1565                         master->cs_gpios = devm_kcalloc(&master->dev,
1566                                 master->num_chipselect, sizeof(int),
1567                                 GFP_KERNEL);
1568                         if (!master->cs_gpios)
1569                                 return -ENOMEM;
1570
1571                         for (i = 0; i < master->num_chipselect; i++)
1572                                 master->cs_gpios[i] = mxc_platform_info->chipselect[i];
1573                 }
1574         } else {
1575                 u32 num_cs;
1576
1577                 if (!of_property_read_u32(np, "num-cs", &num_cs))
1578                         master->num_chipselect = num_cs;
1579                 /* If not preset, default value of 1 is used */
1580         }
1581
1582         spi_imx->bitbang.chipselect = spi_imx_chipselect;
1583         spi_imx->bitbang.setup_transfer = spi_imx_setupxfer;
1584         spi_imx->bitbang.txrx_bufs = spi_imx_transfer;
1585         spi_imx->bitbang.master->setup = spi_imx_setup;
1586         spi_imx->bitbang.master->cleanup = spi_imx_cleanup;
1587         spi_imx->bitbang.master->prepare_message = spi_imx_prepare_message;
1588         spi_imx->bitbang.master->unprepare_message = spi_imx_unprepare_message;
1589         spi_imx->bitbang.master->slave_abort = spi_imx_slave_abort;
1590         spi_imx->bitbang.master->mode_bits = SPI_CPOL | SPI_CPHA | SPI_CS_HIGH \
1591                                              | SPI_NO_CS;
1592         if (is_imx35_cspi(spi_imx) || is_imx51_ecspi(spi_imx) ||
1593             is_imx53_ecspi(spi_imx))
1594                 spi_imx->bitbang.master->mode_bits |= SPI_LOOP | SPI_READY;
1595
1596         spi_imx->spi_drctl = spi_drctl;
1597
1598         init_completion(&spi_imx->xfer_done);
1599
1600         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1601         spi_imx->base = devm_ioremap_resource(&pdev->dev, res);
1602         if (IS_ERR(spi_imx->base)) {
1603                 ret = PTR_ERR(spi_imx->base);
1604                 goto out_master_put;
1605         }
1606         spi_imx->base_phys = res->start;
1607
1608         irq = platform_get_irq(pdev, 0);
1609         if (irq < 0) {
1610                 ret = irq;
1611                 goto out_master_put;
1612         }
1613
1614         ret = devm_request_irq(&pdev->dev, irq, spi_imx_isr, 0,
1615                                dev_name(&pdev->dev), spi_imx);
1616         if (ret) {
1617                 dev_err(&pdev->dev, "can't get irq%d: %d\n", irq, ret);
1618                 goto out_master_put;
1619         }
1620
1621         spi_imx->clk_ipg = devm_clk_get(&pdev->dev, "ipg");
1622         if (IS_ERR(spi_imx->clk_ipg)) {
1623                 ret = PTR_ERR(spi_imx->clk_ipg);
1624                 goto out_master_put;
1625         }
1626
1627         spi_imx->clk_per = devm_clk_get(&pdev->dev, "per");
1628         if (IS_ERR(spi_imx->clk_per)) {
1629                 ret = PTR_ERR(spi_imx->clk_per);
1630                 goto out_master_put;
1631         }
1632
1633         ret = clk_prepare_enable(spi_imx->clk_per);
1634         if (ret)
1635                 goto out_master_put;
1636
1637         ret = clk_prepare_enable(spi_imx->clk_ipg);
1638         if (ret)
1639                 goto out_put_per;
1640
1641         spi_imx->spi_clk = clk_get_rate(spi_imx->clk_per);
1642         /*
1643          * Only validated on i.mx35 and i.mx6 now, can remove the constraint
1644          * if validated on other chips.
1645          */
1646         if (spi_imx->devtype_data->has_dmamode) {
1647                 ret = spi_imx_sdma_init(&pdev->dev, spi_imx, master);
1648                 if (ret == -EPROBE_DEFER)
1649                         goto out_clk_put;
1650
1651                 if (ret < 0)
1652                         dev_err(&pdev->dev, "dma setup error %d, use pio\n",
1653                                 ret);
1654         }
1655
1656         spi_imx->devtype_data->reset(spi_imx);
1657
1658         spi_imx->devtype_data->intctrl(spi_imx, 0);
1659
1660         master->dev.of_node = pdev->dev.of_node;
1661         ret = spi_bitbang_start(&spi_imx->bitbang);
1662         if (ret) {
1663                 dev_err(&pdev->dev, "bitbang start failed with %d\n", ret);
1664                 goto out_clk_put;
1665         }
1666
1667         /* Request GPIO CS lines, if any */
1668         if (!spi_imx->slave_mode && master->cs_gpios) {
1669                 for (i = 0; i < master->num_chipselect; i++) {
1670                         if (!gpio_is_valid(master->cs_gpios[i]))
1671                                 continue;
1672
1673                         ret = devm_gpio_request(&pdev->dev,
1674                                                 master->cs_gpios[i],
1675                                                 DRIVER_NAME);
1676                         if (ret) {
1677                                 dev_err(&pdev->dev, "Can't get CS GPIO %i\n",
1678                                         master->cs_gpios[i]);
1679                                 goto out_spi_bitbang;
1680                         }
1681                 }
1682         }
1683
1684         dev_info(&pdev->dev, "probed\n");
1685
1686         clk_disable(spi_imx->clk_ipg);
1687         clk_disable(spi_imx->clk_per);
1688         return ret;
1689
1690 out_spi_bitbang:
1691         spi_bitbang_stop(&spi_imx->bitbang);
1692 out_clk_put:
1693         clk_disable_unprepare(spi_imx->clk_ipg);
1694 out_put_per:
1695         clk_disable_unprepare(spi_imx->clk_per);
1696 out_master_put:
1697         spi_master_put(master);
1698
1699         return ret;
1700 }
1701
1702 static int spi_imx_remove(struct platform_device *pdev)
1703 {
1704         struct spi_master *master = platform_get_drvdata(pdev);
1705         struct spi_imx_data *spi_imx = spi_master_get_devdata(master);
1706         int ret;
1707
1708         spi_bitbang_stop(&spi_imx->bitbang);
1709
1710         ret = clk_enable(spi_imx->clk_per);
1711         if (ret)
1712                 return ret;
1713
1714         ret = clk_enable(spi_imx->clk_ipg);
1715         if (ret) {
1716                 clk_disable(spi_imx->clk_per);
1717                 return ret;
1718         }
1719
1720         writel(0, spi_imx->base + MXC_CSPICTRL);
1721         clk_disable_unprepare(spi_imx->clk_ipg);
1722         clk_disable_unprepare(spi_imx->clk_per);
1723         spi_imx_sdma_exit(spi_imx);
1724         spi_master_put(master);
1725
1726         return 0;
1727 }
1728
1729 static struct platform_driver spi_imx_driver = {
1730         .driver = {
1731                    .name = DRIVER_NAME,
1732                    .of_match_table = spi_imx_dt_ids,
1733                    },
1734         .id_table = spi_imx_devtype,
1735         .probe = spi_imx_probe,
1736         .remove = spi_imx_remove,
1737 };
1738 module_platform_driver(spi_imx_driver);
1739
1740 MODULE_DESCRIPTION("SPI Controller driver");
1741 MODULE_AUTHOR("Sascha Hauer, Pengutronix");
1742 MODULE_LICENSE("GPL");
1743 MODULE_ALIAS("platform:" DRIVER_NAME);