drivers/mmc/host/sdhci.c

   1 /*
   2  *  linux/drivers/mmc/host/sdhci.c - Secure Digital Host Controller Interface driver
   3  *
   4  *  Copyright (C) 2005-2008 Pierre Ossman, All Rights Reserved.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License as published by
   8  * the Free Software Foundation; either version 2 of the License, or (at
   9  * your option) any later version.
  10  *
  11  * Thanks to the following companies for their support:
  12  *
  13  *     - JMicron (hardware and technical support)
  14  */
  15
  16 #include <linux/delay.h>
  17 #include <linux/ktime.h>
  18 #include <linux/highmem.h>
  19 #include <linux/io.h>
  20 #include <linux/module.h>
  21 #include <linux/dma-mapping.h>
  22 #include <linux/slab.h>
  23 #include <linux/scatterlist.h>
  24 #include <linux/sizes.h>
  25 #include <linux/swiotlb.h>
  26 #include <linux/regulator/consumer.h>
  27 #include <linux/pm_runtime.h>
  28 #include <linux/of.h>
  29
  30 #include <linux/leds.h>
  31
  32 #include <linux/mmc/mmc.h>
  33 #include <linux/mmc/host.h>
  34 #include <linux/mmc/card.h>
  35 #include <linux/mmc/sdio.h>
  36 #include <linux/mmc/slot-gpio.h>
  37
  38 #include "sdhci.h"
  39
  40 #define DRIVER_NAME "sdhci"
  41
  42 #define DBG(f, x...) \
  43         pr_debug("%s: " DRIVER_NAME ": " f, mmc_hostname(host->mmc), ## x)
  44
  45 #define SDHCI_DUMP(f, x...) \
  46         pr_err("%s: " DRIVER_NAME ": " f, mmc_hostname(host->mmc), ## x)
  47
  48 #define MAX_TUNING_LOOP 40
  49
  50 static unsigned int debug_quirks = 0;
  51 static unsigned int debug_quirks2;
  52
  53 static void sdhci_finish_data(struct sdhci_host *);
  54
  55 static void sdhci_enable_preset_value(struct sdhci_host *host, bool enable);
  56
  57 void sdhci_dumpregs(struct sdhci_host *host)
  58 {
  59         SDHCI_DUMP("============ SDHCI REGISTER DUMP ===========\n");
  60
  61         SDHCI_DUMP("Sys addr:  0x%08x | Version:  0x%08x\n",
  62                    sdhci_readl(host, SDHCI_DMA_ADDRESS),
  63                    sdhci_readw(host, SDHCI_HOST_VERSION));
  64         SDHCI_DUMP("Blk size:  0x%08x | Blk cnt:  0x%08x\n",
  65                    sdhci_readw(host, SDHCI_BLOCK_SIZE),
  66                    sdhci_readw(host, SDHCI_BLOCK_COUNT));
  67         SDHCI_DUMP("Argument:  0x%08x | Trn mode: 0x%08x\n",
  68                    sdhci_readl(host, SDHCI_ARGUMENT),
  69                    sdhci_readw(host, SDHCI_TRANSFER_MODE));
  70         SDHCI_DUMP("Present:   0x%08x | Host ctl: 0x%08x\n",
  71                    sdhci_readl(host, SDHCI_PRESENT_STATE),
  72                    sdhci_readb(host, SDHCI_HOST_CONTROL));
  73         SDHCI_DUMP("Power:     0x%08x | Blk gap:  0x%08x\n",
  74                    sdhci_readb(host, SDHCI_POWER_CONTROL),
  75                    sdhci_readb(host, SDHCI_BLOCK_GAP_CONTROL));
  76         SDHCI_DUMP("Wake-up:   0x%08x | Clock:    0x%08x\n",
  77                    sdhci_readb(host, SDHCI_WAKE_UP_CONTROL),
  78                    sdhci_readw(host, SDHCI_CLOCK_CONTROL));
  79         SDHCI_DUMP("Timeout:   0x%08x | Int stat: 0x%08x\n",
  80                    sdhci_readb(host, SDHCI_TIMEOUT_CONTROL),
  81                    sdhci_readl(host, SDHCI_INT_STATUS));
  82         SDHCI_DUMP("Int enab:  0x%08x | Sig enab: 0x%08x\n",
  83                    sdhci_readl(host, SDHCI_INT_ENABLE),
  84                    sdhci_readl(host, SDHCI_SIGNAL_ENABLE));
  85         SDHCI_DUMP("ACmd stat: 0x%08x | Slot int: 0x%08x\n",
  86                    sdhci_readw(host, SDHCI_AUTO_CMD_STATUS),
  87                    sdhci_readw(host, SDHCI_SLOT_INT_STATUS));
  88         SDHCI_DUMP("Caps:      0x%08x | Caps_1:   0x%08x\n",
  89                    sdhci_readl(host, SDHCI_CAPABILITIES),
  90                    sdhci_readl(host, SDHCI_CAPABILITIES_1));
  91         SDHCI_DUMP("Cmd:       0x%08x | Max curr: 0x%08x\n",
  92                    sdhci_readw(host, SDHCI_COMMAND),
  93                    sdhci_readl(host, SDHCI_MAX_CURRENT));
  94         SDHCI_DUMP("Resp[0]:   0x%08x | Resp[1]:  0x%08x\n",
  95                    sdhci_readl(host, SDHCI_RESPONSE),
  96                    sdhci_readl(host, SDHCI_RESPONSE + 4));
  97         SDHCI_DUMP("Resp[2]:   0x%08x | Resp[3]:  0x%08x\n",
  98                    sdhci_readl(host, SDHCI_RESPONSE + 8),
  99                    sdhci_readl(host, SDHCI_RESPONSE + 12));
 100         SDHCI_DUMP("Host ctl2: 0x%08x\n",
 101                    sdhci_readw(host, SDHCI_HOST_CONTROL2));
 102
 103         if (host->flags & SDHCI_USE_ADMA) {
 104                 if (host->flags & SDHCI_USE_64_BIT_DMA) {
 105                         SDHCI_DUMP("ADMA Err:  0x%08x | ADMA Ptr: 0x%08x%08x\n",
 106                                    sdhci_readl(host, SDHCI_ADMA_ERROR),
 107                                    sdhci_readl(host, SDHCI_ADMA_ADDRESS_HI),
 108                                    sdhci_readl(host, SDHCI_ADMA_ADDRESS));
 109                 } else {
 110                         SDHCI_DUMP("ADMA Err:  0x%08x | ADMA Ptr: 0x%08x\n",
 111                                    sdhci_readl(host, SDHCI_ADMA_ERROR),
 112                                    sdhci_readl(host, SDHCI_ADMA_ADDRESS));
 113                 }
 114         }
 115
 116         SDHCI_DUMP("============================================\n");
 117 }
 118 EXPORT_SYMBOL_GPL(sdhci_dumpregs);
 119
 120 /*****************************************************************************\
 121  *                                                                           *
 122  * Low level functions                                                       *
 123  *                                                                           *
 124 \*****************************************************************************/
 125
 126 static inline bool sdhci_data_line_cmd(struct mmc_command *cmd)
 127 {
 128         return cmd->data || cmd->flags & MMC_RSP_BUSY;
 129 }
 130
 131 static void sdhci_set_card_detection(struct sdhci_host *host, bool enable)
 132 {
 133         u32 present;
 134
 135         if ((host->quirks & SDHCI_QUIRK_BROKEN_CARD_DETECTION) ||
 136             !mmc_card_is_removable(host->mmc) || mmc_can_gpio_cd(host->mmc))
 137                 return;
 138
 139         if (enable) {
 140                 present = sdhci_readl(host, SDHCI_PRESENT_STATE) &
 141                                       SDHCI_CARD_PRESENT;
 142
 143                 host->ier |= present ? SDHCI_INT_CARD_REMOVE :
 144                                        SDHCI_INT_CARD_INSERT;
 145         } else {
 146                 host->ier &= ~(SDHCI_INT_CARD_REMOVE | SDHCI_INT_CARD_INSERT);
 147         }
 148
 149         sdhci_writel(host, host->ier, SDHCI_INT_ENABLE);
 150         sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE);
 151 }
 152
 153 static void sdhci_enable_card_detection(struct sdhci_host *host)
 154 {
 155         sdhci_set_card_detection(host, true);
 156 }
 157
 158 static void sdhci_disable_card_detection(struct sdhci_host *host)
 159 {
 160         sdhci_set_card_detection(host, false);
 161 }
 162
 163 static void sdhci_runtime_pm_bus_on(struct sdhci_host *host)
 164 {
 165         if (host->bus_on)
 166                 return;
 167         host->bus_on = true;
 168         pm_runtime_get_noresume(host->mmc->parent);
 169 }
 170
 171 static void sdhci_runtime_pm_bus_off(struct sdhci_host *host)
 172 {
 173         if (!host->bus_on)
 174                 return;
 175         host->bus_on = false;
 176         pm_runtime_put_noidle(host->mmc->parent);
 177 }
 178
 179 void sdhci_reset(struct sdhci_host *host, u8 mask)
 180 {
 181         ktime_t timeout;
 182
 183         sdhci_writeb(host, mask, SDHCI_SOFTWARE_RESET);
 184
 185         if (mask & SDHCI_RESET_ALL) {
 186                 host->clock = 0;
 187                 /* Reset-all turns off SD Bus Power */
 188                 if (host->quirks2 & SDHCI_QUIRK2_CARD_ON_NEEDS_BUS_ON)
 189                         sdhci_runtime_pm_bus_off(host);
 190         }
 191
 192         /* Wait max 100 ms */
 193         timeout = ktime_add_ms(ktime_get(), 100);
 194
 195         /* hw clears the bit when it's done */
 196         while (1) {
 197                 bool timedout = ktime_after(ktime_get(), timeout);
 198
 199                 if (!(sdhci_readb(host, SDHCI_SOFTWARE_RESET) & mask))
 200                         break;
 201                 if (timedout) {
 202                         pr_err("%s: Reset 0x%x never completed.\n",
 203                                 mmc_hostname(host->mmc), (int)mask);
 204                         sdhci_dumpregs(host);
 205                         return;
 206                 }
 207                 udelay(10);
 208         }
 209 }
 210 EXPORT_SYMBOL_GPL(sdhci_reset);
 211
 212 static void sdhci_do_reset(struct sdhci_host *host, u8 mask)
 213 {
 214         if (host->quirks & SDHCI_QUIRK_NO_CARD_NO_RESET) {
 215                 struct mmc_host *mmc = host->mmc;
 216
 217                 if (!mmc->ops->get_cd(mmc))
 218                         return;
 219         }
 220
 221         host->ops->reset(host, mask);
 222
 223         if (mask & SDHCI_RESET_ALL) {
 224                 if (host->flags & (SDHCI_USE_SDMA | SDHCI_USE_ADMA)) {
 225                         if (host->ops->enable_dma)
 226                                 host->ops->enable_dma(host);
 227                 }
 228
 229                 /* Resetting the controller clears many */
 230                 host->preset_enabled = false;
 231         }
 232 }
 233
 234 static void sdhci_set_default_irqs(struct sdhci_host *host)
 235 {
 236         host->ier = SDHCI_INT_BUS_POWER | SDHCI_INT_DATA_END_BIT |
 237                     SDHCI_INT_DATA_CRC | SDHCI_INT_DATA_TIMEOUT |
 238                     SDHCI_INT_INDEX | SDHCI_INT_END_BIT | SDHCI_INT_CRC |
 239                     SDHCI_INT_TIMEOUT | SDHCI_INT_DATA_END |
 240                     SDHCI_INT_RESPONSE;
 241
 242         if (host->tuning_mode == SDHCI_TUNING_MODE_2 ||
 243             host->tuning_mode == SDHCI_TUNING_MODE_3)
 244                 host->ier |= SDHCI_INT_RETUNE;
 245
 246         sdhci_writel(host, host->ier, SDHCI_INT_ENABLE);
 247         sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE);
 248 }
 249
 250 static void sdhci_init(struct sdhci_host *host, int soft)
 251 {
 252         struct mmc_host *mmc = host->mmc;
 253
 254         if (soft)
 255                 sdhci_do_reset(host, SDHCI_RESET_CMD | SDHCI_RESET_DATA);
 256         else
 257                 sdhci_do_reset(host, SDHCI_RESET_ALL);
 258
 259         sdhci_set_default_irqs(host);
 260
 261         host->cqe_on = false;
 262
 263         if (soft) {
 264                 /* force clock reconfiguration */
 265                 host->clock = 0;
 266                 mmc->ops->set_ios(mmc, &mmc->ios);
 267         }
 268 }
 269
 270 static void sdhci_reinit(struct sdhci_host *host)
 271 {
 272         sdhci_init(host, 0);
 273         sdhci_enable_card_detection(host);
 274 }
 275
 276 static void __sdhci_led_activate(struct sdhci_host *host)
 277 {
 278         u8 ctrl;
 279
 280         ctrl = sdhci_readb(host, SDHCI_HOST_CONTROL);
 281         ctrl |= SDHCI_CTRL_LED;
 282         sdhci_writeb(host, ctrl, SDHCI_HOST_CONTROL);
 283 }
 284
 285 static void __sdhci_led_deactivate(struct sdhci_host *host)
 286 {
 287         u8 ctrl;
 288
 289         ctrl = sdhci_readb(host, SDHCI_HOST_CONTROL);
 290         ctrl &= ~SDHCI_CTRL_LED;
 291         sdhci_writeb(host, ctrl, SDHCI_HOST_CONTROL);
 292 }
 293
 294 #if IS_REACHABLE(CONFIG_LEDS_CLASS)
 295 static void sdhci_led_control(struct led_classdev *led,
 296                               enum led_brightness brightness)
 297 {
 298         struct sdhci_host *host = container_of(led, struct sdhci_host, led);
 299         unsigned long flags;
 300
 301         spin_lock_irqsave(&host->lock, flags);
 302
 303         if (host->runtime_suspended)
 304                 goto out;
 305
 306         if (brightness == LED_OFF)
 307                 __sdhci_led_deactivate(host);
 308         else
 309                 __sdhci_led_activate(host);
 310 out:
 311         spin_unlock_irqrestore(&host->lock, flags);
 312 }
 313
 314 static int sdhci_led_register(struct sdhci_host *host)
 315 {
 316         struct mmc_host *mmc = host->mmc;
 317
 318         snprintf(host->led_name, sizeof(host->led_name),
 319                  "%s::", mmc_hostname(mmc));
 320
 321         host->led.name = host->led_name;
 322         host->led.brightness = LED_OFF;
 323         host->led.default_trigger = mmc_hostname(mmc);
 324         host->led.brightness_set = sdhci_led_control;
 325
 326         return led_classdev_register(mmc_dev(mmc), &host->led);
 327 }
 328
 329 static void sdhci_led_unregister(struct sdhci_host *host)
 330 {
 331         led_classdev_unregister(&host->led);
 332 }
 333
 334 static inline void sdhci_led_activate(struct sdhci_host *host)
 335 {
 336 }
 337
 338 static inline void sdhci_led_deactivate(struct sdhci_host *host)
 339 {
 340 }
 341
 342 #else
 343
 344 static inline int sdhci_led_register(struct sdhci_host *host)
 345 {
 346         return 0;
 347 }
 348
 349 static inline void sdhci_led_unregister(struct sdhci_host *host)
 350 {
 351 }
 352
 353 static inline void sdhci_led_activate(struct sdhci_host *host)
 354 {
 355         __sdhci_led_activate(host);
 356 }
 357
 358 static inline void sdhci_led_deactivate(struct sdhci_host *host)
 359 {
 360         __sdhci_led_deactivate(host);
 361 }
 362
 363 #endif
 364
 365 /*****************************************************************************\
 366  *                                                                           *
 367  * Core functions                                                            *
 368  *                                                                           *
 369 \*****************************************************************************/
 370
 371 static void sdhci_read_block_pio(struct sdhci_host *host)
 372 {
 373         unsigned long flags;
 374         size_t blksize, len, chunk;
 375         u32 uninitialized_var(scratch);
 376         u8 *buf;
 377
 378         DBG("PIO reading\n");
 379
 380         blksize = host->data->blksz;
 381         chunk = 0;
 382
 383         local_irq_save(flags);
 384
 385         while (blksize) {
 386                 BUG_ON(!sg_miter_next(&host->sg_miter));
 387
 388                 len = min(host->sg_miter.length, blksize);
 389
 390                 blksize -= len;
 391                 host->sg_miter.consumed = len;
 392
 393                 buf = host->sg_miter.addr;
 394
 395                 while (len) {
 396                         if (chunk == 0) {
 397                                 scratch = sdhci_readl(host, SDHCI_BUFFER);
 398                                 chunk = 4;
 399                         }
 400
 401                         *buf = scratch & 0xFF;
 402
 403                         buf++;
 404                         scratch >>= 8;
 405                         chunk--;
 406                         len--;
 407                 }
 408         }
 409
 410         sg_miter_stop(&host->sg_miter);
 411
 412         local_irq_restore(flags);
 413 }
 414
 415 static void sdhci_write_block_pio(struct sdhci_host *host)
 416 {
 417         unsigned long flags;
 418         size_t blksize, len, chunk;
 419         u32 scratch;
 420         u8 *buf;
 421
 422         DBG("PIO writing\n");
 423
 424         blksize = host->data->blksz;
 425         chunk = 0;
 426         scratch = 0;
 427
 428         local_irq_save(flags);
 429
 430         while (blksize) {
 431                 BUG_ON(!sg_miter_next(&host->sg_miter));
 432
 433                 len = min(host->sg_miter.length, blksize);
 434
 435                 blksize -= len;
 436                 host->sg_miter.consumed = len;
 437
 438                 buf = host->sg_miter.addr;
 439
 440                 while (len) {
 441                         scratch |= (u32)*buf << (chunk * 8);
 442
 443                         buf++;
 444                         chunk++;
 445                         len--;
 446
 447                         if ((chunk == 4) || ((len == 0) && (blksize == 0))) {
 448                                 sdhci_writel(host, scratch, SDHCI_BUFFER);
 449                                 chunk = 0;
 450                                 scratch = 0;
 451                         }
 452                 }
 453         }
 454
 455         sg_miter_stop(&host->sg_miter);
 456
 457         local_irq_restore(flags);
 458 }
 459
 460 static void sdhci_transfer_pio(struct sdhci_host *host)
 461 {
 462         u32 mask;
 463
 464         if (host->blocks == 0)
 465                 return;
 466
 467         if (host->data->flags & MMC_DATA_READ)
 468                 mask = SDHCI_DATA_AVAILABLE;
 469         else
 470                 mask = SDHCI_SPACE_AVAILABLE;
 471
 472         /*
 473          * Some controllers (JMicron JMB38x) mess up the buffer bits
 474          * for transfers < 4 bytes. As long as it is just one block,
 475          * we can ignore the bits.
 476          */
 477         if ((host->quirks & SDHCI_QUIRK_BROKEN_SMALL_PIO) &&
 478                 (host->data->blocks == 1))
 479                 mask = ~0;
 480
 481         while (sdhci_readl(host, SDHCI_PRESENT_STATE) & mask) {
 482                 if (host->quirks & SDHCI_QUIRK_PIO_NEEDS_DELAY)
 483                         udelay(100);
 484
 485                 if (host->data->flags & MMC_DATA_READ)
 486                         sdhci_read_block_pio(host);
 487                 else
 488                         sdhci_write_block_pio(host);
 489
 490                 host->blocks--;
 491                 if (host->blocks == 0)
 492                         break;
 493         }
 494
 495         DBG("PIO transfer complete.\n");
 496 }
 497
 498 static int sdhci_pre_dma_transfer(struct sdhci_host *host,
 499                                   struct mmc_data *data, int cookie)
 500 {
 501         int sg_count;
 502
 503         /*
 504          * If the data buffers are already mapped, return the previous
 505          * dma_map_sg() result.
 506          */
 507         if (data->host_cookie == COOKIE_PRE_MAPPED)
 508                 return data->sg_count;
 509
 510         /* Bounce write requests to the bounce buffer */
 511         if (host->bounce_buffer) {
 512                 unsigned int length = data->blksz * data->blocks;
 513
 514                 if (length > host->bounce_buffer_size) {
 515                         pr_err("%s: asked for transfer of %u bytes exceeds bounce buffer %u bytes\n",
 516                                mmc_hostname(host->mmc), length,
 517                                host->bounce_buffer_size);
 518                         return -EIO;
 519                 }
 520                 if (mmc_get_dma_dir(data) == DMA_TO_DEVICE) {
 521                         /* Copy the data to the bounce buffer */
 522                         sg_copy_to_buffer(data->sg, data->sg_len,
 523                                           host->bounce_buffer,
 524                                           length);
 525                 }
 526                 /* Switch ownership to the DMA */
 527                 dma_sync_single_for_device(host->mmc->parent,
 528                                            host->bounce_addr,
 529                                            host->bounce_buffer_size,
 530                                            mmc_get_dma_dir(data));
 531                 /* Just a dummy value */
 532                 sg_count = 1;
 533         } else {
 534                 /* Just access the data directly from memory */
 535                 sg_count = dma_map_sg(mmc_dev(host->mmc),
 536                                       data->sg, data->sg_len,
 537                                       mmc_get_dma_dir(data));
 538         }
 539
 540         if (sg_count == 0)
 541                 return -ENOSPC;
 542
 543         data->sg_count = sg_count;
 544         data->host_cookie = cookie;
 545
 546         return sg_count;
 547 }
 548
 549 static char *sdhci_kmap_atomic(struct scatterlist *sg, unsigned long *flags)
 550 {
 551         local_irq_save(*flags);
 552         return kmap_atomic(sg_page(sg)) + sg->offset;
 553 }
 554
 555 static void sdhci_kunmap_atomic(void *buffer, unsigned long *flags)
 556 {
 557         kunmap_atomic(buffer);
 558         local_irq_restore(*flags);
 559 }
 560
 561 static void sdhci_adma_write_desc(struct sdhci_host *host, void *desc,
 562                                   dma_addr_t addr, int len, unsigned cmd)
 563 {
 564         struct sdhci_adma2_64_desc *dma_desc = desc;
 565
 566         /* 32-bit and 64-bit descriptors have these members in same position */
 567         dma_desc->cmd = cpu_to_le16(cmd);
 568         dma_desc->len = cpu_to_le16(len);
 569         dma_desc->addr_lo = cpu_to_le32((u32)addr);
 570
 571         if (host->flags & SDHCI_USE_64_BIT_DMA)
 572                 dma_desc->addr_hi = cpu_to_le32((u64)addr >> 32);
 573 }
 574
 575 static void sdhci_adma_mark_end(void *desc)
 576 {
 577         struct sdhci_adma2_64_desc *dma_desc = desc;
 578
 579         /* 32-bit and 64-bit descriptors have 'cmd' in same position */
 580         dma_desc->cmd |= cpu_to_le16(ADMA2_END);
 581 }
 582
 583 static void sdhci_adma_table_pre(struct sdhci_host *host,
 584         struct mmc_data *data, int sg_count)
 585 {
 586         struct scatterlist *sg;
 587         unsigned long flags;
 588         dma_addr_t addr, align_addr;
 589         void *desc, *align;
 590         char *buffer;
 591         int len, offset, i;
 592
 593         /*
 594          * The spec does not specify endianness of descriptor table.
 595          * We currently guess that it is LE.
 596          */
 597
 598         host->sg_count = sg_count;
 599
 600         desc = host->adma_table;
 601         align = host->align_buffer;
 602
 603         align_addr = host->align_addr;
 604
 605         for_each_sg(data->sg, sg, host->sg_count, i) {
 606                 addr = sg_dma_address(sg);
 607                 len = sg_dma_len(sg);
 608
 609                 /*
 610                  * The SDHCI specification states that ADMA addresses must
 611                  * be 32-bit aligned. If they aren't, then we use a bounce
 612                  * buffer for the (up to three) bytes that screw up the
 613                  * alignment.
 614                  */
 615                 offset = (SDHCI_ADMA2_ALIGN - (addr & SDHCI_ADMA2_MASK)) &
 616                          SDHCI_ADMA2_MASK;
 617                 if (offset) {
 618                         if (data->flags & MMC_DATA_WRITE) {
 619                                 buffer = sdhci_kmap_atomic(sg, &flags);
 620                                 memcpy(align, buffer, offset);
 621                                 sdhci_kunmap_atomic(buffer, &flags);
 622                         }
 623
 624                         /* tran, valid */
 625                         sdhci_adma_write_desc(host, desc, align_addr, offset,
 626                                               ADMA2_TRAN_VALID);
 627
 628                         BUG_ON(offset > 65536);
 629
 630                         align += SDHCI_ADMA2_ALIGN;
 631                         align_addr += SDHCI_ADMA2_ALIGN;
 632
 633                         desc += host->desc_sz;
 634
 635                         addr += offset;
 636                         len -= offset;
 637                 }
 638
 639                 BUG_ON(len > 65536);
 640
 641                 if (len) {
 642                         /* tran, valid */
 643                         sdhci_adma_write_desc(host, desc, addr, len,
 644                                               ADMA2_TRAN_VALID);
 645                         desc += host->desc_sz;
 646                 }
 647
 648                 /*
 649                  * If this triggers then we have a calculation bug
 650                  * somewhere. :/
 651                  */
 652                 WARN_ON((desc - host->adma_table) >= host->adma_table_sz);
 653         }
 654
 655         if (host->quirks & SDHCI_QUIRK_NO_ENDATTR_IN_NOPDESC) {
 656                 /* Mark the last descriptor as the terminating descriptor */
 657                 if (desc != host->adma_table) {
 658                         desc -= host->desc_sz;
 659                         sdhci_adma_mark_end(desc);
 660                 }
 661         } else {
 662                 /* Add a terminating entry - nop, end, valid */
 663                 sdhci_adma_write_desc(host, desc, 0, 0, ADMA2_NOP_END_VALID);
 664         }
 665 }
 666
 667 static void sdhci_adma_table_post(struct sdhci_host *host,
 668         struct mmc_data *data)
 669 {
 670         struct scatterlist *sg;
 671         int i, size;
 672         void *align;
 673         char *buffer;
 674         unsigned long flags;
 675
 676         if (data->flags & MMC_DATA_READ) {
 677                 bool has_unaligned = false;
 678
 679                 /* Do a quick scan of the SG list for any unaligned mappings */
 680                 for_each_sg(data->sg, sg, host->sg_count, i)
 681                         if (sg_dma_address(sg) & SDHCI_ADMA2_MASK) {
 682                                 has_unaligned = true;
 683                                 break;
 684                         }
 685
 686                 if (has_unaligned) {
 687                         dma_sync_sg_for_cpu(mmc_dev(host->mmc), data->sg,
 688                                             data->sg_len, DMA_FROM_DEVICE);
 689
 690                         align = host->align_buffer;
 691
 692                         for_each_sg(data->sg, sg, host->sg_count, i) {
 693                                 if (sg_dma_address(sg) & SDHCI_ADMA2_MASK) {
 694                                         size = SDHCI_ADMA2_ALIGN -
 695                                                (sg_dma_address(sg) & SDHCI_ADMA2_MASK);
 696
 697                                         buffer = sdhci_kmap_atomic(sg, &flags);
 698                                         memcpy(buffer, align, size);
 699                                         sdhci_kunmap_atomic(buffer, &flags);
 700
 701                                         align += SDHCI_ADMA2_ALIGN;
 702                                 }
 703                         }
 704                 }
 705         }
 706 }
 707
 708 static u32 sdhci_sdma_address(struct sdhci_host *host)
 709 {
 710         if (host->bounce_buffer)
 711                 return host->bounce_addr;
 712         else
 713                 return sg_dma_address(host->data->sg);
 714 }
 715
 716 static u8 sdhci_calc_timeout(struct sdhci_host *host, struct mmc_command *cmd)
 717 {
 718         u8 count;
 719         struct mmc_data *data = cmd->data;
 720         unsigned target_timeout, current_timeout;
 721
 722         /*
 723          * If the host controller provides us with an incorrect timeout
 724          * value, just skip the check and use 0xE.  The hardware may take
 725          * longer to time out, but that's much better than having a too-short
 726          * timeout value.
 727          */
 728         if (host->quirks & SDHCI_QUIRK_BROKEN_TIMEOUT_VAL)
 729                 return 0xE;
 730
 731         /* Unspecified timeout, assume max */
 732         if (!data && !cmd->busy_timeout)
 733                 return 0xE;
 734
 735         /* timeout in us */
 736         if (!data)
 737                 target_timeout = cmd->busy_timeout * 1000;
 738         else {
 739                 target_timeout = DIV_ROUND_UP(data->timeout_ns, 1000);
 740                 if (host->clock && data->timeout_clks) {
 741                         unsigned long long val;
 742
 743                         /*
 744                          * data->timeout_clks is in units of clock cycles.
 745                          * host->clock is in Hz.  target_timeout is in us.
 746                          * Hence, us = 1000000 * cycles / Hz.  Round up.
 747                          */
 748                         val = 1000000ULL * data->timeout_clks;
 749                         if (do_div(val, host->clock))
 750                                 target_timeout++;
 751                         target_timeout += val;
 752                 }
 753         }
 754
 755         /*
 756          * Figure out needed cycles.
 757          * We do this in steps in order to fit inside a 32 bit int.
 758          * The first step is the minimum timeout, which will have a
 759          * minimum resolution of 6 bits:
 760          * (1) 2^13*1000 > 2^22,
 761          * (2) host->timeout_clk < 2^16
 762          *     =>
 763          *     (1) / (2) > 2^6
 764          */
 765         count = 0;
 766         current_timeout = (1 << 13) * 1000 / host->timeout_clk;
 767         while (current_timeout < target_timeout) {
 768                 count++;
 769                 current_timeout <<= 1;
 770                 if (count >= 0xF)
 771                         break;
 772         }
 773
 774         if (count >= 0xF) {
 775                 DBG("Too large timeout 0x%x requested for CMD%d!\n",
 776                     count, cmd->opcode);
 777                 count = 0xE;
 778         }
 779
 780         return count;
 781 }
 782
 783 static void sdhci_set_transfer_irqs(struct sdhci_host *host)
 784 {
 785         u32 pio_irqs = SDHCI_INT_DATA_AVAIL | SDHCI_INT_SPACE_AVAIL;
 786         u32 dma_irqs = SDHCI_INT_DMA_END | SDHCI_INT_ADMA_ERROR;
 787
 788         if (host->flags & SDHCI_REQ_USE_DMA)
 789                 host->ier = (host->ier & ~pio_irqs) | dma_irqs;
 790         else
 791                 host->ier = (host->ier & ~dma_irqs) | pio_irqs;
 792
 793         if (host->flags & (SDHCI_AUTO_CMD23 | SDHCI_AUTO_CMD12))
 794                 host->ier |= SDHCI_INT_AUTO_CMD_ERR;
 795         else
 796                 host->ier &= ~SDHCI_INT_AUTO_CMD_ERR;
 797
 798         sdhci_writel(host, host->ier, SDHCI_INT_ENABLE);
 799         sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE);
 800 }
 801
 802 static void sdhci_set_timeout(struct sdhci_host *host, struct mmc_command *cmd)
 803 {
 804         u8 count;
 805
 806         if (host->ops->set_timeout) {
 807                 host->ops->set_timeout(host, cmd);
 808         } else {
 809                 count = sdhci_calc_timeout(host, cmd);
 810                 sdhci_writeb(host, count, SDHCI_TIMEOUT_CONTROL);
 811         }
 812 }
 813
 814 static void sdhci_prepare_data(struct sdhci_host *host, struct mmc_command *cmd)
 815 {
 816         u8 ctrl;
 817         struct mmc_data *data = cmd->data;
 818
 819         if (sdhci_data_line_cmd(cmd))
 820                 sdhci_set_timeout(host, cmd);
 821
 822         if (!data)
 823                 return;
 824
 825         WARN_ON(host->data);
 826
 827         /* Sanity checks */
 828         BUG_ON(data->blksz * data->blocks > 524288);
 829         BUG_ON(data->blksz > host->mmc->max_blk_size);
 830         BUG_ON(data->blocks > 65535);
 831
 832         host->data = data;
 833         host->data_early = 0;
 834         host->data->bytes_xfered = 0;
 835
 836         if (host->flags & (SDHCI_USE_SDMA | SDHCI_USE_ADMA)) {
 837                 struct scatterlist *sg;
 838                 unsigned int length_mask, offset_mask;
 839                 int i;
 840
 841                 host->flags |= SDHCI_REQ_USE_DMA;
 842
 843                 /*
 844                  * FIXME: This doesn't account for merging when mapping the
 845                  * scatterlist.
 846                  *
 847                  * The assumption here being that alignment and lengths are
 848                  * the same after DMA mapping to device address space.
 849                  */
 850                 length_mask = 0;
 851                 offset_mask = 0;
 852                 if (host->flags & SDHCI_USE_ADMA) {
 853                         if (host->quirks & SDHCI_QUIRK_32BIT_ADMA_SIZE) {
 854                                 length_mask = 3;
 855                                 /*
 856                                  * As we use up to 3 byte chunks to work
 857                                  * around alignment problems, we need to
 858                                  * check the offset as well.
 859                                  */
 860                                 offset_mask = 3;
 861                         }
 862                 } else {
 863                         if (host->quirks & SDHCI_QUIRK_32BIT_DMA_SIZE)
 864                                 length_mask = 3;
 865                         if (host->quirks & SDHCI_QUIRK_32BIT_DMA_ADDR)
 866                                 offset_mask = 3;
 867                 }
 868
 869                 if (unlikely(length_mask | offset_mask)) {
 870                         for_each_sg(data->sg, sg, data->sg_len, i) {
 871                                 if (sg->length & length_mask) {
 872                                         DBG("Reverting to PIO because of transfer size (%d)\n",
 873                                             sg->length);
 874                                         host->flags &= ~SDHCI_REQ_USE_DMA;
 875                                         break;
 876                                 }
 877                                 if (sg->offset & offset_mask) {
 878                                         DBG("Reverting to PIO because of bad alignment\n");
 879                                         host->flags &= ~SDHCI_REQ_USE_DMA;
 880                                         break;
 881                                 }
 882                         }
 883                 }
 884         }
 885
 886         if (host->flags & SDHCI_REQ_USE_DMA) {
 887                 int sg_cnt = sdhci_pre_dma_transfer(host, data, COOKIE_MAPPED);
 888
 889                 if (sg_cnt <= 0) {
 890                         /*
 891                          * This only happens when someone fed
 892                          * us an invalid request.
 893                          */
 894                         WARN_ON(1);
 895                         host->flags &= ~SDHCI_REQ_USE_DMA;
 896                 } else if (host->flags & SDHCI_USE_ADMA) {
 897                         sdhci_adma_table_pre(host, data, sg_cnt);
 898
 899                         sdhci_writel(host, host->adma_addr, SDHCI_ADMA_ADDRESS);
 900                         if (host->flags & SDHCI_USE_64_BIT_DMA)
 901                                 sdhci_writel(host,
 902                                              (u64)host->adma_addr >> 32,
 903                                              SDHCI_ADMA_ADDRESS_HI);
 904                 } else {
 905                         WARN_ON(sg_cnt != 1);
 906                         sdhci_writel(host, sdhci_sdma_address(host),
 907                                      SDHCI_DMA_ADDRESS);
 908                 }
 909         }
 910
 911         /*
 912          * Always adjust the DMA selection as some controllers
 913          * (e.g. JMicron) can't do PIO properly when the selection
 914          * is ADMA.
 915          */
 916         if (host->version >= SDHCI_SPEC_200) {
 917                 ctrl = sdhci_readb(host, SDHCI_HOST_CONTROL);
 918                 ctrl &= ~SDHCI_CTRL_DMA_MASK;
 919                 if ((host->flags & SDHCI_REQ_USE_DMA) &&
 920                         (host->flags & SDHCI_USE_ADMA)) {
 921                         if (host->flags & SDHCI_USE_64_BIT_DMA)
 922                                 ctrl |= SDHCI_CTRL_ADMA64;
 923                         else
 924                                 ctrl |= SDHCI_CTRL_ADMA32;
 925                 } else {
 926                         ctrl |= SDHCI_CTRL_SDMA;
 927                 }
 928                 sdhci_writeb(host, ctrl, SDHCI_HOST_CONTROL);
 929         }
 930
 931         if (!(host->flags & SDHCI_REQ_USE_DMA)) {
 932                 int flags;
 933
 934                 flags = SG_MITER_ATOMIC;
 935                 if (host->data->flags & MMC_DATA_READ)
 936                         flags |= SG_MITER_TO_SG;
 937                 else
 938                         flags |= SG_MITER_FROM_SG;
 939                 sg_miter_start(&host->sg_miter, data->sg, data->sg_len, flags);
 940                 host->blocks = data->blocks;
 941         }
 942
 943         sdhci_set_transfer_irqs(host);
 944
 945         /* Set the DMA boundary value and block size */
 946         sdhci_writew(host, SDHCI_MAKE_BLKSZ(host->sdma_boundary, data->blksz),
 947                      SDHCI_BLOCK_SIZE);
 948         sdhci_writew(host, data->blocks, SDHCI_BLOCK_COUNT);
 949 }
 950
 951 static inline bool sdhci_auto_cmd12(struct sdhci_host *host,
 952                                     struct mmc_request *mrq)
 953 {
 954         return !mrq->sbc && (host->flags & SDHCI_AUTO_CMD12) &&
 955                !mrq->cap_cmd_during_tfr;
 956 }
 957
 958 static void sdhci_set_transfer_mode(struct sdhci_host *host,
 959         struct mmc_command *cmd)
 960 {
 961         u16 mode = 0;
 962         struct mmc_data *data = cmd->data;
 963
 964         if (data == NULL) {
 965                 if (host->quirks2 &
 966                         SDHCI_QUIRK2_CLEAR_TRANSFERMODE_REG_BEFORE_CMD) {
 967                         sdhci_writew(host, 0x0, SDHCI_TRANSFER_MODE);
 968                 } else {
 969                 /* clear Auto CMD settings for no data CMDs */
 970                         mode = sdhci_readw(host, SDHCI_TRANSFER_MODE);
 971                         sdhci_writew(host, mode & ~(SDHCI_TRNS_AUTO_CMD12 |
 972                                 SDHCI_TRNS_AUTO_CMD23), SDHCI_TRANSFER_MODE);
 973                 }
 974                 return;
 975         }
 976
 977         WARN_ON(!host->data);
 978
 979         if (!(host->quirks2 & SDHCI_QUIRK2_SUPPORT_SINGLE))
 980                 mode = SDHCI_TRNS_BLK_CNT_EN;
 981
 982         if (mmc_op_multi(cmd->opcode) || data->blocks > 1) {
 983                 mode = SDHCI_TRNS_BLK_CNT_EN | SDHCI_TRNS_MULTI;
 984                 /*
 985                  * If we are sending CMD23, CMD12 never gets sent
 986                  * on successful completion (so no Auto-CMD12).
 987                  */
 988                 if (sdhci_auto_cmd12(host, cmd->mrq) &&
 989                     (cmd->opcode != SD_IO_RW_EXTENDED))
 990                         mode |= SDHCI_TRNS_AUTO_CMD12;
 991                 else if (cmd->mrq->sbc && (host->flags & SDHCI_AUTO_CMD23)) {
 992                         mode |= SDHCI_TRNS_AUTO_CMD23;
 993                         sdhci_writel(host, cmd->mrq->sbc->arg, SDHCI_ARGUMENT2);
 994                 }
 995         }
 996
 997         if (data->flags & MMC_DATA_READ)
 998                 mode |= SDHCI_TRNS_READ;
 999         if (host->flags & SDHCI_REQ_USE_DMA)
1000                 mode |= SDHCI_TRNS_DMA;
1001
1002         sdhci_writew(host, mode, SDHCI_TRANSFER_MODE);
1003 }
1004
1005 static bool sdhci_needs_reset(struct sdhci_host *host, struct mmc_request *mrq)
1006 {
1007         return (!(host->flags & SDHCI_DEVICE_DEAD) &&
1008                 ((mrq->cmd && mrq->cmd->error) ||
1009                  (mrq->sbc && mrq->sbc->error) ||
1010                  (mrq->data && mrq->data->stop && mrq->data->stop->error) ||
1011                  (host->quirks & SDHCI_QUIRK_RESET_AFTER_REQUEST)));
1012 }
1013
1014 static void __sdhci_finish_mrq(struct sdhci_host *host, struct mmc_request *mrq)
1015 {
1016         int i;
1017
1018         for (i = 0; i < SDHCI_MAX_MRQS; i++) {
1019                 if (host->mrqs_done[i] == mrq) {
1020                         WARN_ON(1);
1021                         return;
1022                 }
1023         }
1024
1025         for (i = 0; i < SDHCI_MAX_MRQS; i++) {
1026                 if (!host->mrqs_done[i]) {
1027                         host->mrqs_done[i] = mrq;
1028                         break;
1029                 }
1030         }
1031
1032         WARN_ON(i >= SDHCI_MAX_MRQS);
1033
1034         tasklet_schedule(&host->finish_tasklet);
1035 }
1036
1037 static void sdhci_finish_mrq(struct sdhci_host *host, struct mmc_request *mrq)
1038 {
1039         if (host->cmd && host->cmd->mrq == mrq)
1040                 host->cmd = NULL;
1041
1042         if (host->data_cmd && host->data_cmd->mrq == mrq)
1043                 host->data_cmd = NULL;
1044
1045         if (host->data && host->data->mrq == mrq)
1046                 host->data = NULL;
1047
1048         if (sdhci_needs_reset(host, mrq))
1049                 host->pending_reset = true;
1050
1051         __sdhci_finish_mrq(host, mrq);
1052 }
1053
1054 static void sdhci_finish_data(struct sdhci_host *host)
1055 {
1056         struct mmc_command *data_cmd = host->data_cmd;
1057         struct mmc_data *data = host->data;
1058
1059         host->data = NULL;
1060         host->data_cmd = NULL;
1061
1062         /*
1063          * The controller needs a reset of internal state machines upon error
1064          * conditions.
1065          */
1066         if (data->error) {
1067                 if (!host->cmd || host->cmd == data_cmd)
1068                         sdhci_do_reset(host, SDHCI_RESET_CMD);
1069                 sdhci_do_reset(host, SDHCI_RESET_DATA);
1070         }
1071
1072         if ((host->flags & (SDHCI_REQ_USE_DMA | SDHCI_USE_ADMA)) ==
1073             (SDHCI_REQ_USE_DMA | SDHCI_USE_ADMA))
1074                 sdhci_adma_table_post(host, data);
1075
1076         /*
1077          * The specification states that the block count register must
1078          * be updated, but it does not specify at what point in the
1079          * data flow. That makes the register entirely useless to read
1080          * back so we have to assume that nothing made it to the card
1081          * in the event of an error.
1082          */
1083         if (data->error)
1084                 data->bytes_xfered = 0;
1085         else
1086                 data->bytes_xfered = data->blksz * data->blocks;
1087
1088         /*
1089          * Need to send CMD12 if -
1090          * a) open-ended multiblock transfer (no CMD23)
1091          * b) error in multiblock transfer
1092          */
1093         if (data->stop &&
1094             (data->error ||
1095              !data->mrq->sbc)) {
1096                 /*
1097                  * 'cap_cmd_during_tfr' request must not use the command line
1098                  * after mmc_command_done() has been called. It is upper layer's
1099                  * responsibility to send the stop command if required.
1100                  */
1101                 if (data->mrq->cap_cmd_during_tfr) {
1102                         sdhci_finish_mrq(host, data->mrq);
1103                 } else {
1104                         /* Avoid triggering warning in sdhci_send_command() */
1105                         host->cmd = NULL;
1106                         sdhci_send_command(host, data->stop);
1107                 }
1108         } else {
1109                 sdhci_finish_mrq(host, data->mrq);
1110         }
1111 }
1112
1113 static void sdhci_mod_timer(struct sdhci_host *host, struct mmc_request *mrq,
1114                             unsigned long timeout)
1115 {
1116         if (sdhci_data_line_cmd(mrq->cmd))
1117                 mod_timer(&host->data_timer, timeout);
1118         else
1119                 mod_timer(&host->timer, timeout);
1120 }
1121
1122 static void sdhci_del_timer(struct sdhci_host *host, struct mmc_request *mrq)
1123 {
1124         if (sdhci_data_line_cmd(mrq->cmd))
1125                 del_timer(&host->data_timer);
1126         else
1127                 del_timer(&host->timer);
1128 }
1129
1130 void sdhci_send_command(struct sdhci_host *host, struct mmc_command *cmd)
1131 {
1132         int flags;
1133         u32 mask;
1134         unsigned long timeout;
1135
1136         WARN_ON(host->cmd);
1137
1138         /* Initially, a command has no error */
1139         cmd->error = 0;
1140
1141         if ((host->quirks2 & SDHCI_QUIRK2_STOP_WITH_TC) &&
1142             cmd->opcode == MMC_STOP_TRANSMISSION)
1143                 cmd->flags |= MMC_RSP_BUSY;
1144
1145         /* Wait max 10 ms */
1146         timeout = 10;
1147
1148         mask = SDHCI_CMD_INHIBIT;
1149         if (sdhci_data_line_cmd(cmd))
1150                 mask |= SDHCI_DATA_INHIBIT;
1151
1152         /* We shouldn't wait for data inihibit for stop commands, even
1153            though they might use busy signaling */
1154         if (cmd->mrq->data && (cmd == cmd->mrq->data->stop))
1155                 mask &= ~SDHCI_DATA_INHIBIT;
1156
1157         while (sdhci_readl(host, SDHCI_PRESENT_STATE) & mask) {
1158                 if (timeout == 0) {
1159                         pr_err("%s: Controller never released inhibit bit(s).\n",
1160                                mmc_hostname(host->mmc));
1161                         sdhci_dumpregs(host);
1162                         cmd->error = -EIO;
1163                         sdhci_finish_mrq(host, cmd->mrq);
1164                         return;
1165                 }
1166                 timeout--;
1167                 mdelay(1);
1168         }
1169
1170         timeout = jiffies;
1171         if (!cmd->data && cmd->busy_timeout > 9000)
1172                 timeout += DIV_ROUND_UP(cmd->busy_timeout, 1000) * HZ + HZ;
1173         else
1174                 timeout += 10 * HZ;
1175         sdhci_mod_timer(host, cmd->mrq, timeout);
1176
1177         host->cmd = cmd;
1178         if (sdhci_data_line_cmd(cmd)) {
1179                 WARN_ON(host->data_cmd);
1180                 host->data_cmd = cmd;
1181         }
1182
1183         sdhci_prepare_data(host, cmd);
1184
1185         sdhci_writel(host, cmd->arg, SDHCI_ARGUMENT);
1186
1187         sdhci_set_transfer_mode(host, cmd);
1188
1189         if ((cmd->flags & MMC_RSP_136) && (cmd->flags & MMC_RSP_BUSY)) {
1190                 pr_err("%s: Unsupported response type!\n",
1191                         mmc_hostname(host->mmc));
1192                 cmd->error = -EINVAL;
1193                 sdhci_finish_mrq(host, cmd->mrq);
1194                 return;
1195         }
1196
1197         if (!(cmd->flags & MMC_RSP_PRESENT))
1198                 flags = SDHCI_CMD_RESP_NONE;
1199         else if (cmd->flags & MMC_RSP_136)
1200                 flags = SDHCI_CMD_RESP_LONG;
1201         else if (cmd->flags & MMC_RSP_BUSY)
1202                 flags = SDHCI_CMD_RESP_SHORT_BUSY;
1203         else
1204                 flags = SDHCI_CMD_RESP_SHORT;
1205
1206         if (cmd->flags & MMC_RSP_CRC)
1207                 flags |= SDHCI_CMD_CRC;
1208         if (cmd->flags & MMC_RSP_OPCODE)
1209                 flags |= SDHCI_CMD_INDEX;
1210
1211         /* CMD19 is special in that the Data Present Select should be set */
1212         if (cmd->data || cmd->opcode == MMC_SEND_TUNING_BLOCK ||
1213             cmd->opcode == MMC_SEND_TUNING_BLOCK_HS200)
1214                 flags |= SDHCI_CMD_DATA;
1215
1216         sdhci_writew(host, SDHCI_MAKE_CMD(cmd->opcode, flags), SDHCI_COMMAND);
1217 }
1218 EXPORT_SYMBOL_GPL(sdhci_send_command);
1219
1220 static void sdhci_read_rsp_136(struct sdhci_host *host, struct mmc_command *cmd)
1221 {
1222         int i, reg;
1223
1224         for (i = 0; i < 4; i++) {
1225                 reg = SDHCI_RESPONSE + (3 - i) * 4;
1226                 cmd->resp[i] = sdhci_readl(host, reg);
1227         }
1228
1229         if (host->quirks2 & SDHCI_QUIRK2_RSP_136_HAS_CRC)
1230                 return;
1231
1232         /* CRC is stripped so we need to do some shifting */
1233         for (i = 0; i < 4; i++) {
1234                 cmd->resp[i] <<= 8;
1235                 if (i != 3)
1236                         cmd->resp[i] |= cmd->resp[i + 1] >> 24;
1237         }
1238 }
1239
1240 static void sdhci_finish_command(struct sdhci_host *host)
1241 {
1242         struct mmc_command *cmd = host->cmd;
1243
1244         host->cmd = NULL;
1245
1246         if (cmd->flags & MMC_RSP_PRESENT) {
1247                 if (cmd->flags & MMC_RSP_136) {
1248                         sdhci_read_rsp_136(host, cmd);
1249                 } else {
1250                         cmd->resp[0] = sdhci_readl(host, SDHCI_RESPONSE);
1251                 }
1252         }
1253
1254         if (cmd->mrq->cap_cmd_during_tfr && cmd == cmd->mrq->cmd)
1255                 mmc_command_done(host->mmc, cmd->mrq);
1256
1257         /*
1258          * The host can send and interrupt when the busy state has
1259          * ended, allowing us to wait without wasting CPU cycles.
1260          * The busy signal uses DAT0 so this is similar to waiting
1261          * for data to complete.
1262          *
1263          * Note: The 1.0 specification is a bit ambiguous about this
1264          *       feature so there might be some problems with older
1265          *       controllers.
1266          */
1267         if (cmd->flags & MMC_RSP_BUSY) {
1268                 if (cmd->data) {
1269                         DBG("Cannot wait for busy signal when also doing a data transfer");
1270                 } else if (!(host->quirks & SDHCI_QUIRK_NO_BUSY_IRQ) &&
1271                            cmd == host->data_cmd) {
1272                         /* Command complete before busy is ended */
1273                         return;
1274                 }
1275         }
1276
1277         /* Finished CMD23, now send actual command. */
1278         if (cmd == cmd->mrq->sbc) {
1279                 sdhci_send_command(host, cmd->mrq->cmd);
1280         } else {
1281
1282                 /* Processed actual command. */
1283                 if (host->data && host->data_early)
1284                         sdhci_finish_data(host);
1285
1286                 if (!cmd->data)
1287                         sdhci_finish_mrq(host, cmd->mrq);
1288         }
1289 }
1290
1291 static u16 sdhci_get_preset_value(struct sdhci_host *host)
1292 {
1293         u16 preset = 0;
1294
1295         switch (host->timing) {
1296         case MMC_TIMING_MMC_HS:
1297         case MMC_TIMING_SD_HS:
1298                 preset = sdhci_readw(host, SDHCI_PRESET_FOR_HIGH_SPEED);
1299                 break;
1300         case MMC_TIMING_UHS_SDR12:
1301                 preset = sdhci_readw(host, SDHCI_PRESET_FOR_SDR12);
1302                 break;
1303         case MMC_TIMING_UHS_SDR25:
1304                 preset = sdhci_readw(host, SDHCI_PRESET_FOR_SDR25);
1305                 break;
1306         case MMC_TIMING_UHS_SDR50:
1307                 preset = sdhci_readw(host, SDHCI_PRESET_FOR_SDR50);
1308                 break;
1309         case MMC_TIMING_UHS_SDR104:
1310         case MMC_TIMING_MMC_HS200:
1311                 preset = sdhci_readw(host, SDHCI_PRESET_FOR_SDR104);
1312                 break;
1313         case MMC_TIMING_UHS_DDR50:
1314         case MMC_TIMING_MMC_DDR52:
1315                 preset = sdhci_readw(host, SDHCI_PRESET_FOR_DDR50);
1316                 break;
1317         case MMC_TIMING_MMC_HS400:
1318                 preset = sdhci_readw(host, SDHCI_PRESET_FOR_HS400);
1319                 break;
1320         default:
1321                 pr_warn("%s: Invalid UHS-I mode selected\n",
1322                         mmc_hostname(host->mmc));
1323                 preset = sdhci_readw(host, SDHCI_PRESET_FOR_SDR12);
1324                 break;
1325         }
1326         return preset;
1327 }
1328
1329 u16 sdhci_calc_clk(struct sdhci_host *host, unsigned int clock,
1330                    unsigned int *actual_clock)
1331 {
1332         int div = 0; /* Initialized for compiler warning */
1333         int real_div = div, clk_mul = 1;
1334         u16 clk = 0;
1335         bool switch_base_clk = false;
1336
1337         if (host->version >= SDHCI_SPEC_300) {
1338                 if (host->preset_enabled) {
1339                         u16 pre_val;
1340
1341                         clk = sdhci_readw(host, SDHCI_CLOCK_CONTROL);
1342                         pre_val = sdhci_get_preset_value(host);
1343                         div = (pre_val & SDHCI_PRESET_SDCLK_FREQ_MASK)
1344                                 >> SDHCI_PRESET_SDCLK_FREQ_SHIFT;
1345                         if (host->clk_mul &&
1346                                 (pre_val & SDHCI_PRESET_CLKGEN_SEL_MASK)) {
1347                                 clk = SDHCI_PROG_CLOCK_MODE;
1348                                 real_div = div + 1;
1349                                 clk_mul = host->clk_mul;
1350                         } else {
1351                                 real_div = max_t(int, 1, div << 1);
1352                         }
1353                         goto clock_set;
1354                 }
1355
1356                 /*
1357                  * Check if the Host Controller supports Programmable Clock
1358                  * Mode.
1359                  */
1360                 if (host->clk_mul) {
1361                         for (div = 1; div <= 1024; div++) {
1362                                 if ((host->max_clk * host->clk_mul / div)
1363                                         <= clock)
1364                                         break;
1365                         }
1366                         if ((host->max_clk * host->clk_mul / div) <= clock) {
1367                                 /*
1368                                  * Set Programmable Clock Mode in the Clock
1369                                  * Control register.
1370                                  */
1371                                 clk = SDHCI_PROG_CLOCK_MODE;
1372                                 real_div = div;
1373                                 clk_mul = host->clk_mul;
1374                                 div--;
1375                         } else {
1376                                 /*
1377                                  * Divisor can be too small to reach clock
1378                                  * speed requirement. Then use the base clock.
1379                                  */
1380                                 switch_base_clk = true;
1381                         }
1382                 }
1383
1384                 if (!host->clk_mul || switch_base_clk) {
1385                         /* Version 3.00 divisors must be a multiple of 2. */
1386                         if (host->max_clk <= clock)
1387                                 div = 1;
1388                         else {
1389                                 for (div = 2; div < SDHCI_MAX_DIV_SPEC_300;
1390                                      div += 2) {
1391                                         if ((host->max_clk / div) <= clock)
1392                                                 break;
1393                                 }
1394                         }
1395                         real_div = div;
1396                         div >>= 1;
1397                         if ((host->quirks2 & SDHCI_QUIRK2_CLOCK_DIV_ZERO_BROKEN)
1398                                 && !div && host->max_clk <= 25000000)
1399                                 div = 1;
1400                 }
1401         } else {
1402                 /* Version 2.00 divisors must be a power of 2. */
1403                 for (div = 1; div < SDHCI_MAX_DIV_SPEC_200; div *= 2) {
1404                         if ((host->max_clk / div) <= clock)
1405                                 break;
1406                 }
1407                 real_div = div;
1408                 div >>= 1;
1409         }
1410
1411 clock_set:
1412         if (real_div)
1413                 *actual_clock = (host->max_clk * clk_mul) / real_div;
1414         clk |= (div & SDHCI_DIV_MASK) << SDHCI_DIVIDER_SHIFT;
1415         clk |= ((div & SDHCI_DIV_HI_MASK) >> SDHCI_DIV_MASK_LEN)
1416                 << SDHCI_DIVIDER_HI_SHIFT;
1417
1418         return clk;
1419 }
1420 EXPORT_SYMBOL_GPL(sdhci_calc_clk);
1421
1422 void sdhci_enable_clk(struct sdhci_host *host, u16 clk)
1423 {
1424         ktime_t timeout;
1425
1426         clk |= SDHCI_CLOCK_INT_EN;
1427         sdhci_writew(host, clk, SDHCI_CLOCK_CONTROL);
1428
1429         /* Wait max 20 ms */
1430         timeout = ktime_add_ms(ktime_get(), 20);
1431         while (1) {
1432                 bool timedout = ktime_after(ktime_get(), timeout);
1433
1434                 clk = sdhci_readw(host, SDHCI_CLOCK_CONTROL);
1435                 if (clk & SDHCI_CLOCK_INT_STABLE)
1436                         break;
1437                 if (timedout) {
1438                         pr_err("%s: Internal clock never stabilised.\n",
1439                                mmc_hostname(host->mmc));
1440                         sdhci_dumpregs(host);
1441                         return;
1442                 }
1443                 udelay(10);
1444         }
1445
1446         clk |= SDHCI_CLOCK_CARD_EN;
1447         sdhci_writew(host, clk, SDHCI_CLOCK_CONTROL);
1448 }
1449 EXPORT_SYMBOL_GPL(sdhci_enable_clk);
1450
1451 void sdhci_set_clock(struct sdhci_host *host, unsigned int clock)
1452 {
1453         u16 clk;
1454
1455         host->mmc->actual_clock = 0;
1456
1457         sdhci_writew(host, 0, SDHCI_CLOCK_CONTROL);
1458
1459         if (clock == 0)
1460                 return;
1461
1462         clk = sdhci_calc_clk(host, clock, &host->mmc->actual_clock);
1463         sdhci_enable_clk(host, clk);
1464 }
1465 EXPORT_SYMBOL_GPL(sdhci_set_clock);
1466
1467 static void sdhci_set_power_reg(struct sdhci_host *host, unsigned char mode,
1468                                 unsigned short vdd)
1469 {
1470         struct mmc_host *mmc = host->mmc;
1471
1472         mmc_regulator_set_ocr(mmc, mmc->supply.vmmc, vdd);
1473
1474         if (mode != MMC_POWER_OFF)
1475                 sdhci_writeb(host, SDHCI_POWER_ON, SDHCI_POWER_CONTROL);
1476         else
1477                 sdhci_writeb(host, 0, SDHCI_POWER_CONTROL);
1478 }
1479
1480 void sdhci_set_power_noreg(struct sdhci_host *host, unsigned char mode,
1481                            unsigned short vdd)
1482 {
1483         u8 pwr = 0;
1484
1485         if (mode != MMC_POWER_OFF) {
1486                 switch (1 << vdd) {
1487                 case MMC_VDD_165_195:
1488                 /*
1489                  * Without a regulator, SDHCI does not support 2.0v
1490                  * so we only get here if the driver deliberately
1491                  * added the 2.0v range to ocr_avail. Map it to 1.8v
1492                  * for the purpose of turning on the power.
1493                  */
1494                 case MMC_VDD_20_21:
1495                         pwr = SDHCI_POWER_180;
1496                         break;
1497                 case MMC_VDD_29_30:
1498                 case MMC_VDD_30_31:
1499                         pwr = SDHCI_POWER_300;
1500                         break;
1501                 case MMC_VDD_32_33:
1502                 case MMC_VDD_33_34:
1503                 /*
1504                  * 3.4 ~ 3.6V are valid only for those platforms where it's
1505                  * known that the voltage range is supported by hardware.
1506                  */
1507                 case MMC_VDD_34_35:
1508                 case MMC_VDD_35_36:
1509                         pwr = SDHCI_POWER_330;
1510                         break;
1511                 default:
1512                         WARN(1, "%s: Invalid vdd %#x\n",
1513                              mmc_hostname(host->mmc), vdd);
1514                         break;
1515                 }
1516         }
1517
1518         if (host->pwr == pwr)
1519                 return;
1520
1521         host->pwr = pwr;
1522
1523         if (pwr == 0) {
1524                 sdhci_writeb(host, 0, SDHCI_POWER_CONTROL);
1525                 if (host->quirks2 & SDHCI_QUIRK2_CARD_ON_NEEDS_BUS_ON)
1526                         sdhci_runtime_pm_bus_off(host);
1527         } else {
1528                 /*
1529                  * Spec says that we should clear the power reg before setting
1530                  * a new value. Some controllers don't seem to like this though.
1531                  */
1532                 if (!(host->quirks & SDHCI_QUIRK_SINGLE_POWER_WRITE))
1533                         sdhci_writeb(host, 0, SDHCI_POWER_CONTROL);
1534
1535                 /*
1536                  * At least the Marvell CaFe chip gets confused if we set the
1537                  * voltage and set turn on power at the same time, so set the
1538                  * voltage first.
1539                  */
1540                 if (host->quirks & SDHCI_QUIRK_NO_SIMULT_VDD_AND_POWER)
1541                         sdhci_writeb(host, pwr, SDHCI_POWER_CONTROL);
1542
1543                 pwr |= SDHCI_POWER_ON;
1544
1545                 sdhci_writeb(host, pwr, SDHCI_POWER_CONTROL);
1546
1547                 if (host->quirks2 & SDHCI_QUIRK2_CARD_ON_NEEDS_BUS_ON)
1548                         sdhci_runtime_pm_bus_on(host);
1549
1550                 /*
1551                  * Some controllers need an extra 10ms delay of 10ms before
1552                  * they can apply clock after applying power
1553                  */
1554                 if (host->quirks & SDHCI_QUIRK_DELAY_AFTER_POWER)
1555                         mdelay(10);
1556         }
1557 }
1558 EXPORT_SYMBOL_GPL(sdhci_set_power_noreg);
1559
1560 void sdhci_set_power(struct sdhci_host *host, unsigned char mode,
1561                      unsigned short vdd)
1562 {
1563         if (IS_ERR(host->mmc->supply.vmmc))
1564                 sdhci_set_power_noreg(host, mode, vdd);
1565         else
1566                 sdhci_set_power_reg(host, mode, vdd);
1567 }
1568 EXPORT_SYMBOL_GPL(sdhci_set_power);
1569
1570 /*****************************************************************************\
1571  *                                                                           *
1572  * MMC callbacks                                                             *
1573  *                                                                           *
1574 \*****************************************************************************/
1575
1576 static void sdhci_request(struct mmc_host *mmc, struct mmc_request *mrq)
1577 {
1578         struct sdhci_host *host;
1579         int present;
1580         unsigned long flags;
1581
1582         host = mmc_priv(mmc);
1583
1584         /* Firstly check card presence */
1585         present = mmc->ops->get_cd(mmc);
1586
1587         spin_lock_irqsave(&host->lock, flags);
1588
1589         sdhci_led_activate(host);
1590
1591         /*
1592          * Ensure we don't send the STOP for non-SET_BLOCK_COUNTED
1593          * requests if Auto-CMD12 is enabled.
1594          */
1595         if (sdhci_auto_cmd12(host, mrq)) {
1596                 if (mrq->stop) {
1597                         mrq->data->stop = NULL;
1598                         mrq->stop = NULL;
1599                 }
1600         }
1601
1602         if (!present || host->flags & SDHCI_DEVICE_DEAD) {
1603                 mrq->cmd->error = -ENOMEDIUM;
1604                 sdhci_finish_mrq(host, mrq);
1605         } else {
1606                 if (mrq->sbc && !(host->flags & SDHCI_AUTO_CMD23))
1607                         sdhci_send_command(host, mrq->sbc);
1608                 else
1609                         sdhci_send_command(host, mrq->cmd);
1610         }
1611
1612         mmiowb();
1613         spin_unlock_irqrestore(&host->lock, flags);
1614 }
1615
1616 void sdhci_set_bus_width(struct sdhci_host *host, int width)
1617 {
1618         u8 ctrl;
1619
1620         ctrl = sdhci_readb(host, SDHCI_HOST_CONTROL);
1621         if (width == MMC_BUS_WIDTH_8) {
1622                 ctrl &= ~SDHCI_CTRL_4BITBUS;
1623                 ctrl |= SDHCI_CTRL_8BITBUS;
1624         } else {
1625                 if (host->mmc->caps & MMC_CAP_8_BIT_DATA)
1626                         ctrl &= ~SDHCI_CTRL_8BITBUS;
1627                 if (width == MMC_BUS_WIDTH_4)
1628                         ctrl |= SDHCI_CTRL_4BITBUS;
1629                 else
1630                         ctrl &= ~SDHCI_CTRL_4BITBUS;
1631         }
1632         sdhci_writeb(host, ctrl, SDHCI_HOST_CONTROL);
1633 }
1634 EXPORT_SYMBOL_GPL(sdhci_set_bus_width);
1635
1636 void sdhci_set_uhs_signaling(struct sdhci_host *host, unsigned timing)
1637 {
1638         u16 ctrl_2;
1639
1640         ctrl_2 = sdhci_readw(host, SDHCI_HOST_CONTROL2);
1641         /* Select Bus Speed Mode for host */
1642         ctrl_2 &= ~SDHCI_CTRL_UHS_MASK;
1643         if ((timing == MMC_TIMING_MMC_HS200) ||
1644             (timing == MMC_TIMING_UHS_SDR104))
1645                 ctrl_2 |= SDHCI_CTRL_UHS_SDR104;
1646         else if (timing == MMC_TIMING_UHS_SDR12)
1647                 ctrl_2 |= SDHCI_CTRL_UHS_SDR12;
1648         else if (timing == MMC_TIMING_UHS_SDR25)
1649                 ctrl_2 |= SDHCI_CTRL_UHS_SDR25;
1650         else if (timing == MMC_TIMING_UHS_SDR50)
1651                 ctrl_2 |= SDHCI_CTRL_UHS_SDR50;
1652         else if ((timing == MMC_TIMING_UHS_DDR50) ||
1653                  (timing == MMC_TIMING_MMC_DDR52))
1654                 ctrl_2 |= SDHCI_CTRL_UHS_DDR50;
1655         else if (timing == MMC_TIMING_MMC_HS400)
1656                 ctrl_2 |= SDHCI_CTRL_HS400; /* Non-standard */
1657         sdhci_writew(host, ctrl_2, SDHCI_HOST_CONTROL2);
1658 }
1659 EXPORT_SYMBOL_GPL(sdhci_set_uhs_signaling);
1660
1661 void sdhci_set_ios(struct mmc_host *mmc, struct mmc_ios *ios)
1662 {
1663         struct sdhci_host *host = mmc_priv(mmc);
1664         u8 ctrl;
1665
1666         if (ios->power_mode == MMC_POWER_UNDEFINED)
1667                 return;
1668
1669         if (host->flags & SDHCI_DEVICE_DEAD) {
1670                 if (!IS_ERR(mmc->supply.vmmc) &&
1671                     ios->power_mode == MMC_POWER_OFF)
1672                         mmc_regulator_set_ocr(mmc, mmc->supply.vmmc, 0);
1673                 return;
1674         }
1675
1676         /*
1677          * Reset the chip on each power off.
1678          * Should clear out any weird states.
1679          */
1680         if (ios->power_mode == MMC_POWER_OFF) {
1681                 sdhci_writel(host, 0, SDHCI_SIGNAL_ENABLE);
1682                 sdhci_reinit(host);
1683         }
1684
1685         if (host->version >= SDHCI_SPEC_300 &&
1686                 (ios->power_mode == MMC_POWER_UP) &&
1687                 !(host->quirks2 & SDHCI_QUIRK2_PRESET_VALUE_BROKEN))
1688                 sdhci_enable_preset_value(host, false);
1689
1690         if (!ios->clock || ios->clock != host->clock) {
1691                 host->ops->set_clock(host, ios->clock);
1692                 host->clock = ios->clock;
1693
1694                 if (host->quirks & SDHCI_QUIRK_DATA_TIMEOUT_USES_SDCLK &&
1695                     host->clock) {
1696                         host->timeout_clk = host->mmc->actual_clock ?
1697                                                 host->mmc->actual_clock / 1000 :
1698                                                 host->clock / 1000;
1699                         host->mmc->max_busy_timeout =
1700                                 host->ops->get_max_timeout_count ?
1701                                 host->ops->get_max_timeout_count(host) :
1702                                 1 << 27;
1703                         host->mmc->max_busy_timeout /= host->timeout_clk;
1704                 }
1705         }
1706
1707         if (host->ops->set_power)
1708                 host->ops->set_power(host, ios->power_mode, ios->vdd);
1709         else
1710                 sdhci_set_power(host, ios->power_mode, ios->vdd);
1711
1712         if (host->ops->platform_send_init_74_clocks)
1713                 host->ops->platform_send_init_74_clocks(host, ios->power_mode);
1714
1715         host->ops->set_bus_width(host, ios->bus_width);
1716
1717         ctrl = sdhci_readb(host, SDHCI_HOST_CONTROL);
1718
1719         if (!(host->quirks & SDHCI_QUIRK_NO_HISPD_BIT)) {
1720                 if (ios->timing == MMC_TIMING_SD_HS ||
1721                      ios->timing == MMC_TIMING_MMC_HS ||
1722                      ios->timing == MMC_TIMING_MMC_HS400 ||
1723                      ios->timing == MMC_TIMING_MMC_HS200 ||
1724                      ios->timing == MMC_TIMING_MMC_DDR52 ||
1725                      ios->timing == MMC_TIMING_UHS_SDR50 ||
1726                      ios->timing == MMC_TIMING_UHS_SDR104 ||
1727                      ios->timing == MMC_TIMING_UHS_DDR50 ||
1728                      ios->timing == MMC_TIMING_UHS_SDR25)
1729                         ctrl |= SDHCI_CTRL_HISPD;
1730                 else
1731                         ctrl &= ~SDHCI_CTRL_HISPD;
1732         }
1733
1734         if (host->version >= SDHCI_SPEC_300) {
1735                 u16 clk, ctrl_2;
1736
1737                 if (!host->preset_enabled) {
1738                         sdhci_writeb(host, ctrl, SDHCI_HOST_CONTROL);
1739                         /*
1740                          * We only need to set Driver Strength if the
1741                          * preset value enable is not set.
1742                          */
1743                         ctrl_2 = sdhci_readw(host, SDHCI_HOST_CONTROL2);
1744                         ctrl_2 &= ~SDHCI_CTRL_DRV_TYPE_MASK;
1745                         if (ios->drv_type == MMC_SET_DRIVER_TYPE_A)
1746                                 ctrl_2 |= SDHCI_CTRL_DRV_TYPE_A;
1747                         else if (ios->drv_type == MMC_SET_DRIVER_TYPE_B)
1748                                 ctrl_2 |= SDHCI_CTRL_DRV_TYPE_B;
1749                         else if (ios->drv_type == MMC_SET_DRIVER_TYPE_C)
1750                                 ctrl_2 |= SDHCI_CTRL_DRV_TYPE_C;
1751                         else if (ios->drv_type == MMC_SET_DRIVER_TYPE_D)
1752                                 ctrl_2 |= SDHCI_CTRL_DRV_TYPE_D;
1753                         else {
1754                                 pr_warn("%s: invalid driver type, default to driver type B\n",
1755                                         mmc_hostname(mmc));
1756                                 ctrl_2 |= SDHCI_CTRL_DRV_TYPE_B;
1757                         }
1758
1759                         sdhci_writew(host, ctrl_2, SDHCI_HOST_CONTROL2);
1760                 } else {
1761                         /*
1762                          * According to SDHC Spec v3.00, if the Preset Value
1763                          * Enable in the Host Control 2 register is set, we
1764                          * need to reset SD Clock Enable before changing High
1765                          * Speed Enable to avoid generating clock gliches.
1766                          */
1767
1768                         /* Reset SD Clock Enable */
1769                         clk = sdhci_readw(host, SDHCI_CLOCK_CONTROL);
1770                         clk &= ~SDHCI_CLOCK_CARD_EN;
1771                         sdhci_writew(host, clk, SDHCI_CLOCK_CONTROL);
1772
1773                         sdhci_writeb(host, ctrl, SDHCI_HOST_CONTROL);
1774
1775                         /* Re-enable SD Clock */
1776                         host->ops->set_clock(host, host->clock);
1777                 }
1778
1779                 /* Reset SD Clock Enable */
1780                 clk = sdhci_readw(host, SDHCI_CLOCK_CONTROL);
1781                 clk &= ~SDHCI_CLOCK_CARD_EN;
1782                 sdhci_writew(host, clk, SDHCI_CLOCK_CONTROL);
1783
1784                 host->ops->set_uhs_signaling(host, ios->timing);
1785                 host->timing = ios->timing;
1786
1787                 if (!(host->quirks2 & SDHCI_QUIRK2_PRESET_VALUE_BROKEN) &&
1788                                 ((ios->timing == MMC_TIMING_UHS_SDR12) ||
1789                                  (ios->timing == MMC_TIMING_UHS_SDR25) ||
1790                                  (ios->timing == MMC_TIMING_UHS_SDR50) ||
1791                                  (ios->timing == MMC_TIMING_UHS_SDR104) ||
1792                                  (ios->timing == MMC_TIMING_UHS_DDR50) ||
1793                                  (ios->timing == MMC_TIMING_MMC_DDR52))) {
1794                         u16 preset;
1795
1796                         sdhci_enable_preset_value(host, true);
1797                         preset = sdhci_get_preset_value(host);
1798                         ios->drv_type = (preset & SDHCI_PRESET_DRV_MASK)
1799                                 >> SDHCI_PRESET_DRV_SHIFT;
1800                 }
1801
1802                 /* Re-enable SD Clock */
1803                 host->ops->set_clock(host, host->clock);
1804         } else
1805                 sdhci_writeb(host, ctrl, SDHCI_HOST_CONTROL);
1806
1807         /*
1808          * Some (ENE) controllers go apeshit on some ios operation,
1809          * signalling timeout and CRC errors even on CMD0. Resetting
1810          * it on each ios seems to solve the problem.
1811          */
1812         if (host->quirks & SDHCI_QUIRK_RESET_CMD_DATA_ON_IOS)
1813                 sdhci_do_reset(host, SDHCI_RESET_CMD | SDHCI_RESET_DATA);
1814
1815         mmiowb();
1816 }
1817 EXPORT_SYMBOL_GPL(sdhci_set_ios);
1818
1819 static int sdhci_get_cd(struct mmc_host *mmc)
1820 {
1821         struct sdhci_host *host = mmc_priv(mmc);
1822         int gpio_cd = mmc_gpio_get_cd(mmc);
1823
1824         if (host->flags & SDHCI_DEVICE_DEAD)
1825                 return 0;
1826
1827         /* If nonremovable, assume that the card is always present. */
1828         if (!mmc_card_is_removable(host->mmc))
1829                 return 1;
1830
1831         /*
1832          * Try slot gpio detect, if defined it take precedence
1833          * over build in controller functionality
1834          */
1835         if (gpio_cd >= 0)
1836                 return !!gpio_cd;
1837
1838         /* If polling, assume that the card is always present. */
1839         if (host->quirks & SDHCI_QUIRK_BROKEN_CARD_DETECTION)
1840                 return 1;
1841
1842         /* Host native card detect */
1843         return !!(sdhci_readl(host, SDHCI_PRESENT_STATE) & SDHCI_CARD_PRESENT);
1844 }
1845
1846 static int sdhci_check_ro(struct sdhci_host *host)
1847 {
1848         unsigned long flags;
1849         int is_readonly;
1850
1851         spin_lock_irqsave(&host->lock, flags);
1852
1853         if (host->flags & SDHCI_DEVICE_DEAD)
1854                 is_readonly = 0;
1855         else if (host->ops->get_ro)
1856                 is_readonly = host->ops->get_ro(host);
1857         else
1858                 is_readonly = !(sdhci_readl(host, SDHCI_PRESENT_STATE)
1859                                 & SDHCI_WRITE_PROTECT);
1860
1861         spin_unlock_irqrestore(&host->lock, flags);
1862
1863         /* This quirk needs to be replaced by a callback-function later */
1864         return host->quirks & SDHCI_QUIRK_INVERTED_WRITE_PROTECT ?
1865                 !is_readonly : is_readonly;
1866 }
1867
1868 #define SAMPLE_COUNT    5
1869
1870 static int sdhci_get_ro(struct mmc_host *mmc)
1871 {
1872         struct sdhci_host *host = mmc_priv(mmc);
1873         int i, ro_count;
1874
1875         if (!(host->quirks & SDHCI_QUIRK_UNSTABLE_RO_DETECT))
1876                 return sdhci_check_ro(host);
1877
1878         ro_count = 0;
1879         for (i = 0; i < SAMPLE_COUNT; i++) {
1880                 if (sdhci_check_ro(host)) {
1881                         if (++ro_count > SAMPLE_COUNT / 2)
1882                                 return 1;
1883                 }
1884                 msleep(30);
1885         }
1886         return 0;
1887 }
1888
1889 static void sdhci_hw_reset(struct mmc_host *mmc)
1890 {
1891         struct sdhci_host *host = mmc_priv(mmc);
1892
1893         if (host->ops && host->ops->hw_reset)
1894                 host->ops->hw_reset(host);
1895 }
1896
1897 static void sdhci_enable_sdio_irq_nolock(struct sdhci_host *host, int enable)
1898 {
1899         if (!(host->flags & SDHCI_DEVICE_DEAD)) {
1900                 if (enable)
1901                         host->ier |= SDHCI_INT_CARD_INT;
1902                 else
1903                         host->ier &= ~SDHCI_INT_CARD_INT;
1904
1905                 sdhci_writel(host, host->ier, SDHCI_INT_ENABLE);
1906                 sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE);
1907                 mmiowb();
1908         }
1909 }
1910
1911 void sdhci_enable_sdio_irq(struct mmc_host *mmc, int enable)
1912 {
1913         struct sdhci_host *host = mmc_priv(mmc);
1914         unsigned long flags;
1915
1916         if (enable)
1917                 pm_runtime_get_noresume(host->mmc->parent);
1918
1919         spin_lock_irqsave(&host->lock, flags);
1920         if (enable)
1921                 host->flags |= SDHCI_SDIO_IRQ_ENABLED;
1922         else
1923                 host->flags &= ~SDHCI_SDIO_IRQ_ENABLED;
1924
1925         sdhci_enable_sdio_irq_nolock(host, enable);
1926         spin_unlock_irqrestore(&host->lock, flags);
1927
1928         if (!enable)
1929                 pm_runtime_put_noidle(host->mmc->parent);
1930 }
1931 EXPORT_SYMBOL_GPL(sdhci_enable_sdio_irq);
1932
1933 int sdhci_start_signal_voltage_switch(struct mmc_host *mmc,
1934                                       struct mmc_ios *ios)
1935 {
1936         struct sdhci_host *host = mmc_priv(mmc);
1937         u16 ctrl;
1938         int ret;
1939
1940         /*
1941          * Signal Voltage Switching is only applicable for Host Controllers
1942          * v3.00 and above.
1943          */
1944         if (host->version < SDHCI_SPEC_300)
1945                 return 0;
1946
1947         ctrl = sdhci_readw(host, SDHCI_HOST_CONTROL2);
1948
1949         switch (ios->signal_voltage) {
1950         case MMC_SIGNAL_VOLTAGE_330:
1951                 if (!(host->flags & SDHCI_SIGNALING_330))
1952                         return -EINVAL;
1953                 /* Set 1.8V Signal Enable in the Host Control2 register to 0 */
1954                 ctrl &= ~SDHCI_CTRL_VDD_180;
1955                 sdhci_writew(host, ctrl, SDHCI_HOST_CONTROL2);
1956
1957                 if (!IS_ERR(mmc->supply.vqmmc)) {
1958                         ret = mmc_regulator_set_vqmmc(mmc, ios);
1959                         if (ret) {
1960                                 pr_warn("%s: Switching to 3.3V signalling voltage failed\n",
1961                                         mmc_hostname(mmc));
1962                                 return -EIO;
1963                         }
1964                 }
1965                 /* Wait for 5ms */
1966                 usleep_range(5000, 5500);
1967
1968                 /* 3.3V regulator output should be stable within 5 ms */
1969                 ctrl = sdhci_readw(host, SDHCI_HOST_CONTROL2);
1970                 if (!(ctrl & SDHCI_CTRL_VDD_180))
1971                         return 0;
1972
1973                 pr_warn("%s: 3.3V regulator output did not became stable\n",
1974                         mmc_hostname(mmc));
1975
1976                 return -EAGAIN;
1977         case MMC_SIGNAL_VOLTAGE_180:
1978                 if (!(host->flags & SDHCI_SIGNALING_180))
1979                         return -EINVAL;
1980                 if (!IS_ERR(mmc->supply.vqmmc)) {
1981                         ret = mmc_regulator_set_vqmmc(mmc, ios);
1982                         if (ret) {
1983                                 pr_warn("%s: Switching to 1.8V signalling voltage failed\n",
1984                                         mmc_hostname(mmc));
1985                                 return -EIO;
1986                         }
1987                 }
1988
1989                 /*
1990                  * Enable 1.8V Signal Enable in the Host Control2
1991                  * register
1992                  */
1993                 ctrl |= SDHCI_CTRL_VDD_180;
1994                 sdhci_writew(host, ctrl, SDHCI_HOST_CONTROL2);
1995
1996                 /* Some controller need to do more when switching */
1997                 if (host->ops->voltage_switch)
1998                         host->ops->voltage_switch(host);
1999
2000                 /* 1.8V regulator output should be stable within 5 ms */
2001                 ctrl = sdhci_readw(host, SDHCI_HOST_CONTROL2);
2002                 if (ctrl & SDHCI_CTRL_VDD_180)
2003                         return 0;
2004
2005                 pr_warn("%s: 1.8V regulator output did not became stable\n",
2006                         mmc_hostname(mmc));
2007
2008                 return -EAGAIN;
2009         case MMC_SIGNAL_VOLTAGE_120:
2010                 if (!(host->flags & SDHCI_SIGNALING_120))
2011                         return -EINVAL;
2012                 if (!IS_ERR(mmc->supply.vqmmc)) {
2013                         ret = mmc_regulator_set_vqmmc(mmc, ios);
2014                         if (ret) {
2015                                 pr_warn("%s: Switching to 1.2V signalling voltage failed\n",
2016                                         mmc_hostname(mmc));
2017                                 return -EIO;
2018                         }
2019                 }
2020                 return 0;
2021         default:
2022                 /* No signal voltage switch required */
2023                 return 0;
2024         }
2025 }
2026 EXPORT_SYMBOL_GPL(sdhci_start_signal_voltage_switch);
2027
2028 static int sdhci_card_busy(struct mmc_host *mmc)
2029 {
2030         struct sdhci_host *host = mmc_priv(mmc);
2031         u32 present_state;
2032
2033         /* Check whether DAT[0] is 0 */
2034         present_state = sdhci_readl(host, SDHCI_PRESENT_STATE);
2035
2036         return !(present_state & SDHCI_DATA_0_LVL_MASK);
2037 }
2038
2039 static int sdhci_prepare_hs400_tuning(struct mmc_host *mmc, struct mmc_ios *ios)
2040 {
2041         struct sdhci_host *host = mmc_priv(mmc);
2042         unsigned long flags;
2043
2044         spin_lock_irqsave(&host->lock, flags);
2045         host->flags |= SDHCI_HS400_TUNING;
2046         spin_unlock_irqrestore(&host->lock, flags);
2047
2048         return 0;
2049 }
2050
2051 static void sdhci_start_tuning(struct sdhci_host *host)
2052 {
2053         u16 ctrl;
2054
2055         ctrl = sdhci_readw(host, SDHCI_HOST_CONTROL2);
2056         ctrl |= SDHCI_CTRL_EXEC_TUNING;
2057         if (host->quirks2 & SDHCI_QUIRK2_TUNING_WORK_AROUND)
2058                 ctrl |= SDHCI_CTRL_TUNED_CLK;
2059         sdhci_writew(host, ctrl, SDHCI_HOST_CONTROL2);
2060
2061         /*
2062          * As per the Host Controller spec v3.00, tuning command
2063          * generates Buffer Read Ready interrupt, so enable that.
2064          *
2065          * Note: The spec clearly says that when tuning sequence
2066          * is being performed, the controller does not generate
2067          * interrupts other than Buffer Read Ready interrupt. But
2068          * to make sure we don't hit a controller bug, we _only_
2069          * enable Buffer Read Ready interrupt here.
2070          */
2071         sdhci_writel(host, SDHCI_INT_DATA_AVAIL, SDHCI_INT_ENABLE);
2072         sdhci_writel(host, SDHCI_INT_DATA_AVAIL, SDHCI_SIGNAL_ENABLE);
2073 }
2074
2075 static void sdhci_end_tuning(struct sdhci_host *host)
2076 {
2077         sdhci_writel(host, host->ier, SDHCI_INT_ENABLE);
2078         sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE);
2079 }
2080
2081 static void sdhci_reset_tuning(struct sdhci_host *host)
2082 {
2083         u16 ctrl;
2084
2085         ctrl = sdhci_readw(host, SDHCI_HOST_CONTROL2);
2086         ctrl &= ~SDHCI_CTRL_TUNED_CLK;
2087         ctrl &= ~SDHCI_CTRL_EXEC_TUNING;
2088         sdhci_writew(host, ctrl, SDHCI_HOST_CONTROL2);
2089 }
2090
2091 static void sdhci_abort_tuning(struct sdhci_host *host, u32 opcode)
2092 {
2093         sdhci_reset_tuning(host);
2094
2095         sdhci_do_reset(host, SDHCI_RESET_CMD);
2096         sdhci_do_reset(host, SDHCI_RESET_DATA);
2097
2098         sdhci_end_tuning(host);
2099
2100         mmc_abort_tuning(host->mmc, opcode);
2101 }
2102
2103 /*
2104  * We use sdhci_send_tuning() because mmc_send_tuning() is not a good fit. SDHCI
2105  * tuning command does not have a data payload (or rather the hardware does it
2106  * automatically) so mmc_send_tuning() will return -EIO. Also the tuning command
2107  * interrupt setup is different to other commands and there is no timeout
2108  * interrupt so special handling is needed.
2109  */
2110 static void sdhci_send_tuning(struct sdhci_host *host, u32 opcode)
2111 {
2112         struct mmc_host *mmc = host->mmc;
2113         struct mmc_command cmd = {};
2114         struct mmc_request mrq = {};
2115         unsigned long flags;
2116         u32 b = host->sdma_boundary;
2117
2118         spin_lock_irqsave(&host->lock, flags);
2119
2120         cmd.opcode = opcode;
2121         cmd.flags = MMC_RSP_R1 | MMC_CMD_ADTC;
2122         cmd.mrq = &mrq;
2123
2124         mrq.cmd = &cmd;
2125         /*
2126          * In response to CMD19, the card sends 64 bytes of tuning
2127          * block to the Host Controller. So we set the block size
2128          * to 64 here.
2129          */
2130         if (cmd.opcode == MMC_SEND_TUNING_BLOCK_HS200 &&
2131             mmc->ios.bus_width == MMC_BUS_WIDTH_8)
2132                 sdhci_writew(host, SDHCI_MAKE_BLKSZ(b, 128), SDHCI_BLOCK_SIZE);
2133         else
2134                 sdhci_writew(host, SDHCI_MAKE_BLKSZ(b, 64), SDHCI_BLOCK_SIZE);
2135
2136         /*
2137          * The tuning block is sent by the card to the host controller.
2138          * So we set the TRNS_READ bit in the Transfer Mode register.
2139          * This also takes care of setting DMA Enable and Multi Block
2140          * Select in the same register to 0.
2141          */
2142         sdhci_writew(host, SDHCI_TRNS_READ, SDHCI_TRANSFER_MODE);
2143
2144         sdhci_send_command(host, &cmd);
2145
2146         host->cmd = NULL;
2147
2148         sdhci_del_timer(host, &mrq);
2149
2150         host->tuning_done = 0;
2151
2152         mmiowb();
2153         spin_unlock_irqrestore(&host->lock, flags);
2154
2155         /* Wait for Buffer Read Ready interrupt */
2156         wait_event_timeout(host->buf_ready_int, (host->tuning_done == 1),
2157                            msecs_to_jiffies(50));
2158
2159 }
2160
2161 static void __sdhci_execute_tuning(struct sdhci_host *host, u32 opcode)
2162 {
2163         int i;
2164
2165         /*
2166          * Issue opcode repeatedly till Execute Tuning is set to 0 or the number
2167          * of loops reaches 40 times.
2168          */
2169         for (i = 0; i < MAX_TUNING_LOOP; i++) {
2170                 u16 ctrl;
2171
2172                 sdhci_send_tuning(host, opcode);
2173
2174                 if (!host->tuning_done) {
2175                         pr_debug("%s: Tuning timeout, falling back to fixed sampling clock\n",
2176                                  mmc_hostname(host->mmc));
2177                         sdhci_abort_tuning(host, opcode);
2178                         return;
2179                 }
2180
2181                 ctrl = sdhci_readw(host, SDHCI_HOST_CONTROL2);
2182                 if (!(ctrl & SDHCI_CTRL_EXEC_TUNING)) {
2183                         if (ctrl & SDHCI_CTRL_TUNED_CLK)
2184                                 return; /* Success! */
2185                         break;
2186                 }
2187
2188                 /* Spec does not require a delay between tuning cycles */
2189                 if (host->tuning_delay > 0)
2190                         mdelay(host->tuning_delay);
2191         }
2192
2193         pr_info("%s: Tuning failed, falling back to fixed sampling clock\n",
2194                 mmc_hostname(host->mmc));
2195         sdhci_reset_tuning(host);
2196 }
2197
2198 int sdhci_execute_tuning(struct mmc_host *mmc, u32 opcode)
2199 {
2200         struct sdhci_host *host = mmc_priv(mmc);
2201         int err = 0;
2202         unsigned int tuning_count = 0;
2203         bool hs400_tuning;
2204
2205         hs400_tuning = host->flags & SDHCI_HS400_TUNING;
2206
2207         if (host->tuning_mode == SDHCI_TUNING_MODE_1)
2208                 tuning_count = host->tuning_count;
2209
2210         /*
2211          * The Host Controller needs tuning in case of SDR104 and DDR50
2212          * mode, and for SDR50 mode when Use Tuning for SDR50 is set in
2213          * the Capabilities register.
2214          * If the Host Controller supports the HS200 mode then the
2215          * tuning function has to be executed.
2216          */
2217         switch (host->timing) {
2218         /* HS400 tuning is done in HS200 mode */
2219         case MMC_TIMING_MMC_HS400:
2220                 err = -EINVAL;
2221                 goto out;
2222
2223         case MMC_TIMING_MMC_HS200:
2224                 /*
2225                  * Periodic re-tuning for HS400 is not expected to be needed, so
2226                  * disable it here.
2227                  */
2228                 if (hs400_tuning)
2229                         tuning_count = 0;
2230                 break;
2231
2232         case MMC_TIMING_UHS_SDR104:
2233         case MMC_TIMING_UHS_DDR50:
2234                 break;
2235
2236         case MMC_TIMING_UHS_SDR50:
2237                 if (host->flags & SDHCI_SDR50_NEEDS_TUNING)
2238                         break;
2239                 /* FALLTHROUGH */
2240
2241         default:
2242                 goto out;
2243         }
2244
2245         if (host->ops->platform_execute_tuning) {
2246                 err = host->ops->platform_execute_tuning(host, opcode);
2247                 goto out;
2248         }
2249
2250         host->mmc->retune_period = tuning_count;
2251
2252         if (host->tuning_delay < 0)
2253                 host->tuning_delay = opcode == MMC_SEND_TUNING_BLOCK;
2254
2255         sdhci_start_tuning(host);
2256
2257         __sdhci_execute_tuning(host, opcode);
2258
2259         sdhci_end_tuning(host);
2260 out:
2261         host->flags &= ~SDHCI_HS400_TUNING;
2262
2263         return err;
2264 }
2265 EXPORT_SYMBOL_GPL(sdhci_execute_tuning);
2266
2267 static void sdhci_enable_preset_value(struct sdhci_host *host, bool enable)
2268 {
2269         /* Host Controller v3.00 defines preset value registers */
2270         if (host->version < SDHCI_SPEC_300)
2271                 return;
2272
2273         /*
2274          * We only enable or disable Preset Value if they are not already
2275          * enabled or disabled respectively. Otherwise, we bail out.
2276          */
2277         if (host->preset_enabled != enable) {
2278                 u16 ctrl = sdhci_readw(host, SDHCI_HOST_CONTROL2);
2279
2280                 if (enable)
2281                         ctrl |= SDHCI_CTRL_PRESET_VAL_ENABLE;
2282                 else
2283                         ctrl &= ~SDHCI_CTRL_PRESET_VAL_ENABLE;
2284
2285                 sdhci_writew(host, ctrl, SDHCI_HOST_CONTROL2);
2286
2287                 if (enable)
2288                         host->flags |= SDHCI_PV_ENABLED;
2289                 else
2290                         host->flags &= ~SDHCI_PV_ENABLED;
2291
2292                 host->preset_enabled = enable;
2293         }
2294 }
2295
2296 static void sdhci_post_req(struct mmc_host *mmc, struct mmc_request *mrq,
2297                                 int err)
2298 {
2299         struct sdhci_host *host = mmc_priv(mmc);
2300         struct mmc_data *data = mrq->data;
2301
2302         if (data->host_cookie != COOKIE_UNMAPPED)
2303                 dma_unmap_sg(mmc_dev(host->mmc), data->sg, data->sg_len,
2304                              mmc_get_dma_dir(data));
2305
2306         data->host_cookie = COOKIE_UNMAPPED;
2307 }
2308
2309 static void sdhci_pre_req(struct mmc_host *mmc, struct mmc_request *mrq)
2310 {
2311         struct sdhci_host *host = mmc_priv(mmc);
2312
2313         mrq->data->host_cookie = COOKIE_UNMAPPED;
2314
2315         /*
2316          * No pre-mapping in the pre hook if we're using the bounce buffer,
2317          * for that we would need two bounce buffers since one buffer is
2318          * in flight when this is getting called.
2319          */
2320         if (host->flags & SDHCI_REQ_USE_DMA && !host->bounce_buffer)
2321                 sdhci_pre_dma_transfer(host, mrq->data, COOKIE_PRE_MAPPED);
2322 }
2323
2324 static inline bool sdhci_has_requests(struct sdhci_host *host)
2325 {
2326         return host->cmd || host->data_cmd;
2327 }
2328
2329 static void sdhci_error_out_mrqs(struct sdhci_host *host, int err)
2330 {
2331         if (host->data_cmd) {
2332                 host->data_cmd->error = err;
2333                 sdhci_finish_mrq(host, host->data_cmd->mrq);
2334         }
2335
2336         if (host->cmd) {
2337                 host->cmd->error = err;
2338                 sdhci_finish_mrq(host, host->cmd->mrq);
2339         }
2340 }
2341
2342 static void sdhci_card_event(struct mmc_host *mmc)
2343 {
2344         struct sdhci_host *host = mmc_priv(mmc);
2345         unsigned long flags;
2346         int present;
2347
2348         /* First check if client has provided their own card event */
2349         if (host->ops->card_event)
2350                 host->ops->card_event(host);
2351
2352         present = mmc->ops->get_cd(mmc);
2353
2354         spin_lock_irqsave(&host->lock, flags);
2355
2356         /* Check sdhci_has_requests() first in case we are runtime suspended */
2357         if (sdhci_has_requests(host) && !present) {
2358                 pr_err("%s: Card removed during transfer!\n",
2359                         mmc_hostname(host->mmc));
2360                 pr_err("%s: Resetting controller.\n",
2361                         mmc_hostname(host->mmc));
2362
2363                 sdhci_do_reset(host, SDHCI_RESET_CMD);
2364                 sdhci_do_reset(host, SDHCI_RESET_DATA);
2365
2366                 sdhci_error_out_mrqs(host, -ENOMEDIUM);
2367         }
2368
2369         spin_unlock_irqrestore(&host->lock, flags);
2370 }
2371
2372 static const struct mmc_host_ops sdhci_ops = {
2373         .request        = sdhci_request,
2374         .post_req       = sdhci_post_req,
2375         .pre_req        = sdhci_pre_req,
2376         .set_ios        = sdhci_set_ios,
2377         .get_cd         = sdhci_get_cd,
2378         .get_ro         = sdhci_get_ro,
2379         .hw_reset       = sdhci_hw_reset,
2380         .enable_sdio_irq = sdhci_enable_sdio_irq,
2381         .start_signal_voltage_switch    = sdhci_start_signal_voltage_switch,
2382         .prepare_hs400_tuning           = sdhci_prepare_hs400_tuning,
2383         .execute_tuning                 = sdhci_execute_tuning,
2384         .card_event                     = sdhci_card_event,
2385         .card_busy      = sdhci_card_busy,
2386 };
2387
2388 /*****************************************************************************\
2389  *                                                                           *
2390  * Tasklets                                                                  *
2391  *                                                                           *
2392 \*****************************************************************************/
2393
2394 static bool sdhci_request_done(struct sdhci_host *host)
2395 {
2396         unsigned long flags;
2397         struct mmc_request *mrq;
2398         int i;
2399
2400         spin_lock_irqsave(&host->lock, flags);
2401
2402         for (i = 0; i < SDHCI_MAX_MRQS; i++) {
2403                 mrq = host->mrqs_done[i];
2404                 if (mrq)
2405                         break;
2406         }
2407
2408         if (!mrq) {
2409                 spin_unlock_irqrestore(&host->lock, flags);
2410                 return true;
2411         }
2412
2413         sdhci_del_timer(host, mrq);
2414
2415         /*
2416          * Always unmap the data buffers if they were mapped by
2417          * sdhci_prepare_data() whenever we finish with a request.
2418          * This avoids leaking DMA mappings on error.
2419          */
2420         if (host->flags & SDHCI_REQ_USE_DMA) {
2421                 struct mmc_data *data = mrq->data;
2422
2423                 if (data && data->host_cookie == COOKIE_MAPPED) {
2424                         if (host->bounce_buffer) {
2425                                 /*
2426                                  * On reads, copy the bounced data into the
2427                                  * sglist
2428                                  */
2429                                 if (mmc_get_dma_dir(data) == DMA_FROM_DEVICE) {
2430                                         unsigned int length = data->bytes_xfered;
2431
2432                                         if (length > host->bounce_buffer_size) {
2433                                                 pr_err("%s: bounce buffer is %u bytes but DMA claims to have transferred %u bytes\n",
2434                                                        mmc_hostname(host->mmc),
2435                                                        host->bounce_buffer_size,
2436                                                        data->bytes_xfered);
2437                                                 /* Cap it down and continue */
2438                                                 length = host->bounce_buffer_size;
2439                                         }
2440                                         dma_sync_single_for_cpu(
2441                                                 host->mmc->parent,
2442                                                 host->bounce_addr,
2443                                                 host->bounce_buffer_size,
2444                                                 DMA_FROM_DEVICE);
2445                                         sg_copy_from_buffer(data->sg,
2446                                                 data->sg_len,
2447                                                 host->bounce_buffer,
2448                                                 length);
2449                                 } else {
2450                                         /* No copying, just switch ownership */
2451                                         dma_sync_single_for_cpu(
2452                                                 host->mmc->parent,
2453                                                 host->bounce_addr,
2454                                                 host->bounce_buffer_size,
2455                                                 mmc_get_dma_dir(data));
2456                                 }
2457                         } else {
2458                                 /* Unmap the raw data */
2459                                 dma_unmap_sg(mmc_dev(host->mmc), data->sg,
2460                                              data->sg_len,
2461                                              mmc_get_dma_dir(data));
2462                         }
2463                         data->host_cookie = COOKIE_UNMAPPED;
2464                 }
2465         }
2466
2467         /*
2468          * The controller needs a reset of internal state machines
2469          * upon error conditions.
2470          */
2471         if (sdhci_needs_reset(host, mrq)) {
2472                 /*
2473                  * Do not finish until command and data lines are available for
2474                  * reset. Note there can only be one other mrq, so it cannot
2475                  * also be in mrqs_done, otherwise host->cmd and host->data_cmd
2476                  * would both be null.
2477                  */
2478                 if (host->cmd || host->data_cmd) {
2479                         spin_unlock_irqrestore(&host->lock, flags);
2480                         return true;
2481                 }
2482
2483                 /* Some controllers need this kick or reset won't work here */
2484                 if (host->quirks & SDHCI_QUIRK_CLOCK_BEFORE_RESET)
2485                         /* This is to force an update */
2486                         host->ops->set_clock(host, host->clock);
2487
2488                 /* Spec says we should do both at the same time, but Ricoh
2489                    controllers do not like that. */
2490                 sdhci_do_reset(host, SDHCI_RESET_CMD);
2491                 sdhci_do_reset(host, SDHCI_RESET_DATA);
2492
2493                 host->pending_reset = false;
2494         }
2495
2496         if (!sdhci_has_requests(host))
2497                 sdhci_led_deactivate(host);
2498
2499         host->mrqs_done[i] = NULL;
2500
2501         mmiowb();
2502         spin_unlock_irqrestore(&host->lock, flags);
2503
2504         mmc_request_done(host->mmc, mrq);
2505
2506         return false;
2507 }
2508
2509 static void sdhci_tasklet_finish(unsigned long param)
2510 {
2511         struct sdhci_host *host = (struct sdhci_host *)param;
2512
2513         while (!sdhci_request_done(host))
2514                 ;
2515 }
2516
2517 static void sdhci_timeout_timer(unsigned long data)
2518 {
2519         struct sdhci_host *host;
2520         unsigned long flags;
2521
2522         host = (struct sdhci_host*)data;
2523
2524         spin_lock_irqsave(&host->lock, flags);
2525
2526         if (host->cmd && !sdhci_data_line_cmd(host->cmd)) {
2527                 pr_err("%s: Timeout waiting for hardware cmd interrupt.\n",
2528                        mmc_hostname(host->mmc));
2529                 sdhci_dumpregs(host);
2530
2531                 host->cmd->error = -ETIMEDOUT;
2532                 sdhci_finish_mrq(host, host->cmd->mrq);
2533         }
2534
2535         mmiowb();
2536         spin_unlock_irqrestore(&host->lock, flags);
2537 }
2538
2539 static void sdhci_timeout_data_timer(unsigned long data)
2540 {
2541         struct sdhci_host *host;
2542         unsigned long flags;
2543
2544         host = (struct sdhci_host *)data;
2545
2546         spin_lock_irqsave(&host->lock, flags);
2547
2548         if (host->data || host->data_cmd ||
2549             (host->cmd && sdhci_data_line_cmd(host->cmd))) {
2550                 pr_err("%s: Timeout waiting for hardware interrupt.\n",
2551                        mmc_hostname(host->mmc));
2552                 sdhci_dumpregs(host);
2553
2554                 if (host->data) {
2555                         host->data->error = -ETIMEDOUT;
2556                         sdhci_finish_data(host);
2557                 } else if (host->data_cmd) {
2558                         host->data_cmd->error = -ETIMEDOUT;
2559                         sdhci_finish_mrq(host, host->data_cmd->mrq);
2560                 } else {
2561                         host->cmd->error = -ETIMEDOUT;
2562                         sdhci_finish_mrq(host, host->cmd->mrq);
2563                 }
2564         }
2565
2566         mmiowb();
2567         spin_unlock_irqrestore(&host->lock, flags);
2568 }
2569
2570 /*****************************************************************************\
2571  *                                                                           *
2572  * Interrupt handling                                                        *
2573  *                                                                           *
2574 \*****************************************************************************/
2575
2576 static void sdhci_cmd_irq(struct sdhci_host *host, u32 intmask, u32 *intmask_p)
2577 {
2578         /* Handle auto-CMD12 error */
2579         if (intmask & SDHCI_INT_AUTO_CMD_ERR && host->data_cmd) {
2580                 struct mmc_request *mrq = host->data_cmd->mrq;
2581                 u16 auto_cmd_status = sdhci_readw(host, SDHCI_AUTO_CMD_STATUS);
2582                 int data_err_bit = (auto_cmd_status & SDHCI_AUTO_CMD_TIMEOUT) ?
2583                                    SDHCI_INT_DATA_TIMEOUT :
2584                                    SDHCI_INT_DATA_CRC;
2585
2586                 /* Treat auto-CMD12 error the same as data error */
2587                 if (!mrq->sbc && (host->flags & SDHCI_AUTO_CMD12)) {
2588                         *intmask_p |= data_err_bit;
2589                         return;
2590                 }
2591         }
2592
2593         if (!host->cmd) {
2594                 /*
2595                  * SDHCI recovers from errors by resetting the cmd and data
2596                  * circuits.  Until that is done, there very well might be more
2597                  * interrupts, so ignore them in that case.
2598                  */
2599                 if (host->pending_reset)
2600                         return;
2601                 pr_err("%s: Got command interrupt 0x%08x even though no command operation was in progress.\n",
2602                        mmc_hostname(host->mmc), (unsigned)intmask);
2603                 sdhci_dumpregs(host);
2604                 return;
2605         }
2606
2607         if (intmask & (SDHCI_INT_TIMEOUT | SDHCI_INT_CRC |
2608                        SDHCI_INT_END_BIT | SDHCI_INT_INDEX)) {
2609                 if (intmask & SDHCI_INT_TIMEOUT)
2610                         host->cmd->error = -ETIMEDOUT;
2611                 else
2612                         host->cmd->error = -EILSEQ;
2613
2614                 /* Treat data command CRC error the same as data CRC error */
2615                 if (host->cmd->data &&
2616                     (intmask & (SDHCI_INT_CRC | SDHCI_INT_TIMEOUT)) ==
2617                      SDHCI_INT_CRC) {
2618                         host->cmd = NULL;
2619                         *intmask_p |= SDHCI_INT_DATA_CRC;
2620                         return;
2621                 }
2622
2623                 sdhci_finish_mrq(host, host->cmd->mrq);
2624                 return;
2625         }
2626
2627         /* Handle auto-CMD23 error */
2628         if (intmask & SDHCI_INT_AUTO_CMD_ERR) {
2629                 struct mmc_request *mrq = host->cmd->mrq;
2630                 u16 auto_cmd_status = sdhci_readw(host, SDHCI_AUTO_CMD_STATUS);
2631                 int err = (auto_cmd_status & SDHCI_AUTO_CMD_TIMEOUT) ?
2632                           -ETIMEDOUT :
2633                           -EILSEQ;
2634
2635                 if (mrq->sbc && (host->flags & SDHCI_AUTO_CMD23)) {
2636                         mrq->sbc->error = err;
2637                         sdhci_finish_mrq(host, mrq);
2638                         return;
2639                 }
2640         }
2641
2642         if (intmask & SDHCI_INT_RESPONSE)
2643                 sdhci_finish_command(host);
2644 }
2645
2646 static void sdhci_adma_show_error(struct sdhci_host *host)
2647 {
2648         void *desc = host->adma_table;
2649         dma_addr_t dma = host->adma_addr;
2650
2651         sdhci_dumpregs(host);
2652
2653         while (true) {
2654                 struct sdhci_adma2_64_desc *dma_desc = desc;
2655
2656                 if (host->flags & SDHCI_USE_64_BIT_DMA)
2657                         SDHCI_DUMP("%08llx: DMA 0x%08x%08x, LEN 0x%04x, Attr=0x%02x\n",
2658                             (unsigned long long)dma,
2659                             le32_to_cpu(dma_desc->addr_hi),
2660                             le32_to_cpu(dma_desc->addr_lo),
2661                             le16_to_cpu(dma_desc->len),
2662                             le16_to_cpu(dma_desc->cmd));
2663                 else
2664                         SDHCI_DUMP("%08llx: DMA 0x%08x, LEN 0x%04x, Attr=0x%02x\n",
2665                             (unsigned long long)dma,
2666                             le32_to_cpu(dma_desc->addr_lo),
2667                             le16_to_cpu(dma_desc->len),
2668                             le16_to_cpu(dma_desc->cmd));
2669
2670                 desc += host->desc_sz;
2671                 dma += host->desc_sz;
2672
2673                 if (dma_desc->cmd & cpu_to_le16(ADMA2_END))
2674                         break;
2675         }
2676 }
2677
2678 static void sdhci_data_irq(struct sdhci_host *host, u32 intmask)
2679 {
2680         u32 command;
2681
2682         /* CMD19 generates _only_ Buffer Read Ready interrupt */
2683         if (intmask & SDHCI_INT_DATA_AVAIL) {
2684                 command = SDHCI_GET_CMD(sdhci_readw(host, SDHCI_COMMAND));
2685                 if (command == MMC_SEND_TUNING_BLOCK ||
2686                     command == MMC_SEND_TUNING_BLOCK_HS200) {
2687                         host->tuning_done = 1;
2688                         wake_up(&host->buf_ready_int);
2689                         return;
2690                 }
2691         }
2692
2693         if (!host->data) {
2694                 struct mmc_command *data_cmd = host->data_cmd;
2695
2696                 /*
2697                  * The "data complete" interrupt is also used to
2698                  * indicate that a busy state has ended. See comment
2699                  * above in sdhci_cmd_irq().
2700                  */
2701                 if (data_cmd && (data_cmd->flags & MMC_RSP_BUSY)) {
2702                         if (intmask & SDHCI_INT_DATA_TIMEOUT) {
2703                                 host->data_cmd = NULL;
2704                                 data_cmd->error = -ETIMEDOUT;
2705                                 sdhci_finish_mrq(host, data_cmd->mrq);
2706                                 return;
2707                         }
2708                         if (intmask & SDHCI_INT_DATA_END) {
2709                                 host->data_cmd = NULL;
2710                                 /*
2711                                  * Some cards handle busy-end interrupt
2712                                  * before the command completed, so make
2713                                  * sure we do things in the proper order.
2714                                  */
2715                                 if (host->cmd == data_cmd)
2716                                         return;
2717
2718                                 sdhci_finish_mrq(host, data_cmd->mrq);
2719                                 return;
2720                         }
2721                 }
2722
2723                 /*
2724                  * SDHCI recovers from errors by resetting the cmd and data
2725                  * circuits. Until that is done, there very well might be more
2726                  * interrupts, so ignore them in that case.
2727                  */
2728                 if (host->pending_reset)
2729                         return;
2730
2731                 pr_err("%s: Got data interrupt 0x%08x even though no data operation was in progress.\n",
2732                        mmc_hostname(host->mmc), (unsigned)intmask);
2733                 sdhci_dumpregs(host);
2734
2735                 return;
2736         }
2737
2738         if (intmask & SDHCI_INT_DATA_TIMEOUT)
2739                 host->data->error = -ETIMEDOUT;
2740         else if (intmask & SDHCI_INT_DATA_END_BIT)
2741                 host->data->error = -EILSEQ;
2742         else if ((intmask & SDHCI_INT_DATA_CRC) &&
2743                 SDHCI_GET_CMD(sdhci_readw(host, SDHCI_COMMAND))
2744                         != MMC_BUS_TEST_R)
2745                 host->data->error = -EILSEQ;
2746         else if (intmask & SDHCI_INT_ADMA_ERROR) {
2747                 pr_err("%s: ADMA error: 0x%08x\n", mmc_hostname(host->mmc),
2748                        intmask);
2749                 sdhci_adma_show_error(host);
2750                 host->data->error = -EIO;
2751                 if (host->ops->adma_workaround)
2752                         host->ops->adma_workaround(host, intmask);
2753         }
2754
2755         if (host->data->error)
2756                 sdhci_finish_data(host);
2757         else {
2758                 if (intmask & (SDHCI_INT_DATA_AVAIL | SDHCI_INT_SPACE_AVAIL))
2759                         sdhci_transfer_pio(host);
2760
2761                 /*
2762                  * We currently don't do anything fancy with DMA
2763                  * boundaries, but as we can't disable the feature
2764                  * we need to at least restart the transfer.
2765                  *
2766                  * According to the spec sdhci_readl(host, SDHCI_DMA_ADDRESS)
2767                  * should return a valid address to continue from, but as
2768                  * some controllers are faulty, don't trust them.
2769                  */
2770                 if (intmask & SDHCI_INT_DMA_END) {
2771                         u32 dmastart, dmanow;
2772
2773                         dmastart = sdhci_sdma_address(host);
2774                         dmanow = dmastart + host->data->bytes_xfered;
2775                         /*
2776                          * Force update to the next DMA block boundary.
2777                          */
2778                         dmanow = (dmanow &
2779                                 ~(SDHCI_DEFAULT_BOUNDARY_SIZE - 1)) +
2780                                 SDHCI_DEFAULT_BOUNDARY_SIZE;
2781                         host->data->bytes_xfered = dmanow - dmastart;
2782                         DBG("DMA base 0x%08x, transferred 0x%06x bytes, next 0x%08x\n",
2783                             dmastart, host->data->bytes_xfered, dmanow);
2784                         sdhci_writel(host, dmanow, SDHCI_DMA_ADDRESS);
2785                 }
2786
2787                 if (intmask & SDHCI_INT_DATA_END) {
2788                         if (host->cmd == host->data_cmd) {
2789                                 /*
2790                                  * Data managed to finish before the
2791                                  * command completed. Make sure we do
2792                                  * things in the proper order.
2793                                  */
2794                                 host->data_early = 1;
2795                         } else {
2796                                 sdhci_finish_data(host);
2797                         }
2798                 }
2799         }
2800 }
2801
2802 static irqreturn_t sdhci_irq(int irq, void *dev_id)
2803 {
2804         irqreturn_t result = IRQ_NONE;
2805         struct sdhci_host *host = dev_id;
2806         u32 intmask, mask, unexpected = 0;
2807         int max_loops = 16;
2808
2809         spin_lock(&host->lock);
2810
2811         if (host->runtime_suspended && !sdhci_sdio_irq_enabled(host)) {
2812                 spin_unlock(&host->lock);
2813                 return IRQ_NONE;
2814         }
2815
2816         intmask = sdhci_readl(host, SDHCI_INT_STATUS);
2817         if (!intmask || intmask == 0xffffffff) {
2818                 result = IRQ_NONE;
2819                 goto out;
2820         }
2821
2822         do {
2823                 DBG("IRQ status 0x%08x\n", intmask);
2824
2825                 if (host->ops->irq) {
2826                         intmask = host->ops->irq(host, intmask);
2827                         if (!intmask)
2828                                 goto cont;
2829                 }
2830
2831                 /* Clear selected interrupts. */
2832                 mask = intmask & (SDHCI_INT_CMD_MASK | SDHCI_INT_DATA_MASK |
2833                                   SDHCI_INT_BUS_POWER);
2834                 sdhci_writel(host, mask, SDHCI_INT_STATUS);
2835
2836                 if (intmask & (SDHCI_INT_CARD_INSERT | SDHCI_INT_CARD_REMOVE)) {
2837                         u32 present = sdhci_readl(host, SDHCI_PRESENT_STATE) &
2838                                       SDHCI_CARD_PRESENT;
2839
2840                         /*
2841                          * There is a observation on i.mx esdhc.  INSERT
2842                          * bit will be immediately set again when it gets
2843                          * cleared, if a card is inserted.  We have to mask
2844                          * the irq to prevent interrupt storm which will
2845                          * freeze the system.  And the REMOVE gets the
2846                          * same situation.
2847                          *
2848                          * More testing are needed here to ensure it works
2849                          * for other platforms though.
2850                          */
2851                         host->ier &= ~(SDHCI_INT_CARD_INSERT |
2852                                        SDHCI_INT_CARD_REMOVE);
2853                         host->ier |= present ? SDHCI_INT_CARD_REMOVE :
2854                                                SDHCI_INT_CARD_INSERT;
2855                         sdhci_writel(host, host->ier, SDHCI_INT_ENABLE);
2856                         sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE);
2857
2858                         sdhci_writel(host, intmask & (SDHCI_INT_CARD_INSERT |
2859                                      SDHCI_INT_CARD_REMOVE), SDHCI_INT_STATUS);
2860
2861                         host->thread_isr |= intmask & (SDHCI_INT_CARD_INSERT |
2862                                                        SDHCI_INT_CARD_REMOVE);
2863                         result = IRQ_WAKE_THREAD;
2864                 }
2865
2866                 if (intmask & SDHCI_INT_CMD_MASK)
2867                         sdhci_cmd_irq(host, intmask & SDHCI_INT_CMD_MASK, &intmask);
2868
2869                 if (intmask & SDHCI_INT_DATA_MASK)
2870                         sdhci_data_irq(host, intmask & SDHCI_INT_DATA_MASK);
2871
2872                 if (intmask & SDHCI_INT_BUS_POWER)
2873                         pr_err("%s: Card is consuming too much power!\n",
2874                                 mmc_hostname(host->mmc));
2875
2876                 if (intmask & SDHCI_INT_RETUNE)
2877                         mmc_retune_needed(host->mmc);
2878
2879                 if ((intmask & SDHCI_INT_CARD_INT) &&
2880                     (host->ier & SDHCI_INT_CARD_INT)) {
2881                         sdhci_enable_sdio_irq_nolock(host, false);
2882                         host->thread_isr |= SDHCI_INT_CARD_INT;
2883                         result = IRQ_WAKE_THREAD;
2884                 }
2885
2886                 intmask &= ~(SDHCI_INT_CARD_INSERT | SDHCI_INT_CARD_REMOVE |
2887                              SDHCI_INT_CMD_MASK | SDHCI_INT_DATA_MASK |
2888                              SDHCI_INT_ERROR | SDHCI_INT_BUS_POWER |
2889                              SDHCI_INT_RETUNE | SDHCI_INT_CARD_INT);
2890
2891                 if (intmask) {
2892                         unexpected |= intmask;
2893                         sdhci_writel(host, intmask, SDHCI_INT_STATUS);
2894                 }
2895 cont:
2896                 if (result == IRQ_NONE)
2897                         result = IRQ_HANDLED;
2898
2899                 intmask = sdhci_readl(host, SDHCI_INT_STATUS);
2900         } while (intmask && --max_loops);
2901 out:
2902         spin_unlock(&host->lock);
2903
2904         if (unexpected) {
2905                 pr_err("%s: Unexpected interrupt 0x%08x.\n",
2906                            mmc_hostname(host->mmc), unexpected);
2907                 sdhci_dumpregs(host);
2908         }
2909
2910         return result;
2911 }
2912
2913 static irqreturn_t sdhci_thread_irq(int irq, void *dev_id)
2914 {
2915         struct sdhci_host *host = dev_id;
2916         unsigned long flags;
2917         u32 isr;
2918
2919         spin_lock_irqsave(&host->lock, flags);
2920         isr = host->thread_isr;
2921         host->thread_isr = 0;
2922         spin_unlock_irqrestore(&host->lock, flags);
2923
2924         if (isr & (SDHCI_INT_CARD_INSERT | SDHCI_INT_CARD_REMOVE)) {
2925                 struct mmc_host *mmc = host->mmc;
2926
2927                 mmc->ops->card_event(mmc);
2928                 mmc_detect_change(mmc, msecs_to_jiffies(200));
2929         }
2930
2931         if (isr & SDHCI_INT_CARD_INT) {
2932                 sdio_run_irqs(host->mmc);
2933
2934                 spin_lock_irqsave(&host->lock, flags);
2935                 if (host->flags & SDHCI_SDIO_IRQ_ENABLED)
2936                         sdhci_enable_sdio_irq_nolock(host, true);
2937                 spin_unlock_irqrestore(&host->lock, flags);
2938         }
2939
2940         return isr ? IRQ_HANDLED : IRQ_NONE;
2941 }
2942
2943 /*****************************************************************************\
2944  *                                                                           *
2945  * Suspend/resume                                                            *
2946  *                                                                           *
2947 \*****************************************************************************/
2948
2949 #ifdef CONFIG_PM
2950 /*
2951  * To enable wakeup events, the corresponding events have to be enabled in
2952  * the Interrupt Status Enable register too. See 'Table 1-6: Wakeup Signal
2953  * Table' in the SD Host Controller Standard Specification.
2954  * It is useless to restore SDHCI_INT_ENABLE state in
2955  * sdhci_disable_irq_wakeups() since it will be set by
2956  * sdhci_enable_card_detection() or sdhci_init().
2957  */
2958 void sdhci_enable_irq_wakeups(struct sdhci_host *host)
2959 {
2960         u8 val;
2961         u8 mask = SDHCI_WAKE_ON_INSERT | SDHCI_WAKE_ON_REMOVE
2962                         | SDHCI_WAKE_ON_INT;
2963         u32 irq_val = SDHCI_INT_CARD_INSERT | SDHCI_INT_CARD_REMOVE |
2964                       SDHCI_INT_CARD_INT;
2965
2966         val = sdhci_readb(host, SDHCI_WAKE_UP_CONTROL);
2967         val |= mask ;
2968         /* Avoid fake wake up */
2969         if (host->quirks & SDHCI_QUIRK_BROKEN_CARD_DETECTION) {
2970                 val &= ~(SDHCI_WAKE_ON_INSERT | SDHCI_WAKE_ON_REMOVE);
2971                 irq_val &= ~(SDHCI_INT_CARD_INSERT | SDHCI_INT_CARD_REMOVE);
2972         }
2973         sdhci_writeb(host, val, SDHCI_WAKE_UP_CONTROL);
2974         sdhci_writel(host, irq_val, SDHCI_INT_ENABLE);
2975 }
2976 EXPORT_SYMBOL_GPL(sdhci_enable_irq_wakeups);
2977
2978 static void sdhci_disable_irq_wakeups(struct sdhci_host *host)
2979 {
2980         u8 val;
2981         u8 mask = SDHCI_WAKE_ON_INSERT | SDHCI_WAKE_ON_REMOVE
2982                         | SDHCI_WAKE_ON_INT;
2983
2984         val = sdhci_readb(host, SDHCI_WAKE_UP_CONTROL);
2985         val &= ~mask;
2986         sdhci_writeb(host, val, SDHCI_WAKE_UP_CONTROL);
2987 }
2988
2989 int sdhci_suspend_host(struct sdhci_host *host)
2990 {
2991         sdhci_disable_card_detection(host);
2992
2993         mmc_retune_timer_stop(host->mmc);
2994
2995         if (!device_may_wakeup(mmc_dev(host->mmc))) {
2996                 host->ier = 0;
2997                 sdhci_writel(host, 0, SDHCI_INT_ENABLE);
2998                 sdhci_writel(host, 0, SDHCI_SIGNAL_ENABLE);
2999                 free_irq(host->irq, host);
3000         } else {
3001                 sdhci_enable_irq_wakeups(host);
3002                 enable_irq_wake(host->irq);
3003         }
3004         return 0;
3005 }
3006
3007 EXPORT_SYMBOL_GPL(sdhci_suspend_host);
3008
3009 int sdhci_resume_host(struct sdhci_host *host)
3010 {
3011         struct mmc_host *mmc = host->mmc;
3012         int ret = 0;
3013
3014         if (host->flags & (SDHCI_USE_SDMA | SDHCI_USE_ADMA)) {
3015                 if (host->ops->enable_dma)
3016                         host->ops->enable_dma(host);
3017         }
3018
3019         if ((host->mmc->pm_flags & MMC_PM_KEEP_POWER) &&
3020             (host->quirks2 & SDHCI_QUIRK2_HOST_OFF_CARD_ON)) {
3021                 /* Card keeps power but host controller does not */
3022                 sdhci_init(host, 0);
3023                 host->pwr = 0;
3024                 host->clock = 0;
3025                 mmc->ops->set_ios(mmc, &mmc->ios);
3026         } else {
3027                 sdhci_init(host, (host->mmc->pm_flags & MMC_PM_KEEP_POWER));
3028                 mmiowb();
3029         }
3030
3031         if (!device_may_wakeup(mmc_dev(host->mmc))) {
3032                 ret = request_threaded_irq(host->irq, sdhci_irq,
3033                                            sdhci_thread_irq, IRQF_SHARED,
3034                                            mmc_hostname(host->mmc), host);
3035                 if (ret)
3036                         return ret;
3037         } else {
3038                 sdhci_disable_irq_wakeups(host);
3039                 disable_irq_wake(host->irq);
3040         }
3041
3042         sdhci_enable_card_detection(host);
3043
3044         return ret;
3045 }
3046
3047 EXPORT_SYMBOL_GPL(sdhci_resume_host);
3048
3049 int sdhci_runtime_suspend_host(struct sdhci_host *host)
3050 {
3051         unsigned long flags;
3052
3053         mmc_retune_timer_stop(host->mmc);
3054
3055         spin_lock_irqsave(&host->lock, flags);
3056         host->ier &= SDHCI_INT_CARD_INT;
3057         sdhci_writel(host, host->ier, SDHCI_INT_ENABLE);
3058         sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE);
3059         spin_unlock_irqrestore(&host->lock, flags);
3060
3061         synchronize_hardirq(host->irq);
3062
3063         spin_lock_irqsave(&host->lock, flags);
3064         host->runtime_suspended = true;
3065         spin_unlock_irqrestore(&host->lock, flags);
3066
3067         return 0;
3068 }
3069 EXPORT_SYMBOL_GPL(sdhci_runtime_suspend_host);
3070
3071 int sdhci_runtime_resume_host(struct sdhci_host *host)
3072 {
3073         struct mmc_host *mmc = host->mmc;
3074         unsigned long flags;
3075         int host_flags = host->flags;
3076
3077         if (host_flags & (SDHCI_USE_SDMA | SDHCI_USE_ADMA)) {
3078                 if (host->ops->enable_dma)
3079                         host->ops->enable_dma(host);
3080         }
3081
3082         sdhci_init(host, 0);
3083
3084         if (mmc->ios.power_mode != MMC_POWER_UNDEFINED &&
3085             mmc->ios.power_mode != MMC_POWER_OFF) {
3086                 /* Force clock and power re-program */
3087                 host->pwr = 0;
3088                 host->clock = 0;
3089                 mmc->ops->start_signal_voltage_switch(mmc, &mmc->ios);
3090                 mmc->ops->set_ios(mmc, &mmc->ios);
3091
3092                 if ((host_flags & SDHCI_PV_ENABLED) &&
3093                     !(host->quirks2 & SDHCI_QUIRK2_PRESET_VALUE_BROKEN)) {
3094                         spin_lock_irqsave(&host->lock, flags);
3095                         sdhci_enable_preset_value(host, true);
3096                         spin_unlock_irqrestore(&host->lock, flags);
3097                 }
3098
3099                 if ((mmc->caps2 & MMC_CAP2_HS400_ES) &&
3100                     mmc->ops->hs400_enhanced_strobe)
3101                         mmc->ops->hs400_enhanced_strobe(mmc, &mmc->ios);
3102         }
3103
3104         spin_lock_irqsave(&host->lock, flags);
3105
3106         host->runtime_suspended = false;
3107
3108         /* Enable SDIO IRQ */
3109         if (host->flags & SDHCI_SDIO_IRQ_ENABLED)
3110                 sdhci_enable_sdio_irq_nolock(host, true);
3111
3112         /* Enable Card Detection */
3113         sdhci_enable_card_detection(host);
3114
3115         spin_unlock_irqrestore(&host->lock, flags);
3116
3117         return 0;
3118 }
3119 EXPORT_SYMBOL_GPL(sdhci_runtime_resume_host);
3120
3121 #endif /* CONFIG_PM */
3122
3123 /*****************************************************************************\
3124  *                                                                           *
3125  * Command Queue Engine (CQE) helpers                                        *
3126  *                                                                           *
3127 \*****************************************************************************/
3128
3129 void sdhci_cqe_enable(struct mmc_host *mmc)
3130 {
3131         struct sdhci_host *host = mmc_priv(mmc);
3132         unsigned long flags;
3133         u8 ctrl;
3134
3135         spin_lock_irqsave(&host->lock, flags);
3136
3137         ctrl = sdhci_readb(host, SDHCI_HOST_CONTROL);
3138         ctrl &= ~SDHCI_CTRL_DMA_MASK;
3139         if (host->flags & SDHCI_USE_64_BIT_DMA)
3140                 ctrl |= SDHCI_CTRL_ADMA64;
3141         else
3142                 ctrl |= SDHCI_CTRL_ADMA32;
3143         sdhci_writeb(host, ctrl, SDHCI_HOST_CONTROL);
3144
3145         sdhci_writew(host, SDHCI_MAKE_BLKSZ(host->sdma_boundary, 512),
3146                      SDHCI_BLOCK_SIZE);
3147
3148         /* Set maximum timeout */
3149         sdhci_writeb(host, 0xE, SDHCI_TIMEOUT_CONTROL);
3150
3151         host->ier = host->cqe_ier;
3152
3153         sdhci_writel(host, host->ier, SDHCI_INT_ENABLE);
3154         sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE);
3155
3156         host->cqe_on = true;
3157
3158         pr_debug("%s: sdhci: CQE on, IRQ mask %#x, IRQ status %#x\n",
3159                  mmc_hostname(mmc), host->ier,
3160                  sdhci_readl(host, SDHCI_INT_STATUS));
3161
3162         mmiowb();
3163         spin_unlock_irqrestore(&host->lock, flags);
3164 }
3165 EXPORT_SYMBOL_GPL(sdhci_cqe_enable);
3166
3167 void sdhci_cqe_disable(struct mmc_host *mmc, bool recovery)
3168 {
3169         struct sdhci_host *host = mmc_priv(mmc);
3170         unsigned long flags;
3171
3172         spin_lock_irqsave(&host->lock, flags);
3173
3174         sdhci_set_default_irqs(host);
3175
3176         host->cqe_on = false;
3177
3178         if (recovery) {
3179                 sdhci_do_reset(host, SDHCI_RESET_CMD);
3180                 sdhci_do_reset(host, SDHCI_RESET_DATA);
3181         }
3182
3183         pr_debug("%s: sdhci: CQE off, IRQ mask %#x, IRQ status %#x\n",
3184                  mmc_hostname(mmc), host->ier,
3185                  sdhci_readl(host, SDHCI_INT_STATUS));
3186
3187         mmiowb();
3188         spin_unlock_irqrestore(&host->lock, flags);
3189 }
3190 EXPORT_SYMBOL_GPL(sdhci_cqe_disable);
3191
3192 bool sdhci_cqe_irq(struct sdhci_host *host, u32 intmask, int *cmd_error,
3193                    int *data_error)
3194 {
3195         u32 mask;
3196
3197         if (!host->cqe_on)
3198                 return false;
3199
3200         if (intmask & (SDHCI_INT_INDEX | SDHCI_INT_END_BIT | SDHCI_INT_CRC))
3201                 *cmd_error = -EILSEQ;
3202         else if (intmask & SDHCI_INT_TIMEOUT)
3203                 *cmd_error = -ETIMEDOUT;
3204         else
3205                 *cmd_error = 0;
3206
3207         if (intmask & (SDHCI_INT_DATA_END_BIT | SDHCI_INT_DATA_CRC))
3208                 *data_error = -EILSEQ;
3209         else if (intmask & SDHCI_INT_DATA_TIMEOUT)
3210                 *data_error = -ETIMEDOUT;
3211         else if (intmask & SDHCI_INT_ADMA_ERROR)
3212                 *data_error = -EIO;
3213         else
3214                 *data_error = 0;
3215
3216         /* Clear selected interrupts. */
3217         mask = intmask & host->cqe_ier;
3218         sdhci_writel(host, mask, SDHCI_INT_STATUS);
3219
3220         if (intmask & SDHCI_INT_BUS_POWER)
3221                 pr_err("%s: Card is consuming too much power!\n",
3222                        mmc_hostname(host->mmc));
3223
3224         intmask &= ~(host->cqe_ier | SDHCI_INT_ERROR);
3225         if (intmask) {
3226                 sdhci_writel(host, intmask, SDHCI_INT_STATUS);
3227                 pr_err("%s: CQE: Unexpected interrupt 0x%08x.\n",
3228                        mmc_hostname(host->mmc), intmask);
3229                 sdhci_dumpregs(host);
3230         }
3231
3232         return true;
3233 }
3234 EXPORT_SYMBOL_GPL(sdhci_cqe_irq);
3235
3236 /*****************************************************************************\
3237  *                                                                           *
3238  * Device allocation/registration                                            *
3239  *                                                                           *
3240 \*****************************************************************************/
3241
3242 struct sdhci_host *sdhci_alloc_host(struct device *dev,
3243         size_t priv_size)
3244 {
3245         struct mmc_host *mmc;
3246         struct sdhci_host *host;
3247
3248         WARN_ON(dev == NULL);
3249
3250         mmc = mmc_alloc_host(sizeof(struct sdhci_host) + priv_size, dev);
3251         if (!mmc)
3252                 return ERR_PTR(-ENOMEM);
3253
3254         host = mmc_priv(mmc);
3255         host->mmc = mmc;
3256         host->mmc_host_ops = sdhci_ops;
3257         mmc->ops = &host->mmc_host_ops;
3258
3259         host->flags = SDHCI_SIGNALING_330;
3260
3261         host->cqe_ier     = SDHCI_CQE_INT_MASK;
3262         host->cqe_err_ier = SDHCI_CQE_INT_ERR_MASK;
3263
3264         host->tuning_delay = -1;
3265
3266         host->sdma_boundary = SDHCI_DEFAULT_BOUNDARY_ARG;
3267
3268         return host;
3269 }
3270
3271 EXPORT_SYMBOL_GPL(sdhci_alloc_host);
3272
3273 static int sdhci_set_dma_mask(struct sdhci_host *host)
3274 {
3275         struct mmc_host *mmc = host->mmc;
3276         struct device *dev = mmc_dev(mmc);
3277         int ret = -EINVAL;
3278
3279         if (host->quirks2 & SDHCI_QUIRK2_BROKEN_64_BIT_DMA)
3280                 host->flags &= ~SDHCI_USE_64_BIT_DMA;
3281
3282         /* Try 64-bit mask if hardware is capable  of it */
3283         if (host->flags & SDHCI_USE_64_BIT_DMA) {
3284                 ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));
3285                 if (ret) {
3286                         pr_warn("%s: Failed to set 64-bit DMA mask.\n",
3287                                 mmc_hostname(mmc));
3288                         host->flags &= ~SDHCI_USE_64_BIT_DMA;
3289                 }
3290         }
3291
3292         /* 32-bit mask as default & fallback */
3293         if (ret) {
3294                 ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));
3295                 if (ret)
3296                         pr_warn("%s: Failed to set 32-bit DMA mask.\n",
3297                                 mmc_hostname(mmc));
3298         }
3299
3300         return ret;
3301 }
3302
3303 void __sdhci_read_caps(struct sdhci_host *host, u16 *ver, u32 *caps, u32 *caps1)
3304 {
3305         u16 v;
3306         u64 dt_caps_mask = 0;
3307         u64 dt_caps = 0;
3308
3309         if (host->read_caps)
3310                 return;
3311
3312         host->read_caps = true;
3313
3314         if (debug_quirks)
3315                 host->quirks = debug_quirks;
3316
3317         if (debug_quirks2)
3318                 host->quirks2 = debug_quirks2;
3319
3320         sdhci_do_reset(host, SDHCI_RESET_ALL);
3321
3322         of_property_read_u64(mmc_dev(host->mmc)->of_node,
3323                              "sdhci-caps-mask", &dt_caps_mask);
3324         of_property_read_u64(mmc_dev(host->mmc)->of_node,
3325                              "sdhci-caps", &dt_caps);
3326
3327         v = ver ? *ver : sdhci_readw(host, SDHCI_HOST_VERSION);
3328         host->version = (v & SDHCI_SPEC_VER_MASK) >> SDHCI_SPEC_VER_SHIFT;
3329
3330         if (host->quirks & SDHCI_QUIRK_MISSING_CAPS)
3331                 return;
3332
3333         if (caps) {
3334                 host->caps = *caps;
3335         } else {
3336                 host->caps = sdhci_readl(host, SDHCI_CAPABILITIES);
3337                 host->caps &= ~lower_32_bits(dt_caps_mask);
3338                 host->caps |= lower_32_bits(dt_caps);
3339         }
3340
3341         if (host->version < SDHCI_SPEC_300)
3342                 return;
3343
3344         if (caps1) {
3345                 host->caps1 = *caps1;
3346         } else {
3347                 host->caps1 = sdhci_readl(host, SDHCI_CAPABILITIES_1);
3348                 host->caps1 &= ~upper_32_bits(dt_caps_mask);
3349                 host->caps1 |= upper_32_bits(dt_caps);
3350         }
3351 }
3352 EXPORT_SYMBOL_GPL(__sdhci_read_caps);
3353
3354 static int sdhci_allocate_bounce_buffer(struct sdhci_host *host)
3355 {
3356         struct mmc_host *mmc = host->mmc;
3357         unsigned int max_blocks;
3358         unsigned int bounce_size;
3359         int ret;
3360
3361         /*
3362          * Cap the bounce buffer at 64KB. Using a bigger bounce buffer
3363          * has diminishing returns, this is probably because SD/MMC
3364          * cards are usually optimized to handle this size of requests.
3365          */
3366         bounce_size = SZ_64K;
3367         /*
3368          * Adjust downwards to maximum request size if this is less
3369          * than our segment size, else hammer down the maximum
3370          * request size to the maximum buffer size.
3371          */
3372         if (mmc->max_req_size < bounce_size)
3373                 bounce_size = mmc->max_req_size;
3374         max_blocks = bounce_size / 512;
3375
3376         /*
3377          * When we just support one segment, we can get significant
3378          * speedups by the help of a bounce buffer to group scattered
3379          * reads/writes together.
3380          */
3381         host->bounce_buffer = devm_kmalloc(mmc->parent,
3382                                            bounce_size,
3383                                            GFP_KERNEL);
3384         if (!host->bounce_buffer) {
3385                 pr_err("%s: failed to allocate %u bytes for bounce buffer, falling back to single segments\n",
3386                        mmc_hostname(mmc),
3387                        bounce_size);
3388                 /*
3389                  * Exiting with zero here makes sure we proceed with
3390                  * mmc->max_segs == 1.
3391                  */
3392                 return 0;
3393         }
3394
3395         host->bounce_addr = dma_map_single(mmc->parent,
3396                                            host->bounce_buffer,
3397                                            bounce_size,
3398                                            DMA_BIDIRECTIONAL);
3399         ret = dma_mapping_error(mmc->parent, host->bounce_addr);
3400         if (ret)
3401                 /* Again fall back to max_segs == 1 */
3402                 return 0;
3403         host->bounce_buffer_size = bounce_size;
3404
3405         /* Lie about this since we're bouncing */
3406         mmc->max_segs = max_blocks;
3407         mmc->max_seg_size = bounce_size;
3408         mmc->max_req_size = bounce_size;
3409
3410         pr_info("%s bounce up to %u segments into one, max segment size %u bytes\n",
3411                 mmc_hostname(mmc), max_blocks, bounce_size);
3412
3413         return 0;
3414 }
3415
3416 int sdhci_setup_host(struct sdhci_host *host)
3417 {
3418         struct mmc_host *mmc;
3419         u32 max_current_caps;
3420         unsigned int ocr_avail;
3421         unsigned int override_timeout_clk;
3422         u32 max_clk;
3423         int ret;
3424
3425         WARN_ON(host == NULL);
3426         if (host == NULL)
3427                 return -EINVAL;
3428
3429         mmc = host->mmc;
3430
3431         /*
3432          * If there are external regulators, get them. Note this must be done
3433          * early before resetting the host and reading the capabilities so that
3434          * the host can take the appropriate action if regulators are not
3435          * available.
3436          */
3437         ret = mmc_regulator_get_supply(mmc);
3438         if (ret == -EPROBE_DEFER)
3439                 return ret;
3440
3441         DBG("Version:   0x%08x | Present:  0x%08x\n",
3442             sdhci_readw(host, SDHCI_HOST_VERSION),
3443             sdhci_readl(host, SDHCI_PRESENT_STATE));
3444         DBG("Caps:      0x%08x | Caps_1:   0x%08x\n",
3445             sdhci_readl(host, SDHCI_CAPABILITIES),
3446             sdhci_readl(host, SDHCI_CAPABILITIES_1));
3447
3448         sdhci_read_caps(host);
3449
3450         override_timeout_clk = host->timeout_clk;
3451
3452         if (host->version > SDHCI_SPEC_300) {
3453                 pr_err("%s: Unknown controller version (%d). You may experience problems.\n",
3454                        mmc_hostname(mmc), host->version);
3455         }
3456
3457         if (host->quirks & SDHCI_QUIRK_FORCE_DMA)
3458                 host->flags |= SDHCI_USE_SDMA;
3459         else if (!(host->caps & SDHCI_CAN_DO_SDMA))
3460                 DBG("Controller doesn't have SDMA capability\n");
3461         else
3462                 host->flags |= SDHCI_USE_SDMA;
3463
3464         if ((host->quirks & SDHCI_QUIRK_BROKEN_DMA) &&
3465                 (host->flags & SDHCI_USE_SDMA)) {
3466                 DBG("Disabling DMA as it is marked broken\n");
3467                 host->flags &= ~SDHCI_USE_SDMA;
3468         }
3469
3470         if ((host->version >= SDHCI_SPEC_200) &&
3471                 (host->caps & SDHCI_CAN_DO_ADMA2))
3472                 host->flags |= SDHCI_USE_ADMA;
3473
3474         if ((host->quirks & SDHCI_QUIRK_BROKEN_ADMA) &&
3475                 (host->flags & SDHCI_USE_ADMA)) {
3476                 DBG("Disabling ADMA as it is marked broken\n");
3477                 host->flags &= ~SDHCI_USE_ADMA;
3478         }
3479
3480         /*
3481          * It is assumed that a 64-bit capable device has set a 64-bit DMA mask
3482          * and *must* do 64-bit DMA.  A driver has the opportunity to change
3483          * that during the first call to ->enable_dma().  Similarly
3484          * SDHCI_QUIRK2_BROKEN_64_BIT_DMA must be left to the drivers to
3485          * implement.
3486          */
3487         if (host->caps & SDHCI_CAN_64BIT)
3488                 host->flags |= SDHCI_USE_64_BIT_DMA;
3489
3490         if (host->flags & (SDHCI_USE_SDMA | SDHCI_USE_ADMA)) {
3491                 ret = sdhci_set_dma_mask(host);
3492
3493                 if (!ret && host->ops->enable_dma)
3494                         ret = host->ops->enable_dma(host);
3495
3496                 if (ret) {
3497                         pr_warn("%s: No suitable DMA available - falling back to PIO\n",
3498                                 mmc_hostname(mmc));
3499                         host->flags &= ~(SDHCI_USE_SDMA | SDHCI_USE_ADMA);
3500
3501                         ret = 0;
3502                 }
3503         }
3504
3505         /* SDMA does not support 64-bit DMA */
3506         if (host->flags & SDHCI_USE_64_BIT_DMA)
3507                 host->flags &= ~SDHCI_USE_SDMA;
3508
3509         if (host->flags & SDHCI_USE_ADMA) {
3510                 dma_addr_t dma;
3511                 void *buf;
3512
3513                 /*
3514                  * The DMA descriptor table size is calculated as the maximum
3515                  * number of segments times 2, to allow for an alignment
3516                  * descriptor for each segment, plus 1 for a nop end descriptor,
3517                  * all multipled by the descriptor size.
3518                  */
3519                 if (host->flags & SDHCI_USE_64_BIT_DMA) {
3520                         host->adma_table_sz = (SDHCI_MAX_SEGS * 2 + 1) *
3521                                               SDHCI_ADMA2_64_DESC_SZ;
3522                         host->desc_sz = SDHCI_ADMA2_64_DESC_SZ;
3523                 } else {
3524                         host->adma_table_sz = (SDHCI_MAX_SEGS * 2 + 1) *
3525                                               SDHCI_ADMA2_32_DESC_SZ;
3526                         host->desc_sz = SDHCI_ADMA2_32_DESC_SZ;
3527                 }
3528
3529                 host->align_buffer_sz = SDHCI_MAX_SEGS * SDHCI_ADMA2_ALIGN;
3530                 buf = dma_alloc_coherent(mmc_dev(mmc), host->align_buffer_sz +
3531                                          host->adma_table_sz, &dma, GFP_KERNEL);
3532                 if (!buf) {
3533                         pr_warn("%s: Unable to allocate ADMA buffers - falling back to standard DMA\n",
3534                                 mmc_hostname(mmc));
3535                         host->flags &= ~SDHCI_USE_ADMA;
3536                 } else if ((dma + host->align_buffer_sz) &
3537                            (SDHCI_ADMA2_DESC_ALIGN - 1)) {
3538                         pr_warn("%s: unable to allocate aligned ADMA descriptor\n",
3539                                 mmc_hostname(mmc));
3540                         host->flags &= ~SDHCI_USE_ADMA;
3541                         dma_free_coherent(mmc_dev(mmc), host->align_buffer_sz +
3542                                           host->adma_table_sz, buf, dma);
3543                 } else {
3544                         host->align_buffer = buf;
3545                         host->align_addr = dma;
3546
3547                         host->adma_table = buf + host->align_buffer_sz;
3548                         host->adma_addr = dma + host->align_buffer_sz;
3549                 }
3550         }
3551
3552         /*
3553          * If we use DMA, then it's up to the caller to set the DMA
3554          * mask, but PIO does not need the hw shim so we set a new
3555          * mask here in that case.
3556          */
3557         if (!(host->flags & (SDHCI_USE_SDMA | SDHCI_USE_ADMA))) {
3558                 host->dma_mask = DMA_BIT_MASK(64);
3559                 mmc_dev(mmc)->dma_mask = &host->dma_mask;
3560         }
3561
3562         if (host->version >= SDHCI_SPEC_300)
3563                 host->max_clk = (host->caps & SDHCI_CLOCK_V3_BASE_MASK)
3564                         >> SDHCI_CLOCK_BASE_SHIFT;
3565         else
3566                 host->max_clk = (host->caps & SDHCI_CLOCK_BASE_MASK)
3567                         >> SDHCI_CLOCK_BASE_SHIFT;
3568
3569         host->max_clk *= 1000000;
3570         if (host->max_clk == 0 || host->quirks &
3571                         SDHCI_QUIRK_CAP_CLOCK_BASE_BROKEN) {
3572                 if (!host->ops->get_max_clock) {
3573                         pr_err("%s: Hardware doesn't specify base clock frequency.\n",
3574                                mmc_hostname(mmc));
3575                         ret = -ENODEV;
3576                         goto undma;
3577                 }
3578                 host->max_clk = host->ops->get_max_clock(host);
3579         }
3580
3581         /*
3582          * In case of Host Controller v3.00, find out whether clock
3583          * multiplier is supported.
3584          */
3585         host->clk_mul = (host->caps1 & SDHCI_CLOCK_MUL_MASK) >>
3586                         SDHCI_CLOCK_MUL_SHIFT;
3587
3588         /*
3589          * In case the value in Clock Multiplier is 0, then programmable
3590          * clock mode is not supported, otherwise the actual clock
3591          * multiplier is one more than the value of Clock Multiplier
3592          * in the Capabilities Register.
3593          */
3594         if (host->clk_mul)
3595                 host->clk_mul += 1;
3596
3597         /*
3598          * Set host parameters.
3599          */
3600         max_clk = host->max_clk;
3601
3602         if (host->ops->get_min_clock)
3603                 mmc->f_min = host->ops->get_min_clock(host);
3604         else if (host->version >= SDHCI_SPEC_300) {
3605                 if (host->clk_mul)
3606                         max_clk = host->max_clk * host->clk_mul;
3607                 /*
3608                  * Divided Clock Mode minimum clock rate is always less than
3609                  * Programmable Clock Mode minimum clock rate.
3610                  */
3611                 mmc->f_min = host->max_clk / SDHCI_MAX_DIV_SPEC_300;
3612         } else
3613                 mmc->f_min = host->max_clk / SDHCI_MAX_DIV_SPEC_200;
3614
3615         if (!mmc->f_max || mmc->f_max > max_clk)
3616                 mmc->f_max = max_clk;
3617
3618         if (!(host->quirks & SDHCI_QUIRK_DATA_TIMEOUT_USES_SDCLK)) {
3619                 host->timeout_clk = (host->caps & SDHCI_TIMEOUT_CLK_MASK) >>
3620                                         SDHCI_TIMEOUT_CLK_SHIFT;
3621
3622                 if (host->caps & SDHCI_TIMEOUT_CLK_UNIT)
3623                         host->timeout_clk *= 1000;
3624
3625                 if (host->timeout_clk == 0) {
3626                         if (!host->ops->get_timeout_clock) {
3627                                 pr_err("%s: Hardware doesn't specify timeout clock frequency.\n",
3628                                         mmc_hostname(mmc));
3629                                 ret = -ENODEV;
3630                                 goto undma;
3631                         }
3632
3633                         host->timeout_clk =
3634                                 DIV_ROUND_UP(host->ops->get_timeout_clock(host),
3635                                              1000);
3636                 }
3637
3638                 if (override_timeout_clk)
3639                         host->timeout_clk = override_timeout_clk;
3640
3641                 mmc->max_busy_timeout = host->ops->get_max_timeout_count ?
3642                         host->ops->get_max_timeout_count(host) : 1 << 27;
3643                 mmc->max_busy_timeout /= host->timeout_clk;
3644         }
3645
3646         mmc->caps |= MMC_CAP_SDIO_IRQ | MMC_CAP_ERASE | MMC_CAP_CMD23;
3647         mmc->caps2 |= MMC_CAP2_SDIO_IRQ_NOTHREAD;
3648
3649         if (host->quirks & SDHCI_QUIRK_MULTIBLOCK_READ_ACMD12)
3650                 host->flags |= SDHCI_AUTO_CMD12;
3651
3652         /* Auto-CMD23 stuff only works in ADMA or PIO. */
3653         if ((host->version >= SDHCI_SPEC_300) &&
3654             ((host->flags & SDHCI_USE_ADMA) ||
3655              !(host->flags & SDHCI_USE_SDMA)) &&
3656              !(host->quirks2 & SDHCI_QUIRK2_ACMD23_BROKEN)) {
3657                 host->flags |= SDHCI_AUTO_CMD23;
3658                 DBG("Auto-CMD23 available\n");
3659         } else {
3660                 DBG("Auto-CMD23 unavailable\n");
3661         }
3662
3663         /*
3664          * A controller may support 8-bit width, but the board itself
3665          * might not have the pins brought out.  Boards that support
3666          * 8-bit width must set "mmc->caps |= MMC_CAP_8_BIT_DATA;" in
3667          * their platform code before calling sdhci_add_host(), and we
3668          * won't assume 8-bit width for hosts without that CAP.
3669          */
3670         if (!(host->quirks & SDHCI_QUIRK_FORCE_1_BIT_DATA))
3671                 mmc->caps |= MMC_CAP_4_BIT_DATA;
3672
3673         if (host->quirks2 & SDHCI_QUIRK2_HOST_NO_CMD23)
3674                 mmc->caps &= ~MMC_CAP_CMD23;
3675
3676         if (host->caps & SDHCI_CAN_DO_HISPD)
3677                 mmc->caps |= MMC_CAP_SD_HIGHSPEED | MMC_CAP_MMC_HIGHSPEED;
3678
3679         if ((host->quirks & SDHCI_QUIRK_BROKEN_CARD_DETECTION) &&
3680             mmc_card_is_removable(mmc) &&
3681             mmc_gpio_get_cd(host->mmc) < 0)
3682                 mmc->caps |= MMC_CAP_NEEDS_POLL;
3683
3684         if (!IS_ERR(mmc->supply.vqmmc)) {
3685                 ret = regulator_enable(mmc->supply.vqmmc);
3686
3687                 /* If vqmmc provides no 1.8V signalling, then there's no UHS */
3688                 if (!regulator_is_supported_voltage(mmc->supply.vqmmc, 1700000,
3689                                                     1950000))
3690                         host->caps1 &= ~(SDHCI_SUPPORT_SDR104 |
3691                                          SDHCI_SUPPORT_SDR50 |
3692                                          SDHCI_SUPPORT_DDR50);
3693
3694                 /* In eMMC case vqmmc might be a fixed 1.8V regulator */
3695                 if (!regulator_is_supported_voltage(mmc->supply.vqmmc, 2700000,
3696                                                     3600000))
3697                         host->flags &= ~SDHCI_SIGNALING_330;
3698
3699                 if (ret) {
3700                         pr_warn("%s: Failed to enable vqmmc regulator: %d\n",
3701                                 mmc_hostname(mmc), ret);
3702                         mmc->supply.vqmmc = ERR_PTR(-EINVAL);
3703                 }
3704         }
3705
3706         if (host->quirks2 & SDHCI_QUIRK2_NO_1_8_V) {
3707                 host->caps1 &= ~(SDHCI_SUPPORT_SDR104 | SDHCI_SUPPORT_SDR50 |
3708                                  SDHCI_SUPPORT_DDR50);
3709         }
3710
3711         /* Any UHS-I mode in caps implies SDR12 and SDR25 support. */
3712         if (host->caps1 & (SDHCI_SUPPORT_SDR104 | SDHCI_SUPPORT_SDR50 |
3713                            SDHCI_SUPPORT_DDR50))
3714                 mmc->caps |= MMC_CAP_UHS_SDR12 | MMC_CAP_UHS_SDR25;
3715
3716         /* SDR104 supports also implies SDR50 support */
3717         if (host->caps1 & SDHCI_SUPPORT_SDR104) {
3718                 mmc->caps |= MMC_CAP_UHS_SDR104 | MMC_CAP_UHS_SDR50;
3719                 /* SD3.0: SDR104 is supported so (for eMMC) the caps2
3720                  * field can be promoted to support HS200.
3721                  */
3722                 if (!(host->quirks2 & SDHCI_QUIRK2_BROKEN_HS200))
3723                         mmc->caps2 |= MMC_CAP2_HS200;
3724         } else if (host->caps1 & SDHCI_SUPPORT_SDR50) {
3725                 mmc->caps |= MMC_CAP_UHS_SDR50;
3726         }
3727
3728         if (host->quirks2 & SDHCI_QUIRK2_CAPS_BIT63_FOR_HS400 &&
3729             (host->caps1 & SDHCI_SUPPORT_HS400))
3730                 mmc->caps2 |= MMC_CAP2_HS400;
3731
3732         if ((mmc->caps2 & MMC_CAP2_HSX00_1_2V) &&
3733             (IS_ERR(mmc->supply.vqmmc) ||
3734              !regulator_is_supported_voltage(mmc->supply.vqmmc, 1100000,
3735                                              1300000)))
3736                 mmc->caps2 &= ~MMC_CAP2_HSX00_1_2V;
3737
3738         if ((host->caps1 & SDHCI_SUPPORT_DDR50) &&
3739             !(host->quirks2 & SDHCI_QUIRK2_BROKEN_DDR50))
3740                 mmc->caps |= MMC_CAP_UHS_DDR50;
3741
3742         /* Does the host need tuning for SDR50? */
3743         if (host->caps1 & SDHCI_USE_SDR50_TUNING)
3744                 host->flags |= SDHCI_SDR50_NEEDS_TUNING;
3745
3746         /* Driver Type(s) (A, C, D) supported by the host */
3747         if (host->caps1 & SDHCI_DRIVER_TYPE_A)
3748                 mmc->caps |= MMC_CAP_DRIVER_TYPE_A;
3749         if (host->caps1 & SDHCI_DRIVER_TYPE_C)
3750                 mmc->caps |= MMC_CAP_DRIVER_TYPE_C;
3751         if (host->caps1 & SDHCI_DRIVER_TYPE_D)
3752                 mmc->caps |= MMC_CAP_DRIVER_TYPE_D;
3753
3754         /* Initial value for re-tuning timer count */
3755         host->tuning_count = (host->caps1 & SDHCI_RETUNING_TIMER_COUNT_MASK) >>
3756                              SDHCI_RETUNING_TIMER_COUNT_SHIFT;
3757
3758         /*
3759          * In case Re-tuning Timer is not disabled, the actual value of
3760          * re-tuning timer will be 2 ^ (n - 1).
3761          */
3762         if (host->tuning_count)
3763                 host->tuning_count = 1 << (host->tuning_count - 1);
3764
3765         /* Re-tuning mode supported by the Host Controller */
3766         host->tuning_mode = (host->caps1 & SDHCI_RETUNING_MODE_MASK) >>
3767                              SDHCI_RETUNING_MODE_SHIFT;
3768
3769         ocr_avail = 0;
3770
3771         /*
3772          * According to SD Host Controller spec v3.00, if the Host System
3773          * can afford more than 150mA, Host Driver should set XPC to 1. Also
3774          * the value is meaningful only if Voltage Support in the Capabilities
3775          * register is set. The actual current value is 4 times the register
3776          * value.
3777          */
3778         max_current_caps = sdhci_readl(host, SDHCI_MAX_CURRENT);
3779         if (!max_current_caps && !IS_ERR(mmc->supply.vmmc)) {
3780                 int curr = regulator_get_current_limit(mmc->supply.vmmc);
3781                 if (curr > 0) {
3782
3783                         /* convert to SDHCI_MAX_CURRENT format */
3784                         curr = curr/1000;  /* convert to mA */
3785                         curr = curr/SDHCI_MAX_CURRENT_MULTIPLIER;
3786
3787                         curr = min_t(u32, curr, SDHCI_MAX_CURRENT_LIMIT);
3788                         max_current_caps =
3789                                 (curr << SDHCI_MAX_CURRENT_330_SHIFT) |
3790                                 (curr << SDHCI_MAX_CURRENT_300_SHIFT) |
3791                                 (curr << SDHCI_MAX_CURRENT_180_SHIFT);
3792                 }
3793         }
3794
3795         if (host->caps & SDHCI_CAN_VDD_330) {
3796                 ocr_avail |= MMC_VDD_32_33 | MMC_VDD_33_34;
3797
3798                 mmc->max_current_330 = ((max_current_caps &
3799                                    SDHCI_MAX_CURRENT_330_MASK) >>
3800                                    SDHCI_MAX_CURRENT_330_SHIFT) *
3801                                    SDHCI_MAX_CURRENT_MULTIPLIER;
3802         }
3803         if (host->caps & SDHCI_CAN_VDD_300) {
3804                 ocr_avail |= MMC_VDD_29_30 | MMC_VDD_30_31;
3805
3806                 mmc->max_current_300 = ((max_current_caps &
3807                                    SDHCI_MAX_CURRENT_300_MASK) >>
3808                                    SDHCI_MAX_CURRENT_300_SHIFT) *
3809                                    SDHCI_MAX_CURRENT_MULTIPLIER;
3810         }
3811         if (host->caps & SDHCI_CAN_VDD_180) {
3812                 ocr_avail |= MMC_VDD_165_195;
3813
3814                 mmc->max_current_180 = ((max_current_caps &
3815                                    SDHCI_MAX_CURRENT_180_MASK) >>
3816                                    SDHCI_MAX_CURRENT_180_SHIFT) *
3817                                    SDHCI_MAX_CURRENT_MULTIPLIER;
3818         }
3819
3820         /* If OCR set by host, use it instead. */
3821         if (host->ocr_mask)
3822                 ocr_avail = host->ocr_mask;
3823
3824         /* If OCR set by external regulators, give it highest prio. */
3825         if (mmc->ocr_avail)
3826                 ocr_avail = mmc->ocr_avail;
3827
3828         mmc->ocr_avail = ocr_avail;
3829         mmc->ocr_avail_sdio = ocr_avail;
3830         if (host->ocr_avail_sdio)
3831                 mmc->ocr_avail_sdio &= host->ocr_avail_sdio;
3832         mmc->ocr_avail_sd = ocr_avail;
3833         if (host->ocr_avail_sd)
3834                 mmc->ocr_avail_sd &= host->ocr_avail_sd;
3835         else /* normal SD controllers don't support 1.8V */
3836                 mmc->ocr_avail_sd &= ~MMC_VDD_165_195;
3837         mmc->ocr_avail_mmc = ocr_avail;
3838         if (host->ocr_avail_mmc)
3839                 mmc->ocr_avail_mmc &= host->ocr_avail_mmc;
3840
3841         if (mmc->ocr_avail == 0) {
3842                 pr_err("%s: Hardware doesn't report any support voltages.\n",
3843                        mmc_hostname(mmc));
3844                 ret = -ENODEV;
3845                 goto unreg;
3846         }
3847
3848         if ((mmc->caps & (MMC_CAP_UHS_SDR12 | MMC_CAP_UHS_SDR25 |
3849                           MMC_CAP_UHS_SDR50 | MMC_CAP_UHS_SDR104 |
3850                           MMC_CAP_UHS_DDR50 | MMC_CAP_1_8V_DDR)) ||
3851             (mmc->caps2 & (MMC_CAP2_HS200_1_8V_SDR | MMC_CAP2_HS400_1_8V)))
3852                 host->flags |= SDHCI_SIGNALING_180;
3853
3854         if (mmc->caps2 & MMC_CAP2_HSX00_1_2V)
3855                 host->flags |= SDHCI_SIGNALING_120;
3856
3857         spin_lock_init(&host->lock);
3858
3859         /*
3860          * Maximum number of sectors in one transfer. Limited by SDMA boundary
3861          * size (512KiB). Note some tuning modes impose a 4MiB limit, but this
3862          * is less anyway.
3863          */
3864         mmc->max_req_size = 524288;
3865
3866         /*
3867          * Maximum number of segments. Depends on if the hardware
3868          * can do scatter/gather or not.
3869          */
3870         if (host->flags & SDHCI_USE_ADMA) {
3871                 mmc->max_segs = SDHCI_MAX_SEGS;
3872         } else if (host->flags & SDHCI_USE_SDMA) {
3873                 mmc->max_segs = 1;
3874                 if (swiotlb_max_segment()) {
3875                         unsigned int max_req_size = (1 << IO_TLB_SHIFT) *
3876                                                 IO_TLB_SEGSIZE;
3877                         mmc->max_req_size = min(mmc->max_req_size,
3878                                                 max_req_size);
3879                 }
3880         } else { /* PIO */
3881                 mmc->max_segs = SDHCI_MAX_SEGS;
3882         }
3883
3884         /*
3885          * Maximum segment size. Could be one segment with the maximum number
3886          * of bytes. When doing hardware scatter/gather, each entry cannot
3887          * be larger than 64 KiB though.
3888          */
3889         if (host->flags & SDHCI_USE_ADMA) {
3890                 if (host->quirks & SDHCI_QUIRK_BROKEN_ADMA_ZEROLEN_DESC)
3891                         mmc->max_seg_size = 65535;
3892                 else
3893                         mmc->max_seg_size = 65536;
3894         } else {
3895                 mmc->max_seg_size = mmc->max_req_size;
3896         }
3897
3898         /*
3899          * Maximum block size. This varies from controller to controller and
3900          * is specified in the capabilities register.
3901          */
3902         if (host->quirks & SDHCI_QUIRK_FORCE_BLK_SZ_2048) {
3903                 mmc->max_blk_size = 2;
3904         } else {
3905                 mmc->max_blk_size = (host->caps & SDHCI_MAX_BLOCK_MASK) >>
3906                                 SDHCI_MAX_BLOCK_SHIFT;
3907                 if (mmc->max_blk_size >= 3) {
3908                         pr_warn("%s: Invalid maximum block size, assuming 512 bytes\n",
3909                                 mmc_hostname(mmc));
3910                         mmc->max_blk_size = 0;
3911                 }
3912         }
3913
3914         mmc->max_blk_size = 512 << mmc->max_blk_size;
3915
3916         /*
3917          * Maximum block count.
3918          */
3919         mmc->max_blk_count = (host->quirks & SDHCI_QUIRK_NO_MULTIBLOCK) ? 1 : 65535;
3920
3921         if (mmc->max_segs == 1) {
3922                 /* This may alter mmc->*_blk_* parameters */
3923                 ret = sdhci_allocate_bounce_buffer(host);
3924                 if (ret)
3925                         return ret;
3926         }
3927
3928         return 0;
3929
3930 unreg:
3931         if (!IS_ERR(mmc->supply.vqmmc))
3932                 regulator_disable(mmc->supply.vqmmc);
3933 undma:
3934         if (host->align_buffer)
3935                 dma_free_coherent(mmc_dev(mmc), host->align_buffer_sz +
3936                                   host->adma_table_sz, host->align_buffer,
3937                                   host->align_addr);
3938         host->adma_table = NULL;
3939         host->align_buffer = NULL;
3940
3941         return ret;
3942 }
3943 EXPORT_SYMBOL_GPL(sdhci_setup_host);
3944
3945 void sdhci_cleanup_host(struct sdhci_host *host)
3946 {
3947         struct mmc_host *mmc = host->mmc;
3948
3949         if (!IS_ERR(mmc->supply.vqmmc))
3950                 regulator_disable(mmc->supply.vqmmc);
3951
3952         if (host->align_buffer)
3953                 dma_free_coherent(mmc_dev(mmc), host->align_buffer_sz +
3954                                   host->adma_table_sz, host->align_buffer,
3955                                   host->align_addr);
3956         host->adma_table = NULL;
3957         host->align_buffer = NULL;
3958 }
3959 EXPORT_SYMBOL_GPL(sdhci_cleanup_host);
3960
3961 int __sdhci_add_host(struct sdhci_host *host)
3962 {
3963         struct mmc_host *mmc = host->mmc;
3964         int ret;
3965
3966         /*
3967          * Init tasklets.
3968          */
3969         tasklet_init(&host->finish_tasklet,
3970                 sdhci_tasklet_finish, (unsigned long)host);
3971
3972         setup_timer(&host->timer, sdhci_timeout_timer, (unsigned long)host);
3973         setup_timer(&host->data_timer, sdhci_timeout_data_timer,
3974                     (unsigned long)host);
3975
3976         init_waitqueue_head(&host->buf_ready_int);
3977
3978         sdhci_init(host, 0);
3979
3980         ret = request_threaded_irq(host->irq, sdhci_irq, sdhci_thread_irq,
3981                                    IRQF_SHARED, mmc_hostname(mmc), host);
3982         if (ret) {
3983                 pr_err("%s: Failed to request IRQ %d: %d\n",
3984                        mmc_hostname(mmc), host->irq, ret);
3985                 goto untasklet;
3986         }
3987
3988         ret = sdhci_led_register(host);
3989         if (ret) {
3990                 pr_err("%s: Failed to register LED device: %d\n",
3991                        mmc_hostname(mmc), ret);
3992                 goto unirq;
3993         }
3994
3995         mmiowb();
3996
3997         ret = mmc_add_host(mmc);
3998         if (ret)
3999                 goto unled;
4000
4001         pr_info("%s: SDHCI controller on %s [%s] using %s\n",
4002                 mmc_hostname(mmc), host->hw_name, dev_name(mmc_dev(mmc)),
4003                 (host->flags & SDHCI_USE_ADMA) ?
4004                 (host->flags & SDHCI_USE_64_BIT_DMA) ? "ADMA 64-bit" : "ADMA" :
4005                 (host->flags & SDHCI_USE_SDMA) ? "DMA" : "PIO");
4006
4007         sdhci_enable_card_detection(host);
4008
4009         return 0;
4010
4011 unled:
4012         sdhci_led_unregister(host);
4013 unirq:
4014         sdhci_do_reset(host, SDHCI_RESET_ALL);
4015         sdhci_writel(host, 0, SDHCI_INT_ENABLE);
4016         sdhci_writel(host, 0, SDHCI_SIGNAL_ENABLE);
4017         free_irq(host->irq, host);
4018 untasklet:
4019         tasklet_kill(&host->finish_tasklet);
4020
4021         return ret;
4022 }
4023 EXPORT_SYMBOL_GPL(__sdhci_add_host);
4024
4025 int sdhci_add_host(struct sdhci_host *host)
4026 {
4027         int ret;
4028
4029         ret = sdhci_setup_host(host);
4030         if (ret)
4031                 return ret;
4032
4033         ret = __sdhci_add_host(host);
4034         if (ret)
4035                 goto cleanup;
4036
4037         return 0;
4038
4039 cleanup:
4040         sdhci_cleanup_host(host);
4041
4042         return ret;
4043 }
4044 EXPORT_SYMBOL_GPL(sdhci_add_host);
4045
4046 void sdhci_remove_host(struct sdhci_host *host, int dead)
4047 {
4048         struct mmc_host *mmc = host->mmc;
4049         unsigned long flags;
4050
4051         if (dead) {
4052                 spin_lock_irqsave(&host->lock, flags);
4053
4054                 host->flags |= SDHCI_DEVICE_DEAD;
4055
4056                 if (sdhci_has_requests(host)) {
4057                         pr_err("%s: Controller removed during "
4058                                 " transfer!\n", mmc_hostname(mmc));
4059                         sdhci_error_out_mrqs(host, -ENOMEDIUM);
4060                 }
4061
4062                 spin_unlock_irqrestore(&host->lock, flags);
4063         }
4064
4065         sdhci_disable_card_detection(host);
4066
4067         mmc_remove_host(mmc);
4068
4069         sdhci_led_unregister(host);
4070
4071         if (!dead)
4072                 sdhci_do_reset(host, SDHCI_RESET_ALL);
4073
4074         sdhci_writel(host, 0, SDHCI_INT_ENABLE);
4075         sdhci_writel(host, 0, SDHCI_SIGNAL_ENABLE);
4076         free_irq(host->irq, host);
4077
4078         del_timer_sync(&host->timer);
4079         del_timer_sync(&host->data_timer);
4080
4081         tasklet_kill(&host->finish_tasklet);
4082
4083         if (!IS_ERR(mmc->supply.vqmmc))
4084                 regulator_disable(mmc->supply.vqmmc);
4085
4086         if (host->align_buffer)
4087                 dma_free_coherent(mmc_dev(mmc), host->align_buffer_sz +
4088                                   host->adma_table_sz, host->align_buffer,
4089                                   host->align_addr);
4090
4091         host->adma_table = NULL;
4092         host->align_buffer = NULL;
4093 }
4094
4095 EXPORT_SYMBOL_GPL(sdhci_remove_host);
4096
4097 void sdhci_free_host(struct sdhci_host *host)
4098 {
4099         mmc_free_host(host->mmc);
4100 }
4101
4102 EXPORT_SYMBOL_GPL(sdhci_free_host);
4103
4104 /*****************************************************************************\
4105  *                                                                           *
4106  * Driver init/exit                                                          *
4107  *                                                                           *
4108 \*****************************************************************************/
4109
4110 static int __init sdhci_drv_init(void)
4111 {
4112         pr_info(DRIVER_NAME
4113                 ": Secure Digital Host Controller Interface driver\n");
4114         pr_info(DRIVER_NAME ": Copyright(c) Pierre Ossman\n");
4115
4116         return 0;
4117 }
4118
4119 static void __exit sdhci_drv_exit(void)
4120 {
4121 }
4122
4123 module_init(sdhci_drv_init);
4124 module_exit(sdhci_drv_exit);
4125
4126 module_param(debug_quirks, uint, 0444);
4127 module_param(debug_quirks2, uint, 0444);
4128
4129 MODULE_AUTHOR("Pierre Ossman <pierre@ossman.eu>");
4130 MODULE_DESCRIPTION("Secure Digital Host Controller Interface core driver");
4131 MODULE_LICENSE("GPL");
4132
4133 MODULE_PARM_DESC(debug_quirks, "Force certain quirks.");
4134 MODULE_PARM_DESC(debug_quirks2, "Force certain other quirks.");