drivers/mtd/nand/raw/gpmi-nand/gpmi-lib.c

   1 // SPDX-License-Identifier: GPL-2.0+
   2 /*
   3  * Freescale GPMI NAND Flash Driver
   4  *
   5  * Copyright (C) 2008-2011 Freescale Semiconductor, Inc.
   6  * Copyright (C) 2008 Embedded Alley Solutions, Inc.
   7  */
   8 #include <linux/delay.h>
   9 #include <linux/clk.h>
  10 #include <linux/slab.h>
  11
  12 #include "gpmi-nand.h"
  13 #include "gpmi-regs.h"
  14 #include "bch-regs.h"
  15
  16 /* Converts time to clock cycles */
  17 #define TO_CYCLES(duration, period) DIV_ROUND_UP_ULL(duration, period)
  18
  19 #define MXS_SET_ADDR            0x4
  20 #define MXS_CLR_ADDR            0x8
  21 /*
  22  * Clear the bit and poll it cleared.  This is usually called with
  23  * a reset address and mask being either SFTRST(bit 31) or CLKGATE
  24  * (bit 30).
  25  */
  26 static int clear_poll_bit(void __iomem *addr, u32 mask)
  27 {
  28         int timeout = 0x400;
  29
  30         /* clear the bit */
  31         writel(mask, addr + MXS_CLR_ADDR);
  32
  33         /*
  34          * SFTRST needs 3 GPMI clocks to settle, the reference manual
  35          * recommends to wait 1us.
  36          */
  37         udelay(1);
  38
  39         /* poll the bit becoming clear */
  40         while ((readl(addr) & mask) && --timeout)
  41                 /* nothing */;
  42
  43         return !timeout;
  44 }
  45
  46 #define MODULE_CLKGATE          (1 << 30)
  47 #define MODULE_SFTRST           (1 << 31)
  48 /*
  49  * The current mxs_reset_block() will do two things:
  50  *  [1] enable the module.
  51  *  [2] reset the module.
  52  *
  53  * In most of the cases, it's ok.
  54  * But in MX23, there is a hardware bug in the BCH block (see erratum #2847).
  55  * If you try to soft reset the BCH block, it becomes unusable until
  56  * the next hard reset. This case occurs in the NAND boot mode. When the board
  57  * boots by NAND, the ROM of the chip will initialize the BCH blocks itself.
  58  * So If the driver tries to reset the BCH again, the BCH will not work anymore.
  59  * You will see a DMA timeout in this case. The bug has been fixed
  60  * in the following chips, such as MX28.
  61  *
  62  * To avoid this bug, just add a new parameter `just_enable` for
  63  * the mxs_reset_block(), and rewrite it here.
  64  */
  65 static int gpmi_reset_block(void __iomem *reset_addr, bool just_enable)
  66 {
  67         int ret;
  68         int timeout = 0x400;
  69
  70         /* clear and poll SFTRST */
  71         ret = clear_poll_bit(reset_addr, MODULE_SFTRST);
  72         if (unlikely(ret))
  73                 goto error;
  74
  75         /* clear CLKGATE */
  76         writel(MODULE_CLKGATE, reset_addr + MXS_CLR_ADDR);
  77
  78         if (!just_enable) {
  79                 /* set SFTRST to reset the block */
  80                 writel(MODULE_SFTRST, reset_addr + MXS_SET_ADDR);
  81                 udelay(1);
  82
  83                 /* poll CLKGATE becoming set */
  84                 while ((!(readl(reset_addr) & MODULE_CLKGATE)) && --timeout)
  85                         /* nothing */;
  86                 if (unlikely(!timeout))
  87                         goto error;
  88         }
  89
  90         /* clear and poll SFTRST */
  91         ret = clear_poll_bit(reset_addr, MODULE_SFTRST);
  92         if (unlikely(ret))
  93                 goto error;
  94
  95         /* clear and poll CLKGATE */
  96         ret = clear_poll_bit(reset_addr, MODULE_CLKGATE);
  97         if (unlikely(ret))
  98                 goto error;
  99
 100         return 0;
 101
 102 error:
 103         pr_err("%s(%p): module reset timeout\n", __func__, reset_addr);
 104         return -ETIMEDOUT;
 105 }
 106
 107 static int __gpmi_enable_clk(struct gpmi_nand_data *this, bool v)
 108 {
 109         struct clk *clk;
 110         int ret;
 111         int i;
 112
 113         for (i = 0; i < GPMI_CLK_MAX; i++) {
 114                 clk = this->resources.clock[i];
 115                 if (!clk)
 116                         break;
 117
 118                 if (v) {
 119                         ret = clk_prepare_enable(clk);
 120                         if (ret)
 121                                 goto err_clk;
 122                 } else {
 123                         clk_disable_unprepare(clk);
 124                 }
 125         }
 126         return 0;
 127
 128 err_clk:
 129         for (; i > 0; i--)
 130                 clk_disable_unprepare(this->resources.clock[i - 1]);
 131         return ret;
 132 }
 133
 134 int gpmi_enable_clk(struct gpmi_nand_data *this)
 135 {
 136         return __gpmi_enable_clk(this, true);
 137 }
 138
 139 int gpmi_disable_clk(struct gpmi_nand_data *this)
 140 {
 141         return __gpmi_enable_clk(this, false);
 142 }
 143
 144 int gpmi_init(struct gpmi_nand_data *this)
 145 {
 146         struct resources *r = &this->resources;
 147         int ret;
 148
 149         ret = gpmi_enable_clk(this);
 150         if (ret)
 151                 return ret;
 152         ret = gpmi_reset_block(r->gpmi_regs, false);
 153         if (ret)
 154                 goto err_out;
 155
 156         /*
 157          * Reset BCH here, too. We got failures otherwise :(
 158          * See later BCH reset for explanation of MX23 and MX28 handling
 159          */
 160         ret = gpmi_reset_block(r->bch_regs,
 161                                GPMI_IS_MX23(this) || GPMI_IS_MX28(this));
 162         if (ret)
 163                 goto err_out;
 164
 165         /* Choose NAND mode. */
 166         writel(BM_GPMI_CTRL1_GPMI_MODE, r->gpmi_regs + HW_GPMI_CTRL1_CLR);
 167
 168         /* Set the IRQ polarity. */
 169         writel(BM_GPMI_CTRL1_ATA_IRQRDY_POLARITY,
 170                                 r->gpmi_regs + HW_GPMI_CTRL1_SET);
 171
 172         /* Disable Write-Protection. */
 173         writel(BM_GPMI_CTRL1_DEV_RESET, r->gpmi_regs + HW_GPMI_CTRL1_SET);
 174
 175         /* Select BCH ECC. */
 176         writel(BM_GPMI_CTRL1_BCH_MODE, r->gpmi_regs + HW_GPMI_CTRL1_SET);
 177
 178         /*
 179          * Decouple the chip select from dma channel. We use dma0 for all
 180          * the chips.
 181          */
 182         writel(BM_GPMI_CTRL1_DECOUPLE_CS, r->gpmi_regs + HW_GPMI_CTRL1_SET);
 183
 184         gpmi_disable_clk(this);
 185         return 0;
 186 err_out:
 187         gpmi_disable_clk(this);
 188         return ret;
 189 }
 190
 191 /* This function is very useful. It is called only when the bug occur. */
 192 void gpmi_dump_info(struct gpmi_nand_data *this)
 193 {
 194         struct resources *r = &this->resources;
 195         struct bch_geometry *geo = &this->bch_geometry;
 196         u32 reg;
 197         int i;
 198
 199         dev_err(this->dev, "Show GPMI registers :\n");
 200         for (i = 0; i <= HW_GPMI_DEBUG / 0x10 + 1; i++) {
 201                 reg = readl(r->gpmi_regs + i * 0x10);
 202                 dev_err(this->dev, "offset 0x%.3x : 0x%.8x\n", i * 0x10, reg);
 203         }
 204
 205         /* start to print out the BCH info */
 206         dev_err(this->dev, "Show BCH registers :\n");
 207         for (i = 0; i <= HW_BCH_VERSION / 0x10 + 1; i++) {
 208                 reg = readl(r->bch_regs + i * 0x10);
 209                 dev_err(this->dev, "offset 0x%.3x : 0x%.8x\n", i * 0x10, reg);
 210         }
 211         dev_err(this->dev, "BCH Geometry :\n"
 212                 "GF length              : %u\n"
 213                 "ECC Strength           : %u\n"
 214                 "Page Size in Bytes     : %u\n"
 215                 "Metadata Size in Bytes : %u\n"
 216                 "ECC Chunk Size in Bytes: %u\n"
 217                 "ECC Chunk Count        : %u\n"
 218                 "Payload Size in Bytes  : %u\n"
 219                 "Auxiliary Size in Bytes: %u\n"
 220                 "Auxiliary Status Offset: %u\n"
 221                 "Block Mark Byte Offset : %u\n"
 222                 "Block Mark Bit Offset  : %u\n",
 223                 geo->gf_len,
 224                 geo->ecc_strength,
 225                 geo->page_size,
 226                 geo->metadata_size,
 227                 geo->ecc_chunk_size,
 228                 geo->ecc_chunk_count,
 229                 geo->payload_size,
 230                 geo->auxiliary_size,
 231                 geo->auxiliary_status_offset,
 232                 geo->block_mark_byte_offset,
 233                 geo->block_mark_bit_offset);
 234 }
 235
 236 /* Configures the geometry for BCH.  */
 237 int bch_set_geometry(struct gpmi_nand_data *this)
 238 {
 239         struct resources *r = &this->resources;
 240         struct bch_geometry *bch_geo = &this->bch_geometry;
 241         unsigned int block_count;
 242         unsigned int block_size;
 243         unsigned int metadata_size;
 244         unsigned int ecc_strength;
 245         unsigned int page_size;
 246         unsigned int gf_len;
 247         int ret;
 248
 249         ret = common_nfc_set_geometry(this);
 250         if (ret)
 251                 return ret;
 252
 253         block_count   = bch_geo->ecc_chunk_count - 1;
 254         block_size    = bch_geo->ecc_chunk_size;
 255         metadata_size = bch_geo->metadata_size;
 256         ecc_strength  = bch_geo->ecc_strength >> 1;
 257         page_size     = bch_geo->page_size;
 258         gf_len        = bch_geo->gf_len;
 259
 260         ret = gpmi_enable_clk(this);
 261         if (ret)
 262                 return ret;
 263
 264         /*
 265         * Due to erratum #2847 of the MX23, the BCH cannot be soft reset on this
 266         * chip, otherwise it will lock up. So we skip resetting BCH on the MX23.
 267         * and MX28.
 268         */
 269         ret = gpmi_reset_block(r->bch_regs,
 270                                GPMI_IS_MX23(this) || GPMI_IS_MX28(this));
 271         if (ret)
 272                 goto err_out;
 273
 274         /* Configure layout 0. */
 275         writel(BF_BCH_FLASH0LAYOUT0_NBLOCKS(block_count)
 276                         | BF_BCH_FLASH0LAYOUT0_META_SIZE(metadata_size)
 277                         | BF_BCH_FLASH0LAYOUT0_ECC0(ecc_strength, this)
 278                         | BF_BCH_FLASH0LAYOUT0_GF(gf_len, this)
 279                         | BF_BCH_FLASH0LAYOUT0_DATA0_SIZE(block_size, this),
 280                         r->bch_regs + HW_BCH_FLASH0LAYOUT0);
 281
 282         writel(BF_BCH_FLASH0LAYOUT1_PAGE_SIZE(page_size)
 283                         | BF_BCH_FLASH0LAYOUT1_ECCN(ecc_strength, this)
 284                         | BF_BCH_FLASH0LAYOUT1_GF(gf_len, this)
 285                         | BF_BCH_FLASH0LAYOUT1_DATAN_SIZE(block_size, this),
 286                         r->bch_regs + HW_BCH_FLASH0LAYOUT1);
 287
 288         /* Set *all* chip selects to use layout 0. */
 289         writel(0, r->bch_regs + HW_BCH_LAYOUTSELECT);
 290
 291         /* Enable interrupts. */
 292         writel(BM_BCH_CTRL_COMPLETE_IRQ_EN,
 293                                 r->bch_regs + HW_BCH_CTRL_SET);
 294
 295         gpmi_disable_clk(this);
 296         return 0;
 297 err_out:
 298         gpmi_disable_clk(this);
 299         return ret;
 300 }
 301
 302 /*
 303  * <1> Firstly, we should know what's the GPMI-clock means.
 304  *     The GPMI-clock is the internal clock in the gpmi nand controller.
 305  *     If you set 100MHz to gpmi nand controller, the GPMI-clock's period
 306  *     is 10ns. Mark the GPMI-clock's period as GPMI-clock-period.
 307  *
 308  * <2> Secondly, we should know what's the frequency on the nand chip pins.
 309  *     The frequency on the nand chip pins is derived from the GPMI-clock.
 310  *     We can get it from the following equation:
 311  *
 312  *         F = G / (DS + DH)
 313  *
 314  *         F  : the frequency on the nand chip pins.
 315  *         G  : the GPMI clock, such as 100MHz.
 316  *         DS : GPMI_HW_GPMI_TIMING0:DATA_SETUP
 317  *         DH : GPMI_HW_GPMI_TIMING0:DATA_HOLD
 318  *
 319  * <3> Thirdly, when the frequency on the nand chip pins is above 33MHz,
 320  *     the nand EDO(extended Data Out) timing could be applied.
 321  *     The GPMI implements a feedback read strobe to sample the read data.
 322  *     The feedback read strobe can be delayed to support the nand EDO timing
 323  *     where the read strobe may deasserts before the read data is valid, and
 324  *     read data is valid for some time after read strobe.
 325  *
 326  *     The following figure illustrates some aspects of a NAND Flash read:
 327  *
 328  *                   |<---tREA---->|
 329  *                   |             |
 330  *                   |         |   |
 331  *                   |<--tRP-->|   |
 332  *                   |         |   |
 333  *                  __          ___|__________________________________
 334  *     RDN            \________/   |
 335  *                                 |
 336  *                                 /---------\
 337  *     Read Data    --------------<           >---------
 338  *                                 \---------/
 339  *                                |     |
 340  *                                |<-D->|
 341  *     FeedbackRDN  ________             ____________
 342  *                          \___________/
 343  *
 344  *          D stands for delay, set in the HW_GPMI_CTRL1:RDN_DELAY.
 345  *
 346  *
 347  * <4> Now, we begin to describe how to compute the right RDN_DELAY.
 348  *
 349  *  4.1) From the aspect of the nand chip pins:
 350  *        Delay = (tREA + C - tRP)               {1}
 351  *
 352  *        tREA : the maximum read access time.
 353  *        C    : a constant to adjust the delay. default is 4000ps.
 354  *        tRP  : the read pulse width, which is exactly:
 355  *                   tRP = (GPMI-clock-period) * DATA_SETUP
 356  *
 357  *  4.2) From the aspect of the GPMI nand controller:
 358  *         Delay = RDN_DELAY * 0.125 * RP        {2}
 359  *
 360  *         RP   : the DLL reference period.
 361  *            if (GPMI-clock-period > DLL_THRETHOLD)
 362  *                   RP = GPMI-clock-period / 2;
 363  *            else
 364  *                   RP = GPMI-clock-period;
 365  *
 366  *            Set the HW_GPMI_CTRL1:HALF_PERIOD if GPMI-clock-period
 367  *            is greater DLL_THRETHOLD. In other SOCs, the DLL_THRETHOLD
 368  *            is 16000ps, but in mx6q, we use 12000ps.
 369  *
 370  *  4.3) since {1} equals {2}, we get:
 371  *
 372  *                     (tREA + 4000 - tRP) * 8
 373  *         RDN_DELAY = -----------------------     {3}
 374  *                           RP
 375  */
 376 static void gpmi_nfc_compute_timings(struct gpmi_nand_data *this,
 377                                      const struct nand_sdr_timings *sdr)
 378 {
 379         struct gpmi_nfc_hardware_timing *hw = &this->hw;
 380         unsigned int dll_threshold_ps = this->devdata->max_chain_delay;
 381         unsigned int period_ps, reference_period_ps;
 382         unsigned int data_setup_cycles, data_hold_cycles, addr_setup_cycles;
 383         unsigned int tRP_ps;
 384         bool use_half_period;
 385         int sample_delay_ps, sample_delay_factor;
 386         u16 busy_timeout_cycles;
 387         u8 wrn_dly_sel;
 388
 389         if (sdr->tRC_min >= 30000) {
 390                 /* ONFI non-EDO modes [0-3] */
 391                 hw->clk_rate = 22000000;
 392                 wrn_dly_sel = BV_GPMI_CTRL1_WRN_DLY_SEL_4_TO_8NS;
 393         } else if (sdr->tRC_min >= 25000) {
 394                 /* ONFI EDO mode 4 */
 395                 hw->clk_rate = 80000000;
 396                 wrn_dly_sel = BV_GPMI_CTRL1_WRN_DLY_SEL_NO_DELAY;
 397         } else {
 398                 /* ONFI EDO mode 5 */
 399                 hw->clk_rate = 100000000;
 400                 wrn_dly_sel = BV_GPMI_CTRL1_WRN_DLY_SEL_NO_DELAY;
 401         }
 402
 403         /* SDR core timings are given in picoseconds */
 404         period_ps = div_u64((u64)NSEC_PER_SEC * 1000, hw->clk_rate);
 405
 406         addr_setup_cycles = TO_CYCLES(sdr->tALS_min, period_ps);
 407         data_setup_cycles = TO_CYCLES(sdr->tDS_min, period_ps);
 408         data_hold_cycles = TO_CYCLES(sdr->tDH_min, period_ps);
 409         busy_timeout_cycles = TO_CYCLES(sdr->tWB_max + sdr->tR_max, period_ps);
 410
 411         hw->timing0 = BF_GPMI_TIMING0_ADDRESS_SETUP(addr_setup_cycles) |
 412                       BF_GPMI_TIMING0_DATA_HOLD(data_hold_cycles) |
 413                       BF_GPMI_TIMING0_DATA_SETUP(data_setup_cycles);
 414         hw->timing1 = BF_GPMI_TIMING1_BUSY_TIMEOUT(busy_timeout_cycles * 4096);
 415
 416         /*
 417          * Derive NFC ideal delay from {3}:
 418          *
 419          *                     (tREA + 4000 - tRP) * 8
 420          *         RDN_DELAY = -----------------------
 421          *                                RP
 422          */
 423         if (period_ps > dll_threshold_ps) {
 424                 use_half_period = true;
 425                 reference_period_ps = period_ps / 2;
 426         } else {
 427                 use_half_period = false;
 428                 reference_period_ps = period_ps;
 429         }
 430
 431         tRP_ps = data_setup_cycles * period_ps;
 432         sample_delay_ps = (sdr->tREA_max + 4000 - tRP_ps) * 8;
 433         if (sample_delay_ps > 0)
 434                 sample_delay_factor = sample_delay_ps / reference_period_ps;
 435         else
 436                 sample_delay_factor = 0;
 437
 438         hw->ctrl1n = BF_GPMI_CTRL1_WRN_DLY_SEL(wrn_dly_sel);
 439         if (sample_delay_factor)
 440                 hw->ctrl1n |= BF_GPMI_CTRL1_RDN_DELAY(sample_delay_factor) |
 441                               BM_GPMI_CTRL1_DLL_ENABLE |
 442                               (use_half_period ? BM_GPMI_CTRL1_HALF_PERIOD : 0);
 443 }
 444
 445 void gpmi_nfc_apply_timings(struct gpmi_nand_data *this)
 446 {
 447         struct gpmi_nfc_hardware_timing *hw = &this->hw;
 448         struct resources *r = &this->resources;
 449         void __iomem *gpmi_regs = r->gpmi_regs;
 450         unsigned int dll_wait_time_us;
 451
 452         clk_set_rate(r->clock[0], hw->clk_rate);
 453
 454         writel(hw->timing0, gpmi_regs + HW_GPMI_TIMING0);
 455         writel(hw->timing1, gpmi_regs + HW_GPMI_TIMING1);
 456
 457         /*
 458          * Clear several CTRL1 fields, DLL must be disabled when setting
 459          * RDN_DELAY or HALF_PERIOD.
 460          */
 461         writel(BM_GPMI_CTRL1_CLEAR_MASK, gpmi_regs + HW_GPMI_CTRL1_CLR);
 462         writel(hw->ctrl1n, gpmi_regs + HW_GPMI_CTRL1_SET);
 463
 464         /* Wait 64 clock cycles before using the GPMI after enabling the DLL */
 465         dll_wait_time_us = USEC_PER_SEC / hw->clk_rate * 64;
 466         if (!dll_wait_time_us)
 467                 dll_wait_time_us = 1;
 468
 469         /* Wait for the DLL to settle. */
 470         udelay(dll_wait_time_us);
 471 }
 472
 473 int gpmi_setup_data_interface(struct mtd_info *mtd, int chipnr,
 474                               const struct nand_data_interface *conf)
 475 {
 476         struct nand_chip *chip = mtd_to_nand(mtd);
 477         struct gpmi_nand_data *this = nand_get_controller_data(chip);
 478         const struct nand_sdr_timings *sdr;
 479
 480         /* Retrieve required NAND timings */
 481         sdr = nand_get_sdr_timings(conf);
 482         if (IS_ERR(sdr))
 483                 return PTR_ERR(sdr);
 484
 485         /* Only MX6 GPMI controller can reach EDO timings */
 486         if (sdr->tRC_min <= 25000 && !GPMI_IS_MX6(this))
 487                 return -ENOTSUPP;
 488
 489         /* Stop here if this call was just a check */
 490         if (chipnr < 0)
 491                 return 0;
 492
 493         /* Do the actual derivation of the controller timings */
 494         gpmi_nfc_compute_timings(this, sdr);
 495
 496         this->hw.must_apply_timings = true;
 497
 498         return 0;
 499 }
 500
 501 /* Clears a BCH interrupt. */
 502 void gpmi_clear_bch(struct gpmi_nand_data *this)
 503 {
 504         struct resources *r = &this->resources;
 505         writel(BM_BCH_CTRL_COMPLETE_IRQ, r->bch_regs + HW_BCH_CTRL_CLR);
 506 }
 507
 508 /* Returns the Ready/Busy status of the given chip. */
 509 int gpmi_is_ready(struct gpmi_nand_data *this, unsigned chip)
 510 {
 511         struct resources *r = &this->resources;
 512         uint32_t mask = 0;
 513         uint32_t reg = 0;
 514
 515         if (GPMI_IS_MX23(this)) {
 516                 mask = MX23_BM_GPMI_DEBUG_READY0 << chip;
 517                 reg = readl(r->gpmi_regs + HW_GPMI_DEBUG);
 518         } else if (GPMI_IS_MX28(this) || GPMI_IS_MX6(this)) {
 519                 /*
 520                  * In the imx6, all the ready/busy pins are bound
 521                  * together. So we only need to check chip 0.
 522                  */
 523                 if (GPMI_IS_MX6(this))
 524                         chip = 0;
 525
 526                 /* MX28 shares the same R/B register as MX6Q. */
 527                 mask = MX28_BF_GPMI_STAT_READY_BUSY(1 << chip);
 528                 reg = readl(r->gpmi_regs + HW_GPMI_STAT);
 529         } else
 530                 dev_err(this->dev, "unknown arch.\n");
 531         return reg & mask;
 532 }
 533
 534 int gpmi_send_command(struct gpmi_nand_data *this)
 535 {
 536         struct dma_chan *channel = get_dma_chan(this);
 537         struct dma_async_tx_descriptor *desc;
 538         struct scatterlist *sgl;
 539         int chip = this->current_chip;
 540         int ret;
 541         u32 pio[3];
 542
 543         /* [1] send out the PIO words */
 544         pio[0] = BF_GPMI_CTRL0_COMMAND_MODE(BV_GPMI_CTRL0_COMMAND_MODE__WRITE)
 545                 | BM_GPMI_CTRL0_WORD_LENGTH
 546                 | BF_GPMI_CTRL0_CS(chip, this)
 547                 | BF_GPMI_CTRL0_LOCK_CS(LOCK_CS_ENABLE, this)
 548                 | BF_GPMI_CTRL0_ADDRESS(BV_GPMI_CTRL0_ADDRESS__NAND_CLE)
 549                 | BM_GPMI_CTRL0_ADDRESS_INCREMENT
 550                 | BF_GPMI_CTRL0_XFER_COUNT(this->command_length);
 551         pio[1] = pio[2] = 0;
 552         desc = dmaengine_prep_slave_sg(channel,
 553                                         (struct scatterlist *)pio,
 554                                         ARRAY_SIZE(pio), DMA_TRANS_NONE, 0);
 555         if (!desc)
 556                 return -EINVAL;
 557
 558         /* [2] send out the COMMAND + ADDRESS string stored in @buffer */
 559         sgl = &this->cmd_sgl;
 560
 561         sg_init_one(sgl, this->cmd_buffer, this->command_length);
 562         dma_map_sg(this->dev, sgl, 1, DMA_TO_DEVICE);
 563         desc = dmaengine_prep_slave_sg(channel,
 564                                 sgl, 1, DMA_MEM_TO_DEV,
 565                                 DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
 566         if (!desc)
 567                 return -EINVAL;
 568
 569         /* [3] submit the DMA */
 570         ret = start_dma_without_bch_irq(this, desc);
 571
 572         dma_unmap_sg(this->dev, sgl, 1, DMA_TO_DEVICE);
 573
 574         return ret;
 575 }
 576
 577 int gpmi_send_data(struct gpmi_nand_data *this, const void *buf, int len)
 578 {
 579         struct dma_async_tx_descriptor *desc;
 580         struct dma_chan *channel = get_dma_chan(this);
 581         int chip = this->current_chip;
 582         int ret;
 583         uint32_t command_mode;
 584         uint32_t address;
 585         u32 pio[2];
 586
 587         /* [1] PIO */
 588         command_mode = BV_GPMI_CTRL0_COMMAND_MODE__WRITE;
 589         address      = BV_GPMI_CTRL0_ADDRESS__NAND_DATA;
 590
 591         pio[0] = BF_GPMI_CTRL0_COMMAND_MODE(command_mode)
 592                 | BM_GPMI_CTRL0_WORD_LENGTH
 593                 | BF_GPMI_CTRL0_CS(chip, this)
 594                 | BF_GPMI_CTRL0_LOCK_CS(LOCK_CS_ENABLE, this)
 595                 | BF_GPMI_CTRL0_ADDRESS(address)
 596                 | BF_GPMI_CTRL0_XFER_COUNT(len);
 597         pio[1] = 0;
 598         desc = dmaengine_prep_slave_sg(channel, (struct scatterlist *)pio,
 599                                         ARRAY_SIZE(pio), DMA_TRANS_NONE, 0);
 600         if (!desc)
 601                 return -EINVAL;
 602
 603         /* [2] send DMA request */
 604         prepare_data_dma(this, buf, len, DMA_TO_DEVICE);
 605         desc = dmaengine_prep_slave_sg(channel, &this->data_sgl,
 606                                         1, DMA_MEM_TO_DEV,
 607                                         DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
 608         if (!desc)
 609                 return -EINVAL;
 610
 611         /* [3] submit the DMA */
 612         ret = start_dma_without_bch_irq(this, desc);
 613
 614         dma_unmap_sg(this->dev, &this->data_sgl, 1, DMA_TO_DEVICE);
 615
 616         return ret;
 617 }
 618
 619 int gpmi_read_data(struct gpmi_nand_data *this, void *buf, int len)
 620 {
 621         struct dma_async_tx_descriptor *desc;
 622         struct dma_chan *channel = get_dma_chan(this);
 623         int chip = this->current_chip;
 624         int ret;
 625         u32 pio[2];
 626         bool direct;
 627
 628         /* [1] : send PIO */
 629         pio[0] = BF_GPMI_CTRL0_COMMAND_MODE(BV_GPMI_CTRL0_COMMAND_MODE__READ)
 630                 | BM_GPMI_CTRL0_WORD_LENGTH
 631                 | BF_GPMI_CTRL0_CS(chip, this)
 632                 | BF_GPMI_CTRL0_LOCK_CS(LOCK_CS_ENABLE, this)
 633                 | BF_GPMI_CTRL0_ADDRESS(BV_GPMI_CTRL0_ADDRESS__NAND_DATA)
 634                 | BF_GPMI_CTRL0_XFER_COUNT(len);
 635         pio[1] = 0;
 636         desc = dmaengine_prep_slave_sg(channel,
 637                                         (struct scatterlist *)pio,
 638                                         ARRAY_SIZE(pio), DMA_TRANS_NONE, 0);
 639         if (!desc)
 640                 return -EINVAL;
 641
 642         /* [2] : send DMA request */
 643         direct = prepare_data_dma(this, buf, len, DMA_FROM_DEVICE);
 644         desc = dmaengine_prep_slave_sg(channel, &this->data_sgl,
 645                                         1, DMA_DEV_TO_MEM,
 646                                         DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
 647         if (!desc)
 648                 return -EINVAL;
 649
 650         /* [3] : submit the DMA */
 651
 652         ret = start_dma_without_bch_irq(this, desc);
 653
 654         dma_unmap_sg(this->dev, &this->data_sgl, 1, DMA_FROM_DEVICE);
 655         if (!direct)
 656                 memcpy(buf, this->data_buffer_dma, len);
 657
 658         return ret;
 659 }
 660
 661 int gpmi_send_page(struct gpmi_nand_data *this,
 662                         dma_addr_t payload, dma_addr_t auxiliary)
 663 {
 664         struct bch_geometry *geo = &this->bch_geometry;
 665         uint32_t command_mode;
 666         uint32_t address;
 667         uint32_t ecc_command;
 668         uint32_t buffer_mask;
 669         struct dma_async_tx_descriptor *desc;
 670         struct dma_chan *channel = get_dma_chan(this);
 671         int chip = this->current_chip;
 672         u32 pio[6];
 673
 674         /* A DMA descriptor that does an ECC page read. */
 675         command_mode = BV_GPMI_CTRL0_COMMAND_MODE__WRITE;
 676         address      = BV_GPMI_CTRL0_ADDRESS__NAND_DATA;
 677         ecc_command  = BV_GPMI_ECCCTRL_ECC_CMD__BCH_ENCODE;
 678         buffer_mask  = BV_GPMI_ECCCTRL_BUFFER_MASK__BCH_PAGE |
 679                                 BV_GPMI_ECCCTRL_BUFFER_MASK__BCH_AUXONLY;
 680
 681         pio[0] = BF_GPMI_CTRL0_COMMAND_MODE(command_mode)
 682                 | BM_GPMI_CTRL0_WORD_LENGTH
 683                 | BF_GPMI_CTRL0_CS(chip, this)
 684                 | BF_GPMI_CTRL0_LOCK_CS(LOCK_CS_ENABLE, this)
 685                 | BF_GPMI_CTRL0_ADDRESS(address)
 686                 | BF_GPMI_CTRL0_XFER_COUNT(0);
 687         pio[1] = 0;
 688         pio[2] = BM_GPMI_ECCCTRL_ENABLE_ECC
 689                 | BF_GPMI_ECCCTRL_ECC_CMD(ecc_command)
 690                 | BF_GPMI_ECCCTRL_BUFFER_MASK(buffer_mask);
 691         pio[3] = geo->page_size;
 692         pio[4] = payload;
 693         pio[5] = auxiliary;
 694
 695         desc = dmaengine_prep_slave_sg(channel,
 696                                         (struct scatterlist *)pio,
 697                                         ARRAY_SIZE(pio), DMA_TRANS_NONE,
 698                                         DMA_CTRL_ACK);
 699         if (!desc)
 700                 return -EINVAL;
 701
 702         return start_dma_with_bch_irq(this, desc);
 703 }
 704
 705 int gpmi_read_page(struct gpmi_nand_data *this,
 706                                 dma_addr_t payload, dma_addr_t auxiliary)
 707 {
 708         struct bch_geometry *geo = &this->bch_geometry;
 709         uint32_t command_mode;
 710         uint32_t address;
 711         uint32_t ecc_command;
 712         uint32_t buffer_mask;
 713         struct dma_async_tx_descriptor *desc;
 714         struct dma_chan *channel = get_dma_chan(this);
 715         int chip = this->current_chip;
 716         u32 pio[6];
 717
 718         /* [1] Wait for the chip to report ready. */
 719         command_mode = BV_GPMI_CTRL0_COMMAND_MODE__WAIT_FOR_READY;
 720         address      = BV_GPMI_CTRL0_ADDRESS__NAND_DATA;
 721
 722         pio[0] =  BF_GPMI_CTRL0_COMMAND_MODE(command_mode)
 723                 | BM_GPMI_CTRL0_WORD_LENGTH
 724                 | BF_GPMI_CTRL0_CS(chip, this)
 725                 | BF_GPMI_CTRL0_LOCK_CS(LOCK_CS_ENABLE, this)
 726                 | BF_GPMI_CTRL0_ADDRESS(address)
 727                 | BF_GPMI_CTRL0_XFER_COUNT(0);
 728         pio[1] = 0;
 729         desc = dmaengine_prep_slave_sg(channel,
 730                                 (struct scatterlist *)pio, 2,
 731                                 DMA_TRANS_NONE, 0);
 732         if (!desc)
 733                 return -EINVAL;
 734
 735         /* [2] Enable the BCH block and read. */
 736         command_mode = BV_GPMI_CTRL0_COMMAND_MODE__READ;
 737         address      = BV_GPMI_CTRL0_ADDRESS__NAND_DATA;
 738         ecc_command  = BV_GPMI_ECCCTRL_ECC_CMD__BCH_DECODE;
 739         buffer_mask  = BV_GPMI_ECCCTRL_BUFFER_MASK__BCH_PAGE
 740                         | BV_GPMI_ECCCTRL_BUFFER_MASK__BCH_AUXONLY;
 741
 742         pio[0] =  BF_GPMI_CTRL0_COMMAND_MODE(command_mode)
 743                 | BM_GPMI_CTRL0_WORD_LENGTH
 744                 | BF_GPMI_CTRL0_CS(chip, this)
 745                 | BF_GPMI_CTRL0_LOCK_CS(LOCK_CS_ENABLE, this)
 746                 | BF_GPMI_CTRL0_ADDRESS(address)
 747                 | BF_GPMI_CTRL0_XFER_COUNT(geo->page_size);
 748
 749         pio[1] = 0;
 750         pio[2] =  BM_GPMI_ECCCTRL_ENABLE_ECC
 751                 | BF_GPMI_ECCCTRL_ECC_CMD(ecc_command)
 752                 | BF_GPMI_ECCCTRL_BUFFER_MASK(buffer_mask);
 753         pio[3] = geo->page_size;
 754         pio[4] = payload;
 755         pio[5] = auxiliary;
 756         desc = dmaengine_prep_slave_sg(channel,
 757                                         (struct scatterlist *)pio,
 758                                         ARRAY_SIZE(pio), DMA_TRANS_NONE,
 759                                         DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
 760         if (!desc)
 761                 return -EINVAL;
 762
 763         /* [3] Disable the BCH block */
 764         command_mode = BV_GPMI_CTRL0_COMMAND_MODE__WAIT_FOR_READY;
 765         address      = BV_GPMI_CTRL0_ADDRESS__NAND_DATA;
 766
 767         pio[0] = BF_GPMI_CTRL0_COMMAND_MODE(command_mode)
 768                 | BM_GPMI_CTRL0_WORD_LENGTH
 769                 | BF_GPMI_CTRL0_CS(chip, this)
 770                 | BF_GPMI_CTRL0_LOCK_CS(LOCK_CS_ENABLE, this)
 771                 | BF_GPMI_CTRL0_ADDRESS(address)
 772                 | BF_GPMI_CTRL0_XFER_COUNT(geo->page_size);
 773         pio[1] = 0;
 774         pio[2] = 0; /* clear GPMI_HW_GPMI_ECCCTRL, disable the BCH. */
 775         desc = dmaengine_prep_slave_sg(channel,
 776                                 (struct scatterlist *)pio, 3,
 777                                 DMA_TRANS_NONE,
 778                                 DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
 779         if (!desc)
 780                 return -EINVAL;
 781
 782         /* [4] submit the DMA */
 783         return start_dma_with_bch_irq(this, desc);
 784 }
 785
 786 /**
 787  * gpmi_copy_bits - copy bits from one memory region to another
 788  * @dst: destination buffer
 789  * @dst_bit_off: bit offset we're starting to write at
 790  * @src: source buffer
 791  * @src_bit_off: bit offset we're starting to read from
 792  * @nbits: number of bits to copy
 793  *
 794  * This functions copies bits from one memory region to another, and is used by
 795  * the GPMI driver to copy ECC sections which are not guaranteed to be byte
 796  * aligned.
 797  *
 798  * src and dst should not overlap.
 799  *
 800  */
 801 void gpmi_copy_bits(u8 *dst, size_t dst_bit_off,
 802                     const u8 *src, size_t src_bit_off,
 803                     size_t nbits)
 804 {
 805         size_t i;
 806         size_t nbytes;
 807         u32 src_buffer = 0;
 808         size_t bits_in_src_buffer = 0;
 809
 810         if (!nbits)
 811                 return;
 812
 813         /*
 814          * Move src and dst pointers to the closest byte pointer and store bit
 815          * offsets within a byte.
 816          */
 817         src += src_bit_off / 8;
 818         src_bit_off %= 8;
 819
 820         dst += dst_bit_off / 8;
 821         dst_bit_off %= 8;
 822
 823         /*
 824          * Initialize the src_buffer value with bits available in the first
 825          * byte of data so that we end up with a byte aligned src pointer.
 826          */
 827         if (src_bit_off) {
 828                 src_buffer = src[0] >> src_bit_off;
 829                 if (nbits >= (8 - src_bit_off)) {
 830                         bits_in_src_buffer += 8 - src_bit_off;
 831                 } else {
 832                         src_buffer &= GENMASK(nbits - 1, 0);
 833                         bits_in_src_buffer += nbits;
 834                 }
 835                 nbits -= bits_in_src_buffer;
 836                 src++;
 837         }
 838
 839         /* Calculate the number of bytes that can be copied from src to dst. */
 840         nbytes = nbits / 8;
 841
 842         /* Try to align dst to a byte boundary. */
 843         if (dst_bit_off) {
 844                 if (bits_in_src_buffer < (8 - dst_bit_off) && nbytes) {
 845                         src_buffer |= src[0] << bits_in_src_buffer;
 846                         bits_in_src_buffer += 8;
 847                         src++;
 848                         nbytes--;
 849                 }
 850
 851                 if (bits_in_src_buffer >= (8 - dst_bit_off)) {
 852                         dst[0] &= GENMASK(dst_bit_off - 1, 0);
 853                         dst[0] |= src_buffer << dst_bit_off;
 854                         src_buffer >>= (8 - dst_bit_off);
 855                         bits_in_src_buffer -= (8 - dst_bit_off);
 856                         dst_bit_off = 0;
 857                         dst++;
 858                         if (bits_in_src_buffer > 7) {
 859                                 bits_in_src_buffer -= 8;
 860                                 dst[0] = src_buffer;
 861                                 dst++;
 862                                 src_buffer >>= 8;
 863                         }
 864                 }
 865         }
 866
 867         if (!bits_in_src_buffer && !dst_bit_off) {
 868                 /*
 869                  * Both src and dst pointers are byte aligned, thus we can
 870                  * just use the optimized memcpy function.
 871                  */
 872                 if (nbytes)
 873                         memcpy(dst, src, nbytes);
 874         } else {
 875                 /*
 876                  * src buffer is not byte aligned, hence we have to copy each
 877                  * src byte to the src_buffer variable before extracting a byte
 878                  * to store in dst.
 879                  */
 880                 for (i = 0; i < nbytes; i++) {
 881                         src_buffer |= src[i] << bits_in_src_buffer;
 882                         dst[i] = src_buffer;
 883                         src_buffer >>= 8;
 884                 }
 885         }
 886         /* Update dst and src pointers */
 887         dst += nbytes;
 888         src += nbytes;
 889
 890         /*
 891          * nbits is the number of remaining bits. It should not exceed 8 as
 892          * we've already copied as much bytes as possible.
 893          */
 894         nbits %= 8;
 895
 896         /*
 897          * If there's no more bits to copy to the destination and src buffer
 898          * was already byte aligned, then we're done.
 899          */
 900         if (!nbits && !bits_in_src_buffer)
 901                 return;
 902
 903         /* Copy the remaining bits to src_buffer */
 904         if (nbits)
 905                 src_buffer |= (*src & GENMASK(nbits - 1, 0)) <<
 906                               bits_in_src_buffer;
 907         bits_in_src_buffer += nbits;
 908
 909         /*
 910          * In case there were not enough bits to get a byte aligned dst buffer
 911          * prepare the src_buffer variable to match the dst organization (shift
 912          * src_buffer by dst_bit_off and retrieve the least significant bits
 913          * from dst).
 914          */
 915         if (dst_bit_off)
 916                 src_buffer = (src_buffer << dst_bit_off) |
 917                              (*dst & GENMASK(dst_bit_off - 1, 0));
 918         bits_in_src_buffer += dst_bit_off;
 919
 920         /*
 921          * Keep most significant bits from dst if we end up with an unaligned
 922          * number of bits.
 923          */
 924         nbytes = bits_in_src_buffer / 8;
 925         if (bits_in_src_buffer % 8) {
 926                 src_buffer |= (dst[nbytes] &
 927                                GENMASK(7, bits_in_src_buffer % 8)) <<
 928                               (nbytes * 8);
 929                 nbytes++;
 930         }
 931
 932         /* Copy the remaining bytes to dst */
 933         for (i = 0; i < nbytes; i++) {
 934                 dst[i] = src_buffer;
 935                 src_buffer >>= 8;
 936         }
 937 }