drivers/mtd/nand/sunxi_nand.c

   1 /*
   2  * Copyright (C) 2013 Boris BREZILLON <b.brezillon.dev@gmail.com>
   3  *
   4  * Derived from:
   5  *      https://github.com/yuq/sunxi-nfc-mtd
   6  *      Copyright (C) 2013 Qiang Yu <yuq825@gmail.com>
   7  *
   8  *      https://github.com/hno/Allwinner-Info
   9  *      Copyright (C) 2013 Henrik Nordström <Henrik Nordström>
  10  *
  11  *      Copyright (C) 2013 Dmitriy B. <rzk333@gmail.com>
  12  *      Copyright (C) 2013 Sergey Lapin <slapin@ossfans.org>
  13  *
  14  * This program is free software; you can redistribute it and/or modify
  15  * it under the terms of the GNU General Public License as published by
  16  * the Free Software Foundation; either version 2 of the License, or
  17  * (at your option) any later version.
  18  *
  19  * This program is distributed in the hope that it will be useful,
  20  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  21  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  22  * GNU General Public License for more details.
  23  */
  24
  25 #include <linux/dma-mapping.h>
  26 #include <linux/slab.h>
  27 #include <linux/module.h>
  28 #include <linux/moduleparam.h>
  29 #include <linux/platform_device.h>
  30 #include <linux/of.h>
  31 #include <linux/of_device.h>
  32 #include <linux/of_gpio.h>
  33 #include <linux/mtd/mtd.h>
  34 #include <linux/mtd/nand.h>
  35 #include <linux/mtd/partitions.h>
  36 #include <linux/clk.h>
  37 #include <linux/delay.h>
  38 #include <linux/dmaengine.h>
  39 #include <linux/gpio.h>
  40 #include <linux/interrupt.h>
  41 #include <linux/iopoll.h>
  42 #include <linux/reset.h>
  43
  44 #define NFC_REG_CTL             0x0000
  45 #define NFC_REG_ST              0x0004
  46 #define NFC_REG_INT             0x0008
  47 #define NFC_REG_TIMING_CTL      0x000C
  48 #define NFC_REG_TIMING_CFG      0x0010
  49 #define NFC_REG_ADDR_LOW        0x0014
  50 #define NFC_REG_ADDR_HIGH       0x0018
  51 #define NFC_REG_SECTOR_NUM      0x001C
  52 #define NFC_REG_CNT             0x0020
  53 #define NFC_REG_CMD             0x0024
  54 #define NFC_REG_RCMD_SET        0x0028
  55 #define NFC_REG_WCMD_SET        0x002C
  56 #define NFC_REG_IO_DATA         0x0030
  57 #define NFC_REG_ECC_CTL         0x0034
  58 #define NFC_REG_ECC_ST          0x0038
  59 #define NFC_REG_DEBUG           0x003C
  60 #define NFC_REG_ECC_ERR_CNT(x)  ((0x0040 + (x)) & ~0x3)
  61 #define NFC_REG_USER_DATA(x)    (0x0050 + ((x) * 4))
  62 #define NFC_REG_SPARE_AREA      0x00A0
  63 #define NFC_REG_PAT_ID          0x00A4
  64 #define NFC_RAM0_BASE           0x0400
  65 #define NFC_RAM1_BASE           0x0800
  66
  67 /* define bit use in NFC_CTL */
  68 #define NFC_EN                  BIT(0)
  69 #define NFC_RESET               BIT(1)
  70 #define NFC_BUS_WIDTH_MSK       BIT(2)
  71 #define NFC_BUS_WIDTH_8         (0 << 2)
  72 #define NFC_BUS_WIDTH_16        (1 << 2)
  73 #define NFC_RB_SEL_MSK          BIT(3)
  74 #define NFC_RB_SEL(x)           ((x) << 3)
  75 #define NFC_CE_SEL_MSK          GENMASK(26, 24)
  76 #define NFC_CE_SEL(x)           ((x) << 24)
  77 #define NFC_CE_CTL              BIT(6)
  78 #define NFC_PAGE_SHIFT_MSK      GENMASK(11, 8)
  79 #define NFC_PAGE_SHIFT(x)       (((x) < 10 ? 0 : (x) - 10) << 8)
  80 #define NFC_SAM                 BIT(12)
  81 #define NFC_RAM_METHOD          BIT(14)
  82 #define NFC_DEBUG_CTL           BIT(31)
  83
  84 /* define bit use in NFC_ST */
  85 #define NFC_RB_B2R              BIT(0)
  86 #define NFC_CMD_INT_FLAG        BIT(1)
  87 #define NFC_DMA_INT_FLAG        BIT(2)
  88 #define NFC_CMD_FIFO_STATUS     BIT(3)
  89 #define NFC_STA                 BIT(4)
  90 #define NFC_NATCH_INT_FLAG      BIT(5)
  91 #define NFC_RB_STATE(x)         BIT(x + 8)
  92
  93 /* define bit use in NFC_INT */
  94 #define NFC_B2R_INT_ENABLE      BIT(0)
  95 #define NFC_CMD_INT_ENABLE      BIT(1)
  96 #define NFC_DMA_INT_ENABLE      BIT(2)
  97 #define NFC_INT_MASK            (NFC_B2R_INT_ENABLE | \
  98                                  NFC_CMD_INT_ENABLE | \
  99                                  NFC_DMA_INT_ENABLE)
 100
 101 /* define bit use in NFC_TIMING_CTL */
 102 #define NFC_TIMING_CTL_EDO      BIT(8)
 103
 104 /* define NFC_TIMING_CFG register layout */
 105 #define NFC_TIMING_CFG(tWB, tADL, tWHR, tRHW, tCAD)             \
 106         (((tWB) & 0x3) | (((tADL) & 0x3) << 2) |                \
 107         (((tWHR) & 0x3) << 4) | (((tRHW) & 0x3) << 6) |         \
 108         (((tCAD) & 0x7) << 8))
 109
 110 /* define bit use in NFC_CMD */
 111 #define NFC_CMD_LOW_BYTE_MSK    GENMASK(7, 0)
 112 #define NFC_CMD_HIGH_BYTE_MSK   GENMASK(15, 8)
 113 #define NFC_CMD(x)              (x)
 114 #define NFC_ADR_NUM_MSK         GENMASK(18, 16)
 115 #define NFC_ADR_NUM(x)          (((x) - 1) << 16)
 116 #define NFC_SEND_ADR            BIT(19)
 117 #define NFC_ACCESS_DIR          BIT(20)
 118 #define NFC_DATA_TRANS          BIT(21)
 119 #define NFC_SEND_CMD1           BIT(22)
 120 #define NFC_WAIT_FLAG           BIT(23)
 121 #define NFC_SEND_CMD2           BIT(24)
 122 #define NFC_SEQ                 BIT(25)
 123 #define NFC_DATA_SWAP_METHOD    BIT(26)
 124 #define NFC_ROW_AUTO_INC        BIT(27)
 125 #define NFC_SEND_CMD3           BIT(28)
 126 #define NFC_SEND_CMD4           BIT(29)
 127 #define NFC_CMD_TYPE_MSK        GENMASK(31, 30)
 128 #define NFC_NORMAL_OP           (0 << 30)
 129 #define NFC_ECC_OP              (1 << 30)
 130 #define NFC_PAGE_OP             (2 << 30)
 131
 132 /* define bit use in NFC_RCMD_SET */
 133 #define NFC_READ_CMD_MSK        GENMASK(7, 0)
 134 #define NFC_RND_READ_CMD0_MSK   GENMASK(15, 8)
 135 #define NFC_RND_READ_CMD1_MSK   GENMASK(23, 16)
 136
 137 /* define bit use in NFC_WCMD_SET */
 138 #define NFC_PROGRAM_CMD_MSK     GENMASK(7, 0)
 139 #define NFC_RND_WRITE_CMD_MSK   GENMASK(15, 8)
 140 #define NFC_READ_CMD0_MSK       GENMASK(23, 16)
 141 #define NFC_READ_CMD1_MSK       GENMASK(31, 24)
 142
 143 /* define bit use in NFC_ECC_CTL */
 144 #define NFC_ECC_EN              BIT(0)
 145 #define NFC_ECC_PIPELINE        BIT(3)
 146 #define NFC_ECC_EXCEPTION       BIT(4)
 147 #define NFC_ECC_BLOCK_SIZE_MSK  BIT(5)
 148 #define NFC_RANDOM_EN           BIT(9)
 149 #define NFC_RANDOM_DIRECTION    BIT(10)
 150 #define NFC_ECC_MODE_MSK        GENMASK(15, 12)
 151 #define NFC_ECC_MODE(x)         ((x) << 12)
 152 #define NFC_RANDOM_SEED_MSK     GENMASK(30, 16)
 153 #define NFC_RANDOM_SEED(x)      ((x) << 16)
 154
 155 /* define bit use in NFC_ECC_ST */
 156 #define NFC_ECC_ERR(x)          BIT(x)
 157 #define NFC_ECC_ERR_MSK         GENMASK(15, 0)
 158 #define NFC_ECC_PAT_FOUND(x)    BIT(x + 16)
 159 #define NFC_ECC_ERR_CNT(b, x)   (((x) >> (((b) % 4) * 8)) & 0xff)
 160
 161 #define NFC_DEFAULT_TIMEOUT_MS  1000
 162
 163 #define NFC_SRAM_SIZE           1024
 164
 165 #define NFC_MAX_CS              7
 166
 167 /*
 168  * Ready/Busy detection type: describes the Ready/Busy detection modes
 169  *
 170  * @RB_NONE:    no external detection available, rely on STATUS command
 171  *              and software timeouts
 172  * @RB_NATIVE:  use sunxi NAND controller Ready/Busy support. The Ready/Busy
 173  *              pin of the NAND flash chip must be connected to one of the
 174  *              native NAND R/B pins (those which can be muxed to the NAND
 175  *              Controller)
 176  * @RB_GPIO:    use a simple GPIO to handle Ready/Busy status. The Ready/Busy
 177  *              pin of the NAND flash chip must be connected to a GPIO capable
 178  *              pin.
 179  */
 180 enum sunxi_nand_rb_type {
 181         RB_NONE,
 182         RB_NATIVE,
 183         RB_GPIO,
 184 };
 185
 186 /*
 187  * Ready/Busy structure: stores information related to Ready/Busy detection
 188  *
 189  * @type:       the Ready/Busy detection mode
 190  * @info:       information related to the R/B detection mode. Either a gpio
 191  *              id or a native R/B id (those supported by the NAND controller).
 192  */
 193 struct sunxi_nand_rb {
 194         enum sunxi_nand_rb_type type;
 195         union {
 196                 int gpio;
 197                 int nativeid;
 198         } info;
 199 };
 200
 201 /*
 202  * Chip Select structure: stores information related to NAND Chip Select
 203  *
 204  * @cs:         the NAND CS id used to communicate with a NAND Chip
 205  * @rb:         the Ready/Busy description
 206  */
 207 struct sunxi_nand_chip_sel {
 208         u8 cs;
 209         struct sunxi_nand_rb rb;
 210 };
 211
 212 /*
 213  * sunxi HW ECC infos: stores information related to HW ECC support
 214  *
 215  * @mode:       the sunxi ECC mode field deduced from ECC requirements
 216  */
 217 struct sunxi_nand_hw_ecc {
 218         int mode;
 219 };
 220
 221 /*
 222  * NAND chip structure: stores NAND chip device related information
 223  *
 224  * @node:               used to store NAND chips into a list
 225  * @nand:               base NAND chip structure
 226  * @mtd:                base MTD structure
 227  * @clk_rate:           clk_rate required for this NAND chip
 228  * @timing_cfg          TIMING_CFG register value for this NAND chip
 229  * @selected:           current active CS
 230  * @nsels:              number of CS lines required by the NAND chip
 231  * @sels:               array of CS lines descriptions
 232  */
 233 struct sunxi_nand_chip {
 234         struct list_head node;
 235         struct nand_chip nand;
 236         unsigned long clk_rate;
 237         u32 timing_cfg;
 238         u32 timing_ctl;
 239         int selected;
 240         int addr_cycles;
 241         u32 addr[2];
 242         int cmd_cycles;
 243         u8 cmd[2];
 244         int nsels;
 245         struct sunxi_nand_chip_sel sels[0];
 246 };
 247
 248 static inline struct sunxi_nand_chip *to_sunxi_nand(struct nand_chip *nand)
 249 {
 250         return container_of(nand, struct sunxi_nand_chip, nand);
 251 }
 252
 253 /*
 254  * NAND Controller structure: stores sunxi NAND controller information
 255  *
 256  * @controller:         base controller structure
 257  * @dev:                parent device (used to print error messages)
 258  * @regs:               NAND controller registers
 259  * @ahb_clk:            NAND Controller AHB clock
 260  * @mod_clk:            NAND Controller mod clock
 261  * @assigned_cs:        bitmask describing already assigned CS lines
 262  * @clk_rate:           NAND controller current clock rate
 263  * @chips:              a list containing all the NAND chips attached to
 264  *                      this NAND controller
 265  * @complete:           a completion object used to wait for NAND
 266  *                      controller events
 267  */
 268 struct sunxi_nfc {
 269         struct nand_hw_control controller;
 270         struct device *dev;
 271         void __iomem *regs;
 272         struct clk *ahb_clk;
 273         struct clk *mod_clk;
 274         struct reset_control *reset;
 275         unsigned long assigned_cs;
 276         unsigned long clk_rate;
 277         struct list_head chips;
 278         struct completion complete;
 279         struct dma_chan *dmac;
 280 };
 281
 282 static inline struct sunxi_nfc *to_sunxi_nfc(struct nand_hw_control *ctrl)
 283 {
 284         return container_of(ctrl, struct sunxi_nfc, controller);
 285 }
 286
 287 static irqreturn_t sunxi_nfc_interrupt(int irq, void *dev_id)
 288 {
 289         struct sunxi_nfc *nfc = dev_id;
 290         u32 st = readl(nfc->regs + NFC_REG_ST);
 291         u32 ien = readl(nfc->regs + NFC_REG_INT);
 292
 293         if (!(ien & st))
 294                 return IRQ_NONE;
 295
 296         if ((ien & st) == ien)
 297                 complete(&nfc->complete);
 298
 299         writel(st & NFC_INT_MASK, nfc->regs + NFC_REG_ST);
 300         writel(~st & ien & NFC_INT_MASK, nfc->regs + NFC_REG_INT);
 301
 302         return IRQ_HANDLED;
 303 }
 304
 305 static int sunxi_nfc_wait_events(struct sunxi_nfc *nfc, u32 events,
 306                                  bool use_polling, unsigned int timeout_ms)
 307 {
 308         int ret;
 309
 310         if (events & ~NFC_INT_MASK)
 311                 return -EINVAL;
 312
 313         if (!timeout_ms)
 314                 timeout_ms = NFC_DEFAULT_TIMEOUT_MS;
 315
 316         if (!use_polling) {
 317                 init_completion(&nfc->complete);
 318
 319                 writel(events, nfc->regs + NFC_REG_INT);
 320
 321                 ret = wait_for_completion_timeout(&nfc->complete,
 322                                                 msecs_to_jiffies(timeout_ms));
 323                 if (!ret)
 324                         ret = -ETIMEDOUT;
 325                 else
 326                         ret = 0;
 327
 328                 writel(0, nfc->regs + NFC_REG_INT);
 329         } else {
 330                 u32 status;
 331
 332                 ret = readl_poll_timeout(nfc->regs + NFC_REG_ST, status,
 333                                          (status & events) == events, 1,
 334                                          timeout_ms * 1000);
 335         }
 336
 337         writel(events & NFC_INT_MASK, nfc->regs + NFC_REG_ST);
 338
 339         if (ret)
 340                 dev_err(nfc->dev, "wait interrupt timedout\n");
 341
 342         return ret;
 343 }
 344
 345 static int sunxi_nfc_wait_cmd_fifo_empty(struct sunxi_nfc *nfc)
 346 {
 347         u32 status;
 348         int ret;
 349
 350         ret = readl_poll_timeout(nfc->regs + NFC_REG_ST, status,
 351                                  !(status & NFC_CMD_FIFO_STATUS), 1,
 352                                  NFC_DEFAULT_TIMEOUT_MS * 1000);
 353         if (ret)
 354                 dev_err(nfc->dev, "wait for empty cmd FIFO timedout\n");
 355
 356         return ret;
 357 }
 358
 359 static int sunxi_nfc_rst(struct sunxi_nfc *nfc)
 360 {
 361         u32 ctl;
 362         int ret;
 363
 364         writel(0, nfc->regs + NFC_REG_ECC_CTL);
 365         writel(NFC_RESET, nfc->regs + NFC_REG_CTL);
 366
 367         ret = readl_poll_timeout(nfc->regs + NFC_REG_CTL, ctl,
 368                                  !(ctl & NFC_RESET), 1,
 369                                  NFC_DEFAULT_TIMEOUT_MS * 1000);
 370         if (ret)
 371                 dev_err(nfc->dev, "wait for NAND controller reset timedout\n");
 372
 373         return ret;
 374 }
 375
 376 static int sunxi_nfc_dma_op_prepare(struct mtd_info *mtd, const void *buf,
 377                                     int chunksize, int nchunks,
 378                                     enum dma_data_direction ddir,
 379                                     struct scatterlist *sg)
 380 {
 381         struct nand_chip *nand = mtd_to_nand(mtd);
 382         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 383         struct dma_async_tx_descriptor *dmad;
 384         enum dma_transfer_direction tdir;
 385         dma_cookie_t dmat;
 386         int ret;
 387
 388         if (ddir == DMA_FROM_DEVICE)
 389                 tdir = DMA_DEV_TO_MEM;
 390         else
 391                 tdir = DMA_MEM_TO_DEV;
 392
 393         sg_init_one(sg, buf, nchunks * chunksize);
 394         ret = dma_map_sg(nfc->dev, sg, 1, ddir);
 395         if (!ret)
 396                 return -ENOMEM;
 397
 398         dmad = dmaengine_prep_slave_sg(nfc->dmac, sg, 1, tdir, DMA_CTRL_ACK);
 399         if (!dmad) {
 400                 ret = -EINVAL;
 401                 goto err_unmap_buf;
 402         }
 403
 404         writel(readl(nfc->regs + NFC_REG_CTL) | NFC_RAM_METHOD,
 405                nfc->regs + NFC_REG_CTL);
 406         writel(nchunks, nfc->regs + NFC_REG_SECTOR_NUM);
 407         writel(chunksize, nfc->regs + NFC_REG_CNT);
 408         dmat = dmaengine_submit(dmad);
 409
 410         ret = dma_submit_error(dmat);
 411         if (ret)
 412                 goto err_clr_dma_flag;
 413
 414         return 0;
 415
 416 err_clr_dma_flag:
 417         writel(readl(nfc->regs + NFC_REG_CTL) & ~NFC_RAM_METHOD,
 418                nfc->regs + NFC_REG_CTL);
 419
 420 err_unmap_buf:
 421         dma_unmap_sg(nfc->dev, sg, 1, ddir);
 422         return ret;
 423 }
 424
 425 static void sunxi_nfc_dma_op_cleanup(struct mtd_info *mtd,
 426                                      enum dma_data_direction ddir,
 427                                      struct scatterlist *sg)
 428 {
 429         struct nand_chip *nand = mtd_to_nand(mtd);
 430         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 431
 432         dma_unmap_sg(nfc->dev, sg, 1, ddir);
 433         writel(readl(nfc->regs + NFC_REG_CTL) & ~NFC_RAM_METHOD,
 434                nfc->regs + NFC_REG_CTL);
 435 }
 436
 437 static int sunxi_nfc_dev_ready(struct mtd_info *mtd)
 438 {
 439         struct nand_chip *nand = mtd_to_nand(mtd);
 440         struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
 441         struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
 442         struct sunxi_nand_rb *rb;
 443         int ret;
 444
 445         if (sunxi_nand->selected < 0)
 446                 return 0;
 447
 448         rb = &sunxi_nand->sels[sunxi_nand->selected].rb;
 449
 450         switch (rb->type) {
 451         case RB_NATIVE:
 452                 ret = !!(readl(nfc->regs + NFC_REG_ST) &
 453                          NFC_RB_STATE(rb->info.nativeid));
 454                 break;
 455         case RB_GPIO:
 456                 ret = gpio_get_value(rb->info.gpio);
 457                 break;
 458         case RB_NONE:
 459         default:
 460                 ret = 0;
 461                 dev_err(nfc->dev, "cannot check R/B NAND status!\n");
 462                 break;
 463         }
 464
 465         return ret;
 466 }
 467
 468 static void sunxi_nfc_select_chip(struct mtd_info *mtd, int chip)
 469 {
 470         struct nand_chip *nand = mtd_to_nand(mtd);
 471         struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
 472         struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
 473         struct sunxi_nand_chip_sel *sel;
 474         u32 ctl;
 475
 476         if (chip > 0 && chip >= sunxi_nand->nsels)
 477                 return;
 478
 479         if (chip == sunxi_nand->selected)
 480                 return;
 481
 482         ctl = readl(nfc->regs + NFC_REG_CTL) &
 483               ~(NFC_PAGE_SHIFT_MSK | NFC_CE_SEL_MSK | NFC_RB_SEL_MSK | NFC_EN);
 484
 485         if (chip >= 0) {
 486                 sel = &sunxi_nand->sels[chip];
 487
 488                 ctl |= NFC_CE_SEL(sel->cs) | NFC_EN |
 489                        NFC_PAGE_SHIFT(nand->page_shift);
 490                 if (sel->rb.type == RB_NONE) {
 491                         nand->dev_ready = NULL;
 492                 } else {
 493                         nand->dev_ready = sunxi_nfc_dev_ready;
 494                         if (sel->rb.type == RB_NATIVE)
 495                                 ctl |= NFC_RB_SEL(sel->rb.info.nativeid);
 496                 }
 497
 498                 writel(mtd->writesize, nfc->regs + NFC_REG_SPARE_AREA);
 499
 500                 if (nfc->clk_rate != sunxi_nand->clk_rate) {
 501                         clk_set_rate(nfc->mod_clk, sunxi_nand->clk_rate);
 502                         nfc->clk_rate = sunxi_nand->clk_rate;
 503                 }
 504         }
 505
 506         writel(sunxi_nand->timing_ctl, nfc->regs + NFC_REG_TIMING_CTL);
 507         writel(sunxi_nand->timing_cfg, nfc->regs + NFC_REG_TIMING_CFG);
 508         writel(ctl, nfc->regs + NFC_REG_CTL);
 509
 510         sunxi_nand->selected = chip;
 511 }
 512
 513 static void sunxi_nfc_read_buf(struct mtd_info *mtd, uint8_t *buf, int len)
 514 {
 515         struct nand_chip *nand = mtd_to_nand(mtd);
 516         struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
 517         struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
 518         int ret;
 519         int cnt;
 520         int offs = 0;
 521         u32 tmp;
 522
 523         while (len > offs) {
 524                 cnt = min(len - offs, NFC_SRAM_SIZE);
 525
 526                 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
 527                 if (ret)
 528                         break;
 529
 530                 writel(cnt, nfc->regs + NFC_REG_CNT);
 531                 tmp = NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD;
 532                 writel(tmp, nfc->regs + NFC_REG_CMD);
 533
 534                 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, true, 0);
 535                 if (ret)
 536                         break;
 537
 538                 if (buf)
 539                         memcpy_fromio(buf + offs, nfc->regs + NFC_RAM0_BASE,
 540                                       cnt);
 541                 offs += cnt;
 542         }
 543 }
 544
 545 static void sunxi_nfc_write_buf(struct mtd_info *mtd, const uint8_t *buf,
 546                                 int len)
 547 {
 548         struct nand_chip *nand = mtd_to_nand(mtd);
 549         struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
 550         struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
 551         int ret;
 552         int cnt;
 553         int offs = 0;
 554         u32 tmp;
 555
 556         while (len > offs) {
 557                 cnt = min(len - offs, NFC_SRAM_SIZE);
 558
 559                 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
 560                 if (ret)
 561                         break;
 562
 563                 writel(cnt, nfc->regs + NFC_REG_CNT);
 564                 memcpy_toio(nfc->regs + NFC_RAM0_BASE, buf + offs, cnt);
 565                 tmp = NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD |
 566                       NFC_ACCESS_DIR;
 567                 writel(tmp, nfc->regs + NFC_REG_CMD);
 568
 569                 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, true, 0);
 570                 if (ret)
 571                         break;
 572
 573                 offs += cnt;
 574         }
 575 }
 576
 577 static uint8_t sunxi_nfc_read_byte(struct mtd_info *mtd)
 578 {
 579         uint8_t ret;
 580
 581         sunxi_nfc_read_buf(mtd, &ret, 1);
 582
 583         return ret;
 584 }
 585
 586 static void sunxi_nfc_cmd_ctrl(struct mtd_info *mtd, int dat,
 587                                unsigned int ctrl)
 588 {
 589         struct nand_chip *nand = mtd_to_nand(mtd);
 590         struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
 591         struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
 592         int ret;
 593
 594         ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
 595         if (ret)
 596                 return;
 597
 598         if (dat == NAND_CMD_NONE && (ctrl & NAND_NCE) &&
 599             !(ctrl & (NAND_CLE | NAND_ALE))) {
 600                 u32 cmd = 0;
 601
 602                 if (!sunxi_nand->addr_cycles && !sunxi_nand->cmd_cycles)
 603                         return;
 604
 605                 if (sunxi_nand->cmd_cycles--)
 606                         cmd |= NFC_SEND_CMD1 | sunxi_nand->cmd[0];
 607
 608                 if (sunxi_nand->cmd_cycles--) {
 609                         cmd |= NFC_SEND_CMD2;
 610                         writel(sunxi_nand->cmd[1],
 611                                nfc->regs + NFC_REG_RCMD_SET);
 612                 }
 613
 614                 sunxi_nand->cmd_cycles = 0;
 615
 616                 if (sunxi_nand->addr_cycles) {
 617                         cmd |= NFC_SEND_ADR |
 618                                NFC_ADR_NUM(sunxi_nand->addr_cycles);
 619                         writel(sunxi_nand->addr[0],
 620                                nfc->regs + NFC_REG_ADDR_LOW);
 621                 }
 622
 623                 if (sunxi_nand->addr_cycles > 4)
 624                         writel(sunxi_nand->addr[1],
 625                                nfc->regs + NFC_REG_ADDR_HIGH);
 626
 627                 writel(cmd, nfc->regs + NFC_REG_CMD);
 628                 sunxi_nand->addr[0] = 0;
 629                 sunxi_nand->addr[1] = 0;
 630                 sunxi_nand->addr_cycles = 0;
 631                 sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, true, 0);
 632         }
 633
 634         if (ctrl & NAND_CLE) {
 635                 sunxi_nand->cmd[sunxi_nand->cmd_cycles++] = dat;
 636         } else if (ctrl & NAND_ALE) {
 637                 sunxi_nand->addr[sunxi_nand->addr_cycles / 4] |=
 638                                 dat << ((sunxi_nand->addr_cycles % 4) * 8);
 639                 sunxi_nand->addr_cycles++;
 640         }
 641 }
 642
 643 /* These seed values have been extracted from Allwinner's BSP */
 644 static const u16 sunxi_nfc_randomizer_page_seeds[] = {
 645         0x2b75, 0x0bd0, 0x5ca3, 0x62d1, 0x1c93, 0x07e9, 0x2162, 0x3a72,
 646         0x0d67, 0x67f9, 0x1be7, 0x077d, 0x032f, 0x0dac, 0x2716, 0x2436,
 647         0x7922, 0x1510, 0x3860, 0x5287, 0x480f, 0x4252, 0x1789, 0x5a2d,
 648         0x2a49, 0x5e10, 0x437f, 0x4b4e, 0x2f45, 0x216e, 0x5cb7, 0x7130,
 649         0x2a3f, 0x60e4, 0x4dc9, 0x0ef0, 0x0f52, 0x1bb9, 0x6211, 0x7a56,
 650         0x226d, 0x4ea7, 0x6f36, 0x3692, 0x38bf, 0x0c62, 0x05eb, 0x4c55,
 651         0x60f4, 0x728c, 0x3b6f, 0x2037, 0x7f69, 0x0936, 0x651a, 0x4ceb,
 652         0x6218, 0x79f3, 0x383f, 0x18d9, 0x4f05, 0x5c82, 0x2912, 0x6f17,
 653         0x6856, 0x5938, 0x1007, 0x61ab, 0x3e7f, 0x57c2, 0x542f, 0x4f62,
 654         0x7454, 0x2eac, 0x7739, 0x42d4, 0x2f90, 0x435a, 0x2e52, 0x2064,
 655         0x637c, 0x66ad, 0x2c90, 0x0bad, 0x759c, 0x0029, 0x0986, 0x7126,
 656         0x1ca7, 0x1605, 0x386a, 0x27f5, 0x1380, 0x6d75, 0x24c3, 0x0f8e,
 657         0x2b7a, 0x1418, 0x1fd1, 0x7dc1, 0x2d8e, 0x43af, 0x2267, 0x7da3,
 658         0x4e3d, 0x1338, 0x50db, 0x454d, 0x764d, 0x40a3, 0x42e6, 0x262b,
 659         0x2d2e, 0x1aea, 0x2e17, 0x173d, 0x3a6e, 0x71bf, 0x25f9, 0x0a5d,
 660         0x7c57, 0x0fbe, 0x46ce, 0x4939, 0x6b17, 0x37bb, 0x3e91, 0x76db,
 661 };
 662
 663 /*
 664  * sunxi_nfc_randomizer_ecc512_seeds and sunxi_nfc_randomizer_ecc1024_seeds
 665  * have been generated using
 666  * sunxi_nfc_randomizer_step(seed, (step_size * 8) + 15), which is what
 667  * the randomizer engine does internally before de/scrambling OOB data.
 668  *
 669  * Those tables are statically defined to avoid calculating randomizer state
 670  * at runtime.
 671  */
 672 static const u16 sunxi_nfc_randomizer_ecc512_seeds[] = {
 673         0x3346, 0x367f, 0x1f18, 0x769a, 0x4f64, 0x068c, 0x2ef1, 0x6b64,
 674         0x28a9, 0x15d7, 0x30f8, 0x3659, 0x53db, 0x7c5f, 0x71d4, 0x4409,
 675         0x26eb, 0x03cc, 0x655d, 0x47d4, 0x4daa, 0x0877, 0x712d, 0x3617,
 676         0x3264, 0x49aa, 0x7f9e, 0x588e, 0x4fbc, 0x7176, 0x7f91, 0x6c6d,
 677         0x4b95, 0x5fb7, 0x3844, 0x4037, 0x0184, 0x081b, 0x0ee8, 0x5b91,
 678         0x293d, 0x1f71, 0x0e6f, 0x402b, 0x5122, 0x1e52, 0x22be, 0x3d2d,
 679         0x75bc, 0x7c60, 0x6291, 0x1a2f, 0x61d4, 0x74aa, 0x4140, 0x29ab,
 680         0x472d, 0x2852, 0x017e, 0x15e8, 0x5ec2, 0x17cf, 0x7d0f, 0x06b8,
 681         0x117a, 0x6b94, 0x789b, 0x3126, 0x6ac5, 0x5be7, 0x150f, 0x51f8,
 682         0x7889, 0x0aa5, 0x663d, 0x77e8, 0x0b87, 0x3dcb, 0x360d, 0x218b,
 683         0x512f, 0x7dc9, 0x6a4d, 0x630a, 0x3547, 0x1dd2, 0x5aea, 0x69a5,
 684         0x7bfa, 0x5e4f, 0x1519, 0x6430, 0x3a0e, 0x5eb3, 0x5425, 0x0c7a,
 685         0x5540, 0x3670, 0x63c1, 0x31e9, 0x5a39, 0x2de7, 0x5979, 0x2891,
 686         0x1562, 0x014b, 0x5b05, 0x2756, 0x5a34, 0x13aa, 0x6cb5, 0x2c36,
 687         0x5e72, 0x1306, 0x0861, 0x15ef, 0x1ee8, 0x5a37, 0x7ac4, 0x45dd,
 688         0x44c4, 0x7266, 0x2f41, 0x3ccc, 0x045e, 0x7d40, 0x7c66, 0x0fa0,
 689 };
 690
 691 static const u16 sunxi_nfc_randomizer_ecc1024_seeds[] = {
 692         0x2cf5, 0x35f1, 0x63a4, 0x5274, 0x2bd2, 0x778b, 0x7285, 0x32b6,
 693         0x6a5c, 0x70d6, 0x757d, 0x6769, 0x5375, 0x1e81, 0x0cf3, 0x3982,
 694         0x6787, 0x042a, 0x6c49, 0x1925, 0x56a8, 0x40a9, 0x063e, 0x7bd9,
 695         0x4dbf, 0x55ec, 0x672e, 0x7334, 0x5185, 0x4d00, 0x232a, 0x7e07,
 696         0x445d, 0x6b92, 0x528f, 0x4255, 0x53ba, 0x7d82, 0x2a2e, 0x3a4e,
 697         0x75eb, 0x450c, 0x6844, 0x1b5d, 0x581a, 0x4cc6, 0x0379, 0x37b2,
 698         0x419f, 0x0e92, 0x6b27, 0x5624, 0x01e3, 0x07c1, 0x44a5, 0x130c,
 699         0x13e8, 0x5910, 0x0876, 0x60c5, 0x54e3, 0x5b7f, 0x2269, 0x509f,
 700         0x7665, 0x36fd, 0x3e9a, 0x0579, 0x6295, 0x14ef, 0x0a81, 0x1bcc,
 701         0x4b16, 0x64db, 0x0514, 0x4f07, 0x0591, 0x3576, 0x6853, 0x0d9e,
 702         0x259f, 0x38b7, 0x64fb, 0x3094, 0x4693, 0x6ddd, 0x29bb, 0x0bc8,
 703         0x3f47, 0x490e, 0x0c0e, 0x7933, 0x3c9e, 0x5840, 0x398d, 0x3e68,
 704         0x4af1, 0x71f5, 0x57cf, 0x1121, 0x64eb, 0x3579, 0x15ac, 0x584d,
 705         0x5f2a, 0x47e2, 0x6528, 0x6eac, 0x196e, 0x6b96, 0x0450, 0x0179,
 706         0x609c, 0x06e1, 0x4626, 0x42c7, 0x273e, 0x486f, 0x0705, 0x1601,
 707         0x145b, 0x407e, 0x062b, 0x57a5, 0x53f9, 0x5659, 0x4410, 0x3ccd,
 708 };
 709
 710 static u16 sunxi_nfc_randomizer_step(u16 state, int count)
 711 {
 712         state &= 0x7fff;
 713
 714         /*
 715          * This loop is just a simple implementation of a Fibonacci LFSR using
 716          * the x16 + x15 + 1 polynomial.
 717          */
 718         while (count--)
 719                 state = ((state >> 1) |
 720                          (((state ^ (state >> 1)) & 1) << 14)) & 0x7fff;
 721
 722         return state;
 723 }
 724
 725 static u16 sunxi_nfc_randomizer_state(struct mtd_info *mtd, int page, bool ecc)
 726 {
 727         const u16 *seeds = sunxi_nfc_randomizer_page_seeds;
 728         int mod = mtd_div_by_ws(mtd->erasesize, mtd);
 729
 730         if (mod > ARRAY_SIZE(sunxi_nfc_randomizer_page_seeds))
 731                 mod = ARRAY_SIZE(sunxi_nfc_randomizer_page_seeds);
 732
 733         if (ecc) {
 734                 if (mtd->ecc_step_size == 512)
 735                         seeds = sunxi_nfc_randomizer_ecc512_seeds;
 736                 else
 737                         seeds = sunxi_nfc_randomizer_ecc1024_seeds;
 738         }
 739
 740         return seeds[page % mod];
 741 }
 742
 743 static void sunxi_nfc_randomizer_config(struct mtd_info *mtd,
 744                                         int page, bool ecc)
 745 {
 746         struct nand_chip *nand = mtd_to_nand(mtd);
 747         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 748         u32 ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL);
 749         u16 state;
 750
 751         if (!(nand->options & NAND_NEED_SCRAMBLING))
 752                 return;
 753
 754         ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL);
 755         state = sunxi_nfc_randomizer_state(mtd, page, ecc);
 756         ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_RANDOM_SEED_MSK;
 757         writel(ecc_ctl | NFC_RANDOM_SEED(state), nfc->regs + NFC_REG_ECC_CTL);
 758 }
 759
 760 static void sunxi_nfc_randomizer_enable(struct mtd_info *mtd)
 761 {
 762         struct nand_chip *nand = mtd_to_nand(mtd);
 763         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 764
 765         if (!(nand->options & NAND_NEED_SCRAMBLING))
 766                 return;
 767
 768         writel(readl(nfc->regs + NFC_REG_ECC_CTL) | NFC_RANDOM_EN,
 769                nfc->regs + NFC_REG_ECC_CTL);
 770 }
 771
 772 static void sunxi_nfc_randomizer_disable(struct mtd_info *mtd)
 773 {
 774         struct nand_chip *nand = mtd_to_nand(mtd);
 775         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 776
 777         if (!(nand->options & NAND_NEED_SCRAMBLING))
 778                 return;
 779
 780         writel(readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_RANDOM_EN,
 781                nfc->regs + NFC_REG_ECC_CTL);
 782 }
 783
 784 static void sunxi_nfc_randomize_bbm(struct mtd_info *mtd, int page, u8 *bbm)
 785 {
 786         u16 state = sunxi_nfc_randomizer_state(mtd, page, true);
 787
 788         bbm[0] ^= state;
 789         bbm[1] ^= sunxi_nfc_randomizer_step(state, 8);
 790 }
 791
 792 static void sunxi_nfc_randomizer_write_buf(struct mtd_info *mtd,
 793                                            const uint8_t *buf, int len,
 794                                            bool ecc, int page)
 795 {
 796         sunxi_nfc_randomizer_config(mtd, page, ecc);
 797         sunxi_nfc_randomizer_enable(mtd);
 798         sunxi_nfc_write_buf(mtd, buf, len);
 799         sunxi_nfc_randomizer_disable(mtd);
 800 }
 801
 802 static void sunxi_nfc_randomizer_read_buf(struct mtd_info *mtd, uint8_t *buf,
 803                                           int len, bool ecc, int page)
 804 {
 805         sunxi_nfc_randomizer_config(mtd, page, ecc);
 806         sunxi_nfc_randomizer_enable(mtd);
 807         sunxi_nfc_read_buf(mtd, buf, len);
 808         sunxi_nfc_randomizer_disable(mtd);
 809 }
 810
 811 static void sunxi_nfc_hw_ecc_enable(struct mtd_info *mtd)
 812 {
 813         struct nand_chip *nand = mtd_to_nand(mtd);
 814         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 815         struct sunxi_nand_hw_ecc *data = nand->ecc.priv;
 816         u32 ecc_ctl;
 817
 818         ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL);
 819         ecc_ctl &= ~(NFC_ECC_MODE_MSK | NFC_ECC_PIPELINE |
 820                      NFC_ECC_BLOCK_SIZE_MSK);
 821         ecc_ctl |= NFC_ECC_EN | NFC_ECC_MODE(data->mode) | NFC_ECC_EXCEPTION |
 822                    NFC_ECC_PIPELINE;
 823
 824         writel(ecc_ctl, nfc->regs + NFC_REG_ECC_CTL);
 825 }
 826
 827 static void sunxi_nfc_hw_ecc_disable(struct mtd_info *mtd)
 828 {
 829         struct nand_chip *nand = mtd_to_nand(mtd);
 830         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 831
 832         writel(readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_ECC_EN,
 833                nfc->regs + NFC_REG_ECC_CTL);
 834 }
 835
 836 static inline void sunxi_nfc_user_data_to_buf(u32 user_data, u8 *buf)
 837 {
 838         buf[0] = user_data;
 839         buf[1] = user_data >> 8;
 840         buf[2] = user_data >> 16;
 841         buf[3] = user_data >> 24;
 842 }
 843
 844 static inline u32 sunxi_nfc_buf_to_user_data(const u8 *buf)
 845 {
 846         return buf[0] | (buf[1] << 8) | (buf[2] << 16) | (buf[3] << 24);
 847 }
 848
 849 static void sunxi_nfc_hw_ecc_get_prot_oob_bytes(struct mtd_info *mtd, u8 *oob,
 850                                                 int step, bool bbm, int page)
 851 {
 852         struct nand_chip *nand = mtd_to_nand(mtd);
 853         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 854
 855         sunxi_nfc_user_data_to_buf(readl(nfc->regs + NFC_REG_USER_DATA(step)),
 856                                    oob);
 857
 858         /* De-randomize the Bad Block Marker. */
 859         if (bbm && (nand->options & NAND_NEED_SCRAMBLING))
 860                 sunxi_nfc_randomize_bbm(mtd, page, oob);
 861 }
 862
 863 static void sunxi_nfc_hw_ecc_set_prot_oob_bytes(struct mtd_info *mtd,
 864                                                 const u8 *oob, int step,
 865                                                 bool bbm, int page)
 866 {
 867         struct nand_chip *nand = mtd_to_nand(mtd);
 868         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 869         u8 user_data[4];
 870
 871         /* Randomize the Bad Block Marker. */
 872         if (bbm && (nand->options & NAND_NEED_SCRAMBLING)) {
 873                 memcpy(user_data, oob, sizeof(user_data));
 874                 sunxi_nfc_randomize_bbm(mtd, page, user_data);
 875                 oob = user_data;
 876         }
 877
 878         writel(sunxi_nfc_buf_to_user_data(oob),
 879                nfc->regs + NFC_REG_USER_DATA(step));
 880 }
 881
 882 static void sunxi_nfc_hw_ecc_update_stats(struct mtd_info *mtd,
 883                                           unsigned int *max_bitflips, int ret)
 884 {
 885         if (ret < 0) {
 886                 mtd->ecc_stats.failed++;
 887         } else {
 888                 mtd->ecc_stats.corrected += ret;
 889                 *max_bitflips = max_t(unsigned int, *max_bitflips, ret);
 890         }
 891 }
 892
 893 static int sunxi_nfc_hw_ecc_correct(struct mtd_info *mtd, u8 *data, u8 *oob,
 894                                     int step, u32 status, bool *erased)
 895 {
 896         struct nand_chip *nand = mtd_to_nand(mtd);
 897         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 898         struct nand_ecc_ctrl *ecc = &nand->ecc;
 899         u32 tmp;
 900
 901         *erased = false;
 902
 903         if (status & NFC_ECC_ERR(step))
 904                 return -EBADMSG;
 905
 906         if (status & NFC_ECC_PAT_FOUND(step)) {
 907                 u8 pattern;
 908
 909                 if (unlikely(!(readl(nfc->regs + NFC_REG_PAT_ID) & 0x1))) {
 910                         pattern = 0x0;
 911                 } else {
 912                         pattern = 0xff;
 913                         *erased = true;
 914                 }
 915
 916                 if (data)
 917                         memset(data, pattern, ecc->size);
 918
 919                 if (oob)
 920                         memset(oob, pattern, ecc->bytes + 4);
 921
 922                 return 0;
 923         }
 924
 925         tmp = readl(nfc->regs + NFC_REG_ECC_ERR_CNT(step));
 926
 927         return NFC_ECC_ERR_CNT(step, tmp);
 928 }
 929
 930 static int sunxi_nfc_hw_ecc_read_chunk(struct mtd_info *mtd,
 931                                        u8 *data, int data_off,
 932                                        u8 *oob, int oob_off,
 933                                        int *cur_off,
 934                                        unsigned int *max_bitflips,
 935                                        bool bbm, bool oob_required, int page)
 936 {
 937         struct nand_chip *nand = mtd_to_nand(mtd);
 938         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 939         struct nand_ecc_ctrl *ecc = &nand->ecc;
 940         int raw_mode = 0;
 941         bool erased;
 942         int ret;
 943
 944         if (*cur_off != data_off)
 945                 nand->cmdfunc(mtd, NAND_CMD_RNDOUT, data_off, -1);
 946
 947         sunxi_nfc_randomizer_read_buf(mtd, NULL, ecc->size, false, page);
 948
 949         if (data_off + ecc->size != oob_off)
 950                 nand->cmdfunc(mtd, NAND_CMD_RNDOUT, oob_off, -1);
 951
 952         ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
 953         if (ret)
 954                 return ret;
 955
 956         sunxi_nfc_randomizer_enable(mtd);
 957         writel(NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD | NFC_ECC_OP,
 958                nfc->regs + NFC_REG_CMD);
 959
 960         ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, true, 0);
 961         sunxi_nfc_randomizer_disable(mtd);
 962         if (ret)
 963                 return ret;
 964
 965         *cur_off = oob_off + ecc->bytes + 4;
 966
 967         ret = sunxi_nfc_hw_ecc_correct(mtd, data, oob_required ? oob : NULL, 0,
 968                                        readl(nfc->regs + NFC_REG_ECC_ST),
 969                                        &erased);
 970         if (erased)
 971                 return 1;
 972
 973         if (ret < 0) {
 974                 /*
 975                  * Re-read the data with the randomizer disabled to identify
 976                  * bitflips in erased pages.
 977                  */
 978                 if (nand->options & NAND_NEED_SCRAMBLING) {
 979                         nand->cmdfunc(mtd, NAND_CMD_RNDOUT, data_off, -1);
 980                         nand->read_buf(mtd, data, ecc->size);
 981                 } else {
 982                         memcpy_fromio(data, nfc->regs + NFC_RAM0_BASE,
 983                                       ecc->size);
 984                 }
 985
 986                 nand->cmdfunc(mtd, NAND_CMD_RNDOUT, oob_off, -1);
 987                 nand->read_buf(mtd, oob, ecc->bytes + 4);
 988
 989                 ret = nand_check_erased_ecc_chunk(data, ecc->size,
 990                                                   oob, ecc->bytes + 4,
 991                                                   NULL, 0, ecc->strength);
 992                 if (ret >= 0)
 993                         raw_mode = 1;
 994         } else {
 995                 memcpy_fromio(data, nfc->regs + NFC_RAM0_BASE, ecc->size);
 996
 997                 if (oob_required) {
 998                         nand->cmdfunc(mtd, NAND_CMD_RNDOUT, oob_off, -1);
 999                         sunxi_nfc_randomizer_read_buf(mtd, oob, ecc->bytes + 4,
1000                                                       true, page);
1001
1002                         sunxi_nfc_hw_ecc_get_prot_oob_bytes(mtd, oob, 0,
1003                                                             bbm, page);
1004                 }
1005         }
1006
1007         sunxi_nfc_hw_ecc_update_stats(mtd, max_bitflips, ret);
1008
1009         return raw_mode;
1010 }
1011
1012 static void sunxi_nfc_hw_ecc_read_extra_oob(struct mtd_info *mtd,
1013                                             u8 *oob, int *cur_off,
1014                                             bool randomize, int page)
1015 {
1016         struct nand_chip *nand = mtd_to_nand(mtd);
1017         struct nand_ecc_ctrl *ecc = &nand->ecc;
1018         int offset = ((ecc->bytes + 4) * ecc->steps);
1019         int len = mtd->oobsize - offset;
1020
1021         if (len <= 0)
1022                 return;
1023
1024         if (!cur_off || *cur_off != offset)
1025                 nand->cmdfunc(mtd, NAND_CMD_RNDOUT,
1026                               offset + mtd->writesize, -1);
1027
1028         if (!randomize)
1029                 sunxi_nfc_read_buf(mtd, oob + offset, len);
1030         else
1031                 sunxi_nfc_randomizer_read_buf(mtd, oob + offset, len,
1032                                               false, page);
1033
1034         if (cur_off)
1035                 *cur_off = mtd->oobsize + mtd->writesize;
1036 }
1037
1038 static int sunxi_nfc_hw_ecc_read_chunks_dma(struct mtd_info *mtd, uint8_t *buf,
1039                                             int oob_required, int page,
1040                                             int nchunks)
1041 {
1042         struct nand_chip *nand = mtd_to_nand(mtd);
1043         bool randomized = nand->options & NAND_NEED_SCRAMBLING;
1044         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
1045         struct nand_ecc_ctrl *ecc = &nand->ecc;
1046         unsigned int max_bitflips = 0;
1047         int ret, i, raw_mode = 0;
1048         struct scatterlist sg;
1049         u32 status;
1050
1051         ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
1052         if (ret)
1053                 return ret;
1054
1055         ret = sunxi_nfc_dma_op_prepare(mtd, buf, ecc->size, nchunks,
1056                                        DMA_FROM_DEVICE, &sg);
1057         if (ret)
1058                 return ret;
1059
1060         sunxi_nfc_hw_ecc_enable(mtd);
1061         sunxi_nfc_randomizer_config(mtd, page, false);
1062         sunxi_nfc_randomizer_enable(mtd);
1063
1064         writel((NAND_CMD_RNDOUTSTART << 16) | (NAND_CMD_RNDOUT << 8) |
1065                NAND_CMD_READSTART, nfc->regs + NFC_REG_RCMD_SET);
1066
1067         dma_async_issue_pending(nfc->dmac);
1068
1069         writel(NFC_PAGE_OP | NFC_DATA_SWAP_METHOD | NFC_DATA_TRANS,
1070                nfc->regs + NFC_REG_CMD);
1071
1072         ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, true, 0);
1073         if (ret)
1074                 dmaengine_terminate_all(nfc->dmac);
1075
1076         sunxi_nfc_randomizer_disable(mtd);
1077         sunxi_nfc_hw_ecc_disable(mtd);
1078
1079         sunxi_nfc_dma_op_cleanup(mtd, DMA_FROM_DEVICE, &sg);
1080
1081         if (ret)
1082                 return ret;
1083
1084         status = readl(nfc->regs + NFC_REG_ECC_ST);
1085
1086         for (i = 0; i < nchunks; i++) {
1087                 int data_off = i * ecc->size;
1088                 int oob_off = i * (ecc->bytes + 4);
1089                 u8 *data = buf + data_off;
1090                 u8 *oob = nand->oob_poi + oob_off;
1091                 bool erased;
1092
1093                 ret = sunxi_nfc_hw_ecc_correct(mtd, randomized ? data : NULL,
1094                                                oob_required ? oob : NULL,
1095                                                i, status, &erased);
1096
1097                 /* ECC errors are handled in the second loop. */
1098                 if (ret < 0)
1099                         continue;
1100
1101                 if (oob_required && !erased) {
1102                         /* TODO: use DMA to retrieve OOB */
1103                         nand->cmdfunc(mtd, NAND_CMD_RNDOUT,
1104                                       mtd->writesize + oob_off, -1);
1105                         nand->read_buf(mtd, oob, ecc->bytes + 4);
1106
1107                         sunxi_nfc_hw_ecc_get_prot_oob_bytes(mtd, oob, i,
1108                                                             !i, page);
1109                 }
1110
1111                 if (erased)
1112                         raw_mode = 1;
1113
1114                 sunxi_nfc_hw_ecc_update_stats(mtd, &max_bitflips, ret);
1115         }
1116
1117         if (status & NFC_ECC_ERR_MSK) {
1118                 for (i = 0; i < nchunks; i++) {
1119                         int data_off = i * ecc->size;
1120                         int oob_off = i * (ecc->bytes + 4);
1121                         u8 *data = buf + data_off;
1122                         u8 *oob = nand->oob_poi + oob_off;
1123
1124                         if (!(status & NFC_ECC_ERR(i)))
1125                                 continue;
1126
1127                         /*
1128                          * Re-read the data with the randomizer disabled to
1129                          * identify bitflips in erased pages.
1130                          */
1131                         if (randomized) {
1132                                 /* TODO: use DMA to read page in raw mode */
1133                                 nand->cmdfunc(mtd, NAND_CMD_RNDOUT,
1134                                               data_off, -1);
1135                                 nand->read_buf(mtd, data, ecc->size);
1136                         }
1137
1138                         /* TODO: use DMA to retrieve OOB */
1139                         nand->cmdfunc(mtd, NAND_CMD_RNDOUT,
1140                                       mtd->writesize + oob_off, -1);
1141                         nand->read_buf(mtd, oob, ecc->bytes + 4);
1142
1143                         ret = nand_check_erased_ecc_chunk(data, ecc->size,
1144                                                           oob, ecc->bytes + 4,
1145                                                           NULL, 0,
1146                                                           ecc->strength);
1147                         if (ret >= 0)
1148                                 raw_mode = 1;
1149
1150                         sunxi_nfc_hw_ecc_update_stats(mtd, &max_bitflips, ret);
1151                 }
1152         }
1153
1154         if (oob_required)
1155                 sunxi_nfc_hw_ecc_read_extra_oob(mtd, nand->oob_poi,
1156                                                 NULL, !raw_mode,
1157                                                 page);
1158
1159         return max_bitflips;
1160 }
1161
1162 static int sunxi_nfc_hw_ecc_write_chunk(struct mtd_info *mtd,
1163                                         const u8 *data, int data_off,
1164                                         const u8 *oob, int oob_off,
1165                                         int *cur_off, bool bbm,
1166                                         int page)
1167 {
1168         struct nand_chip *nand = mtd_to_nand(mtd);
1169         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
1170         struct nand_ecc_ctrl *ecc = &nand->ecc;
1171         int ret;
1172
1173         if (data_off != *cur_off)
1174                 nand->cmdfunc(mtd, NAND_CMD_RNDIN, data_off, -1);
1175
1176         sunxi_nfc_randomizer_write_buf(mtd, data, ecc->size, false, page);
1177
1178         if (data_off + ecc->size != oob_off)
1179                 nand->cmdfunc(mtd, NAND_CMD_RNDIN, oob_off, -1);
1180
1181         ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
1182         if (ret)
1183                 return ret;
1184
1185         sunxi_nfc_randomizer_enable(mtd);
1186         sunxi_nfc_hw_ecc_set_prot_oob_bytes(mtd, oob, 0, bbm, page);
1187
1188         writel(NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD |
1189                NFC_ACCESS_DIR | NFC_ECC_OP,
1190                nfc->regs + NFC_REG_CMD);
1191
1192         ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, true, 0);
1193         sunxi_nfc_randomizer_disable(mtd);
1194         if (ret)
1195                 return ret;
1196
1197         *cur_off = oob_off + ecc->bytes + 4;
1198
1199         return 0;
1200 }
1201
1202 static void sunxi_nfc_hw_ecc_write_extra_oob(struct mtd_info *mtd,
1203                                              u8 *oob, int *cur_off,
1204                                              int page)
1205 {
1206         struct nand_chip *nand = mtd_to_nand(mtd);
1207         struct nand_ecc_ctrl *ecc = &nand->ecc;
1208         int offset = ((ecc->bytes + 4) * ecc->steps);
1209         int len = mtd->oobsize - offset;
1210
1211         if (len <= 0)
1212                 return;
1213
1214         if (!cur_off || *cur_off != offset)
1215                 nand->cmdfunc(mtd, NAND_CMD_RNDIN,
1216                               offset + mtd->writesize, -1);
1217
1218         sunxi_nfc_randomizer_write_buf(mtd, oob + offset, len, false, page);
1219
1220         if (cur_off)
1221                 *cur_off = mtd->oobsize + mtd->writesize;
1222 }
1223
1224 static int sunxi_nfc_hw_ecc_read_page(struct mtd_info *mtd,
1225                                       struct nand_chip *chip, uint8_t *buf,
1226                                       int oob_required, int page)
1227 {
1228         struct nand_ecc_ctrl *ecc = &chip->ecc;
1229         unsigned int max_bitflips = 0;
1230         int ret, i, cur_off = 0;
1231         bool raw_mode = false;
1232
1233         sunxi_nfc_hw_ecc_enable(mtd);
1234
1235         for (i = 0; i < ecc->steps; i++) {
1236                 int data_off = i * ecc->size;
1237                 int oob_off = i * (ecc->bytes + 4);
1238                 u8 *data = buf + data_off;
1239                 u8 *oob = chip->oob_poi + oob_off;
1240
1241                 ret = sunxi_nfc_hw_ecc_read_chunk(mtd, data, data_off, oob,
1242                                                   oob_off + mtd->writesize,
1243                                                   &cur_off, &max_bitflips,
1244                                                   !i, oob_required, page);
1245                 if (ret < 0)
1246                         return ret;
1247                 else if (ret)
1248                         raw_mode = true;
1249         }
1250
1251         if (oob_required)
1252                 sunxi_nfc_hw_ecc_read_extra_oob(mtd, chip->oob_poi, &cur_off,
1253                                                 !raw_mode, page);
1254
1255         sunxi_nfc_hw_ecc_disable(mtd);
1256
1257         return max_bitflips;
1258 }
1259
1260 static int sunxi_nfc_hw_ecc_read_page_dma(struct mtd_info *mtd,
1261                                           struct nand_chip *chip, u8 *buf,
1262                                           int oob_required, int page)
1263 {
1264         int ret;
1265
1266         ret = sunxi_nfc_hw_ecc_read_chunks_dma(mtd, buf, oob_required, page,
1267                                                chip->ecc.steps);
1268         if (ret >= 0)
1269                 return ret;
1270
1271         /* Fallback to PIO mode */
1272         chip->cmdfunc(mtd, NAND_CMD_RNDOUT, 0, -1);
1273
1274         return sunxi_nfc_hw_ecc_read_page(mtd, chip, buf, oob_required, page);
1275 }
1276
1277 static int sunxi_nfc_hw_ecc_read_subpage(struct mtd_info *mtd,
1278                                          struct nand_chip *chip,
1279                                          u32 data_offs, u32 readlen,
1280                                          u8 *bufpoi, int page)
1281 {
1282         struct nand_ecc_ctrl *ecc = &chip->ecc;
1283         int ret, i, cur_off = 0;
1284         unsigned int max_bitflips = 0;
1285
1286         sunxi_nfc_hw_ecc_enable(mtd);
1287
1288         chip->cmdfunc(mtd, NAND_CMD_READ0, 0, page);
1289         for (i = data_offs / ecc->size;
1290              i < DIV_ROUND_UP(data_offs + readlen, ecc->size); i++) {
1291                 int data_off = i * ecc->size;
1292                 int oob_off = i * (ecc->bytes + 4);
1293                 u8 *data = bufpoi + data_off;
1294                 u8 *oob = chip->oob_poi + oob_off;
1295
1296                 ret = sunxi_nfc_hw_ecc_read_chunk(mtd, data, data_off,
1297                                                   oob,
1298                                                   oob_off + mtd->writesize,
1299                                                   &cur_off, &max_bitflips, !i,
1300                                                   false, page);
1301                 if (ret < 0)
1302                         return ret;
1303         }
1304
1305         sunxi_nfc_hw_ecc_disable(mtd);
1306
1307         return max_bitflips;
1308 }
1309
1310 static int sunxi_nfc_hw_ecc_read_subpage_dma(struct mtd_info *mtd,
1311                                              struct nand_chip *chip,
1312                                              u32 data_offs, u32 readlen,
1313                                              u8 *buf, int page)
1314 {
1315         int nchunks = DIV_ROUND_UP(data_offs + readlen, chip->ecc.size);
1316         int ret;
1317
1318         ret = sunxi_nfc_hw_ecc_read_chunks_dma(mtd, buf, false, page, nchunks);
1319         if (ret >= 0)
1320                 return ret;
1321
1322         /* Fallback to PIO mode */
1323         chip->cmdfunc(mtd, NAND_CMD_RNDOUT, 0, -1);
1324
1325         return sunxi_nfc_hw_ecc_read_subpage(mtd, chip, data_offs, readlen,
1326                                              buf, page);
1327 }
1328
1329 static int sunxi_nfc_hw_ecc_write_page(struct mtd_info *mtd,
1330                                        struct nand_chip *chip,
1331                                        const uint8_t *buf, int oob_required,
1332                                        int page)
1333 {
1334         struct nand_ecc_ctrl *ecc = &chip->ecc;
1335         int ret, i, cur_off = 0;
1336
1337         sunxi_nfc_hw_ecc_enable(mtd);
1338
1339         for (i = 0; i < ecc->steps; i++) {
1340                 int data_off = i * ecc->size;
1341                 int oob_off = i * (ecc->bytes + 4);
1342                 const u8 *data = buf + data_off;
1343                 const u8 *oob = chip->oob_poi + oob_off;
1344
1345                 ret = sunxi_nfc_hw_ecc_write_chunk(mtd, data, data_off, oob,
1346                                                    oob_off + mtd->writesize,
1347                                                    &cur_off, !i, page);
1348                 if (ret)
1349                         return ret;
1350         }
1351
1352         if (oob_required || (chip->options & NAND_NEED_SCRAMBLING))
1353                 sunxi_nfc_hw_ecc_write_extra_oob(mtd, chip->oob_poi,
1354                                                  &cur_off, page);
1355
1356         sunxi_nfc_hw_ecc_disable(mtd);
1357
1358         return 0;
1359 }
1360
1361 static int sunxi_nfc_hw_ecc_write_subpage(struct mtd_info *mtd,
1362                                           struct nand_chip *chip,
1363                                           u32 data_offs, u32 data_len,
1364                                           const u8 *buf, int oob_required,
1365                                           int page)
1366 {
1367         struct nand_ecc_ctrl *ecc = &chip->ecc;
1368         int ret, i, cur_off = 0;
1369
1370         sunxi_nfc_hw_ecc_enable(mtd);
1371
1372         for (i = data_offs / ecc->size;
1373              i < DIV_ROUND_UP(data_offs + data_len, ecc->size); i++) {
1374                 int data_off = i * ecc->size;
1375                 int oob_off = i * (ecc->bytes + 4);
1376                 const u8 *data = buf + data_off;
1377                 const u8 *oob = chip->oob_poi + oob_off;
1378
1379                 ret = sunxi_nfc_hw_ecc_write_chunk(mtd, data, data_off, oob,
1380                                                    oob_off + mtd->writesize,
1381                                                    &cur_off, !i, page);
1382                 if (ret)
1383                         return ret;
1384         }
1385
1386         sunxi_nfc_hw_ecc_disable(mtd);
1387
1388         return 0;
1389 }
1390
1391 static int sunxi_nfc_hw_ecc_write_page_dma(struct mtd_info *mtd,
1392                                            struct nand_chip *chip,
1393                                            const u8 *buf,
1394                                            int oob_required,
1395                                            int page)
1396 {
1397         struct nand_chip *nand = mtd_to_nand(mtd);
1398         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
1399         struct nand_ecc_ctrl *ecc = &nand->ecc;
1400         struct scatterlist sg;
1401         int ret, i;
1402
1403         ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
1404         if (ret)
1405                 return ret;
1406
1407         ret = sunxi_nfc_dma_op_prepare(mtd, buf, ecc->size, ecc->steps,
1408                                        DMA_TO_DEVICE, &sg);
1409         if (ret)
1410                 goto pio_fallback;
1411
1412         for (i = 0; i < ecc->steps; i++) {
1413                 const u8 *oob = nand->oob_poi + (i * (ecc->bytes + 4));
1414
1415                 sunxi_nfc_hw_ecc_set_prot_oob_bytes(mtd, oob, i, !i, page);
1416         }
1417
1418         sunxi_nfc_hw_ecc_enable(mtd);
1419         sunxi_nfc_randomizer_config(mtd, page, false);
1420         sunxi_nfc_randomizer_enable(mtd);
1421
1422         writel((NAND_CMD_RNDIN << 8) | NAND_CMD_PAGEPROG,
1423                nfc->regs + NFC_REG_WCMD_SET);
1424
1425         dma_async_issue_pending(nfc->dmac);
1426
1427         writel(NFC_PAGE_OP | NFC_DATA_SWAP_METHOD |
1428                NFC_DATA_TRANS | NFC_ACCESS_DIR,
1429                nfc->regs + NFC_REG_CMD);
1430
1431         ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, true, 0);
1432         if (ret)
1433                 dmaengine_terminate_all(nfc->dmac);
1434
1435         sunxi_nfc_randomizer_disable(mtd);
1436         sunxi_nfc_hw_ecc_disable(mtd);
1437
1438         sunxi_nfc_dma_op_cleanup(mtd, DMA_TO_DEVICE, &sg);
1439
1440         if (ret)
1441                 return ret;
1442
1443         if (oob_required || (chip->options & NAND_NEED_SCRAMBLING))
1444                 /* TODO: use DMA to transfer extra OOB bytes ? */
1445                 sunxi_nfc_hw_ecc_write_extra_oob(mtd, chip->oob_poi,
1446                                                  NULL, page);
1447
1448         return 0;
1449
1450 pio_fallback:
1451         return sunxi_nfc_hw_ecc_write_page(mtd, chip, buf, oob_required, page);
1452 }
1453
1454 static int sunxi_nfc_hw_syndrome_ecc_read_page(struct mtd_info *mtd,
1455                                                struct nand_chip *chip,
1456                                                uint8_t *buf, int oob_required,
1457                                                int page)
1458 {
1459         struct nand_ecc_ctrl *ecc = &chip->ecc;
1460         unsigned int max_bitflips = 0;
1461         int ret, i, cur_off = 0;
1462         bool raw_mode = false;
1463
1464         sunxi_nfc_hw_ecc_enable(mtd);
1465
1466         for (i = 0; i < ecc->steps; i++) {
1467                 int data_off = i * (ecc->size + ecc->bytes + 4);
1468                 int oob_off = data_off + ecc->size;
1469                 u8 *data = buf + (i * ecc->size);
1470                 u8 *oob = chip->oob_poi + (i * (ecc->bytes + 4));
1471
1472                 ret = sunxi_nfc_hw_ecc_read_chunk(mtd, data, data_off, oob,
1473                                                   oob_off, &cur_off,
1474                                                   &max_bitflips, !i,
1475                                                   oob_required,
1476                                                   page);
1477                 if (ret < 0)
1478                         return ret;
1479                 else if (ret)
1480                         raw_mode = true;
1481         }
1482
1483         if (oob_required)
1484                 sunxi_nfc_hw_ecc_read_extra_oob(mtd, chip->oob_poi, &cur_off,
1485                                                 !raw_mode, page);
1486
1487         sunxi_nfc_hw_ecc_disable(mtd);
1488
1489         return max_bitflips;
1490 }
1491
1492 static int sunxi_nfc_hw_syndrome_ecc_write_page(struct mtd_info *mtd,
1493                                                 struct nand_chip *chip,
1494                                                 const uint8_t *buf,
1495                                                 int oob_required, int page)
1496 {
1497         struct nand_ecc_ctrl *ecc = &chip->ecc;
1498         int ret, i, cur_off = 0;
1499
1500         sunxi_nfc_hw_ecc_enable(mtd);
1501
1502         for (i = 0; i < ecc->steps; i++) {
1503                 int data_off = i * (ecc->size + ecc->bytes + 4);
1504                 int oob_off = data_off + ecc->size;
1505                 const u8 *data = buf + (i * ecc->size);
1506                 const u8 *oob = chip->oob_poi + (i * (ecc->bytes + 4));
1507
1508                 ret = sunxi_nfc_hw_ecc_write_chunk(mtd, data, data_off,
1509                                                    oob, oob_off, &cur_off,
1510                                                    false, page);
1511                 if (ret)
1512                         return ret;
1513         }
1514
1515         if (oob_required || (chip->options & NAND_NEED_SCRAMBLING))
1516                 sunxi_nfc_hw_ecc_write_extra_oob(mtd, chip->oob_poi,
1517                                                  &cur_off, page);
1518
1519         sunxi_nfc_hw_ecc_disable(mtd);
1520
1521         return 0;
1522 }
1523
1524 static int sunxi_nfc_hw_common_ecc_read_oob(struct mtd_info *mtd,
1525                                             struct nand_chip *chip,
1526                                             int page)
1527 {
1528         chip->cmdfunc(mtd, NAND_CMD_READ0, 0, page);
1529
1530         chip->pagebuf = -1;
1531
1532         return chip->ecc.read_page(mtd, chip, chip->buffers->databuf, 1, page);
1533 }
1534
1535 static int sunxi_nfc_hw_common_ecc_write_oob(struct mtd_info *mtd,
1536                                              struct nand_chip *chip,
1537                                              int page)
1538 {
1539         int ret, status;
1540
1541         chip->cmdfunc(mtd, NAND_CMD_SEQIN, 0, page);
1542
1543         chip->pagebuf = -1;
1544
1545         memset(chip->buffers->databuf, 0xff, mtd->writesize);
1546         ret = chip->ecc.write_page(mtd, chip, chip->buffers->databuf, 1, page);
1547         if (ret)
1548                 return ret;
1549
1550         /* Send command to program the OOB data */
1551         chip->cmdfunc(mtd, NAND_CMD_PAGEPROG, -1, -1);
1552
1553         status = chip->waitfunc(mtd, chip);
1554
1555         return status & NAND_STATUS_FAIL ? -EIO : 0;
1556 }
1557
1558 static const s32 tWB_lut[] = {6, 12, 16, 20};
1559 static const s32 tRHW_lut[] = {4, 8, 12, 20};
1560
1561 static int _sunxi_nand_lookup_timing(const s32 *lut, int lut_size, u32 duration,
1562                 u32 clk_period)
1563 {
1564         u32 clk_cycles = DIV_ROUND_UP(duration, clk_period);
1565         int i;
1566
1567         for (i = 0; i < lut_size; i++) {
1568                 if (clk_cycles <= lut[i])
1569                         return i;
1570         }
1571
1572         /* Doesn't fit */
1573         return -EINVAL;
1574 }
1575
1576 #define sunxi_nand_lookup_timing(l, p, c) \
1577                         _sunxi_nand_lookup_timing(l, ARRAY_SIZE(l), p, c)
1578
1579 static int sunxi_nfc_setup_data_interface(struct mtd_info *mtd,
1580                                         const struct nand_data_interface *conf,
1581                                         bool check_only)
1582 {
1583         struct nand_chip *nand = mtd_to_nand(mtd);
1584         struct sunxi_nand_chip *chip = to_sunxi_nand(nand);
1585         struct sunxi_nfc *nfc = to_sunxi_nfc(chip->nand.controller);
1586         const struct nand_sdr_timings *timings;
1587         u32 min_clk_period = 0;
1588         s32 tWB, tADL, tWHR, tRHW, tCAD;
1589         long real_clk_rate;
1590
1591         timings = nand_get_sdr_timings(conf);
1592         if (IS_ERR(timings))
1593                 return -ENOTSUPP;
1594
1595         /* T1 <=> tCLS */
1596         if (timings->tCLS_min > min_clk_period)
1597                 min_clk_period = timings->tCLS_min;
1598
1599         /* T2 <=> tCLH */
1600         if (timings->tCLH_min > min_clk_period)
1601                 min_clk_period = timings->tCLH_min;
1602
1603         /* T3 <=> tCS */
1604         if (timings->tCS_min > min_clk_period)
1605                 min_clk_period = timings->tCS_min;
1606
1607         /* T4 <=> tCH */
1608         if (timings->tCH_min > min_clk_period)
1609                 min_clk_period = timings->tCH_min;
1610
1611         /* T5 <=> tWP */
1612         if (timings->tWP_min > min_clk_period)
1613                 min_clk_period = timings->tWP_min;
1614
1615         /* T6 <=> tWH */
1616         if (timings->tWH_min > min_clk_period)
1617                 min_clk_period = timings->tWH_min;
1618
1619         /* T7 <=> tALS */
1620         if (timings->tALS_min > min_clk_period)
1621                 min_clk_period = timings->tALS_min;
1622
1623         /* T8 <=> tDS */
1624         if (timings->tDS_min > min_clk_period)
1625                 min_clk_period = timings->tDS_min;
1626
1627         /* T9 <=> tDH */
1628         if (timings->tDH_min > min_clk_period)
1629                 min_clk_period = timings->tDH_min;
1630
1631         /* T10 <=> tRR */
1632         if (timings->tRR_min > (min_clk_period * 3))
1633                 min_clk_period = DIV_ROUND_UP(timings->tRR_min, 3);
1634
1635         /* T11 <=> tALH */
1636         if (timings->tALH_min > min_clk_period)
1637                 min_clk_period = timings->tALH_min;
1638
1639         /* T12 <=> tRP */
1640         if (timings->tRP_min > min_clk_period)
1641                 min_clk_period = timings->tRP_min;
1642
1643         /* T13 <=> tREH */
1644         if (timings->tREH_min > min_clk_period)
1645                 min_clk_period = timings->tREH_min;
1646
1647         /* T14 <=> tRC */
1648         if (timings->tRC_min > (min_clk_period * 2))
1649                 min_clk_period = DIV_ROUND_UP(timings->tRC_min, 2);
1650
1651         /* T15 <=> tWC */
1652         if (timings->tWC_min > (min_clk_period * 2))
1653                 min_clk_period = DIV_ROUND_UP(timings->tWC_min, 2);
1654
1655         /* T16 - T19 + tCAD */
1656         if (timings->tWB_max > (min_clk_period * 20))
1657                 min_clk_period = DIV_ROUND_UP(timings->tWB_max, 20);
1658
1659         if (timings->tADL_min > (min_clk_period * 32))
1660                 min_clk_period = DIV_ROUND_UP(timings->tADL_min, 32);
1661
1662         if (timings->tWHR_min > (min_clk_period * 32))
1663                 min_clk_period = DIV_ROUND_UP(timings->tWHR_min, 32);
1664
1665         if (timings->tRHW_min > (min_clk_period * 20))
1666                 min_clk_period = DIV_ROUND_UP(timings->tRHW_min, 20);
1667
1668         tWB  = sunxi_nand_lookup_timing(tWB_lut, timings->tWB_max,
1669                                         min_clk_period);
1670         if (tWB < 0) {
1671                 dev_err(nfc->dev, "unsupported tWB\n");
1672                 return tWB;
1673         }
1674
1675         tADL = DIV_ROUND_UP(timings->tADL_min, min_clk_period) >> 3;
1676         if (tADL > 3) {
1677                 dev_err(nfc->dev, "unsupported tADL\n");
1678                 return -EINVAL;
1679         }
1680
1681         tWHR = DIV_ROUND_UP(timings->tWHR_min, min_clk_period) >> 3;
1682         if (tWHR > 3) {
1683                 dev_err(nfc->dev, "unsupported tWHR\n");
1684                 return -EINVAL;
1685         }
1686
1687         tRHW = sunxi_nand_lookup_timing(tRHW_lut, timings->tRHW_min,
1688                                         min_clk_period);
1689         if (tRHW < 0) {
1690                 dev_err(nfc->dev, "unsupported tRHW\n");
1691                 return tRHW;
1692         }
1693
1694         if (check_only)
1695                 return 0;
1696
1697         /*
1698          * TODO: according to ONFI specs this value only applies for DDR NAND,
1699          * but Allwinner seems to set this to 0x7. Mimic them for now.
1700          */
1701         tCAD = 0x7;
1702
1703         /* TODO: A83 has some more bits for CDQSS, CS, CLHZ, CCS, WC */
1704         chip->timing_cfg = NFC_TIMING_CFG(tWB, tADL, tWHR, tRHW, tCAD);
1705
1706         /* Convert min_clk_period from picoseconds to nanoseconds */
1707         min_clk_period = DIV_ROUND_UP(min_clk_period, 1000);
1708
1709         /*
1710          * Unlike what is stated in Allwinner datasheet, the clk_rate should
1711          * be set to (1 / min_clk_period), and not (2 / min_clk_period).
1712          * This new formula was verified with a scope and validated by
1713          * Allwinner engineers.
1714          */
1715         chip->clk_rate = NSEC_PER_SEC / min_clk_period;
1716         real_clk_rate = clk_round_rate(nfc->mod_clk, chip->clk_rate);
1717
1718         /*
1719          * ONFI specification 3.1, paragraph 4.15.2 dictates that EDO data
1720          * output cycle timings shall be used if the host drives tRC less than
1721          * 30 ns.
1722          */
1723         min_clk_period = NSEC_PER_SEC / real_clk_rate;
1724         chip->timing_ctl = ((min_clk_period * 2) < 30) ?
1725                            NFC_TIMING_CTL_EDO : 0;
1726
1727         return 0;
1728 }
1729
1730 static int sunxi_nand_ooblayout_ecc(struct mtd_info *mtd, int section,
1731                                     struct mtd_oob_region *oobregion)
1732 {
1733         struct nand_chip *nand = mtd_to_nand(mtd);
1734         struct nand_ecc_ctrl *ecc = &nand->ecc;
1735
1736         if (section >= ecc->steps)
1737                 return -ERANGE;
1738
1739         oobregion->offset = section * (ecc->bytes + 4) + 4;
1740         oobregion->length = ecc->bytes;
1741
1742         return 0;
1743 }
1744
1745 static int sunxi_nand_ooblayout_free(struct mtd_info *mtd, int section,
1746                                      struct mtd_oob_region *oobregion)
1747 {
1748         struct nand_chip *nand = mtd_to_nand(mtd);
1749         struct nand_ecc_ctrl *ecc = &nand->ecc;
1750
1751         if (section > ecc->steps)
1752                 return -ERANGE;
1753
1754         /*
1755          * The first 2 bytes are used for BB markers, hence we
1756          * only have 2 bytes available in the first user data
1757          * section.
1758          */
1759         if (!section && ecc->mode == NAND_ECC_HW) {
1760                 oobregion->offset = 2;
1761                 oobregion->length = 2;
1762
1763                 return 0;
1764         }
1765
1766         oobregion->offset = section * (ecc->bytes + 4);
1767
1768         if (section < ecc->steps)
1769                 oobregion->length = 4;
1770         else
1771                 oobregion->offset = mtd->oobsize - oobregion->offset;
1772
1773         return 0;
1774 }
1775
1776 static const struct mtd_ooblayout_ops sunxi_nand_ooblayout_ops = {
1777         .ecc = sunxi_nand_ooblayout_ecc,
1778         .free = sunxi_nand_ooblayout_free,
1779 };
1780
1781 static int sunxi_nand_hw_common_ecc_ctrl_init(struct mtd_info *mtd,
1782                                               struct nand_ecc_ctrl *ecc,
1783                                               struct device_node *np)
1784 {
1785         static const u8 strengths[] = { 16, 24, 28, 32, 40, 48, 56, 60, 64 };
1786         struct nand_chip *nand = mtd_to_nand(mtd);
1787         struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
1788         struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
1789         struct sunxi_nand_hw_ecc *data;
1790         int nsectors;
1791         int ret;
1792         int i;
1793
1794         if (ecc->options & NAND_ECC_MAXIMIZE) {
1795                 int bytes;
1796
1797                 ecc->size = 1024;
1798                 nsectors = mtd->writesize / ecc->size;
1799
1800                 /* Reserve 2 bytes for the BBM */
1801                 bytes = (mtd->oobsize - 2) / nsectors;
1802
1803                 /* 4 non-ECC bytes are added before each ECC bytes section */
1804                 bytes -= 4;
1805
1806                 /* and bytes has to be even. */
1807                 if (bytes % 2)
1808                         bytes--;
1809
1810                 ecc->strength = bytes * 8 / fls(8 * ecc->size);
1811
1812                 for (i = 0; i < ARRAY_SIZE(strengths); i++) {
1813                         if (strengths[i] > ecc->strength)
1814                                 break;
1815                 }
1816
1817                 if (!i)
1818                         ecc->strength = 0;
1819                 else
1820                         ecc->strength = strengths[i - 1];
1821         }
1822
1823         if (ecc->size != 512 && ecc->size != 1024)
1824                 return -EINVAL;
1825
1826         data = kzalloc(sizeof(*data), GFP_KERNEL);
1827         if (!data)
1828                 return -ENOMEM;
1829
1830         /* Prefer 1k ECC chunk over 512 ones */
1831         if (ecc->size == 512 && mtd->writesize > 512) {
1832                 ecc->size = 1024;
1833                 ecc->strength *= 2;
1834         }
1835
1836         /* Add ECC info retrieval from DT */
1837         for (i = 0; i < ARRAY_SIZE(strengths); i++) {
1838                 if (ecc->strength <= strengths[i]) {
1839                         /*
1840                          * Update ecc->strength value with the actual strength
1841                          * that will be used by the ECC engine.
1842                          */
1843                         ecc->strength = strengths[i];
1844                         break;
1845                 }
1846         }
1847
1848         if (i >= ARRAY_SIZE(strengths)) {
1849                 dev_err(nfc->dev, "unsupported strength\n");
1850                 ret = -ENOTSUPP;
1851                 goto err;
1852         }
1853
1854         data->mode = i;
1855
1856         /* HW ECC always request ECC bytes for 1024 bytes blocks */
1857         ecc->bytes = DIV_ROUND_UP(ecc->strength * fls(8 * 1024), 8);
1858
1859         /* HW ECC always work with even numbers of ECC bytes */
1860         ecc->bytes = ALIGN(ecc->bytes, 2);
1861
1862         nsectors = mtd->writesize / ecc->size;
1863
1864         if (mtd->oobsize < ((ecc->bytes + 4) * nsectors)) {
1865                 ret = -EINVAL;
1866                 goto err;
1867         }
1868
1869         ecc->read_oob = sunxi_nfc_hw_common_ecc_read_oob;
1870         ecc->write_oob = sunxi_nfc_hw_common_ecc_write_oob;
1871         mtd_set_ooblayout(mtd, &sunxi_nand_ooblayout_ops);
1872         ecc->priv = data;
1873
1874         return 0;
1875
1876 err:
1877         kfree(data);
1878
1879         return ret;
1880 }
1881
1882 static void sunxi_nand_hw_common_ecc_ctrl_cleanup(struct nand_ecc_ctrl *ecc)
1883 {
1884         kfree(ecc->priv);
1885 }
1886
1887 static int sunxi_nand_hw_ecc_ctrl_init(struct mtd_info *mtd,
1888                                        struct nand_ecc_ctrl *ecc,
1889                                        struct device_node *np)
1890 {
1891         struct nand_chip *nand = mtd_to_nand(mtd);
1892         struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
1893         struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
1894         int ret;
1895
1896         ret = sunxi_nand_hw_common_ecc_ctrl_init(mtd, ecc, np);
1897         if (ret)
1898                 return ret;
1899
1900         if (nfc->dmac) {
1901                 ecc->read_page = sunxi_nfc_hw_ecc_read_page_dma;
1902                 ecc->read_subpage = sunxi_nfc_hw_ecc_read_subpage_dma;
1903                 ecc->write_page = sunxi_nfc_hw_ecc_write_page_dma;
1904                 nand->options |= NAND_USE_BOUNCE_BUFFER;
1905         } else {
1906                 ecc->read_page = sunxi_nfc_hw_ecc_read_page;
1907                 ecc->read_subpage = sunxi_nfc_hw_ecc_read_subpage;
1908                 ecc->write_page = sunxi_nfc_hw_ecc_write_page;
1909         }
1910
1911         /* TODO: support DMA for raw accesses and subpage write */
1912         ecc->write_subpage = sunxi_nfc_hw_ecc_write_subpage;
1913         ecc->read_oob_raw = nand_read_oob_std;
1914         ecc->write_oob_raw = nand_write_oob_std;
1915         ecc->read_subpage = sunxi_nfc_hw_ecc_read_subpage;
1916
1917         return 0;
1918 }
1919
1920 static int sunxi_nand_hw_syndrome_ecc_ctrl_init(struct mtd_info *mtd,
1921                                                 struct nand_ecc_ctrl *ecc,
1922                                                 struct device_node *np)
1923 {
1924         int ret;
1925
1926         ret = sunxi_nand_hw_common_ecc_ctrl_init(mtd, ecc, np);
1927         if (ret)
1928                 return ret;
1929
1930         ecc->prepad = 4;
1931         ecc->read_page = sunxi_nfc_hw_syndrome_ecc_read_page;
1932         ecc->write_page = sunxi_nfc_hw_syndrome_ecc_write_page;
1933         ecc->read_oob_raw = nand_read_oob_syndrome;
1934         ecc->write_oob_raw = nand_write_oob_syndrome;
1935
1936         return 0;
1937 }
1938
1939 static void sunxi_nand_ecc_cleanup(struct nand_ecc_ctrl *ecc)
1940 {
1941         switch (ecc->mode) {
1942         case NAND_ECC_HW:
1943         case NAND_ECC_HW_SYNDROME:
1944                 sunxi_nand_hw_common_ecc_ctrl_cleanup(ecc);
1945                 break;
1946         case NAND_ECC_NONE:
1947         default:
1948                 break;
1949         }
1950 }
1951
1952 static int sunxi_nand_ecc_init(struct mtd_info *mtd, struct nand_ecc_ctrl *ecc,
1953                                struct device_node *np)
1954 {
1955         struct nand_chip *nand = mtd_to_nand(mtd);
1956         int ret;
1957
1958         if (!ecc->size) {
1959                 ecc->size = nand->ecc_step_ds;
1960                 ecc->strength = nand->ecc_strength_ds;
1961         }
1962
1963         if (!ecc->size || !ecc->strength)
1964                 return -EINVAL;
1965
1966         switch (ecc->mode) {
1967         case NAND_ECC_HW:
1968                 ret = sunxi_nand_hw_ecc_ctrl_init(mtd, ecc, np);
1969                 if (ret)
1970                         return ret;
1971                 break;
1972         case NAND_ECC_HW_SYNDROME:
1973                 ret = sunxi_nand_hw_syndrome_ecc_ctrl_init(mtd, ecc, np);
1974                 if (ret)
1975                         return ret;
1976                 break;
1977         case NAND_ECC_NONE:
1978         case NAND_ECC_SOFT:
1979                 break;
1980         default:
1981                 return -EINVAL;
1982         }
1983
1984         return 0;
1985 }
1986
1987 static int sunxi_nand_chip_init(struct device *dev, struct sunxi_nfc *nfc,
1988                                 struct device_node *np)
1989 {
1990         struct sunxi_nand_chip *chip;
1991         struct mtd_info *mtd;
1992         struct nand_chip *nand;
1993         int nsels;
1994         int ret;
1995         int i;
1996         u32 tmp;
1997
1998         if (!of_get_property(np, "reg", &nsels))
1999                 return -EINVAL;
2000
2001         nsels /= sizeof(u32);
2002         if (!nsels) {
2003                 dev_err(dev, "invalid reg property size\n");
2004                 return -EINVAL;
2005         }
2006
2007         chip = devm_kzalloc(dev,
2008                             sizeof(*chip) +
2009                             (nsels * sizeof(struct sunxi_nand_chip_sel)),
2010                             GFP_KERNEL);
2011         if (!chip) {
2012                 dev_err(dev, "could not allocate chip\n");
2013                 return -ENOMEM;
2014         }
2015
2016         chip->nsels = nsels;
2017         chip->selected = -1;
2018
2019         for (i = 0; i < nsels; i++) {
2020                 ret = of_property_read_u32_index(np, "reg", i, &tmp);
2021                 if (ret) {
2022                         dev_err(dev, "could not retrieve reg property: %d\n",
2023                                 ret);
2024                         return ret;
2025                 }
2026
2027                 if (tmp > NFC_MAX_CS) {
2028                         dev_err(dev,
2029                                 "invalid reg value: %u (max CS = 7)\n",
2030                                 tmp);
2031                         return -EINVAL;
2032                 }
2033
2034                 if (test_and_set_bit(tmp, &nfc->assigned_cs)) {
2035                         dev_err(dev, "CS %d already assigned\n", tmp);
2036                         return -EINVAL;
2037                 }
2038
2039                 chip->sels[i].cs = tmp;
2040
2041                 if (!of_property_read_u32_index(np, "allwinner,rb", i, &tmp) &&
2042                     tmp < 2) {
2043                         chip->sels[i].rb.type = RB_NATIVE;
2044                         chip->sels[i].rb.info.nativeid = tmp;
2045                 } else {
2046                         ret = of_get_named_gpio(np, "rb-gpios", i);
2047                         if (ret >= 0) {
2048                                 tmp = ret;
2049                                 chip->sels[i].rb.type = RB_GPIO;
2050                                 chip->sels[i].rb.info.gpio = tmp;
2051                                 ret = devm_gpio_request(dev, tmp, "nand-rb");
2052                                 if (ret)
2053                                         return ret;
2054
2055                                 ret = gpio_direction_input(tmp);
2056                                 if (ret)
2057                                         return ret;
2058                         } else {
2059                                 chip->sels[i].rb.type = RB_NONE;
2060                         }
2061                 }
2062         }
2063
2064         nand = &chip->nand;
2065         /* Default tR value specified in the ONFI spec (chapter 4.15.1) */
2066         nand->chip_delay = 200;
2067         nand->controller = &nfc->controller;
2068         /*
2069          * Set the ECC mode to the default value in case nothing is specified
2070          * in the DT.
2071          */
2072         nand->ecc.mode = NAND_ECC_HW;
2073         nand_set_flash_node(nand, np);
2074         nand->select_chip = sunxi_nfc_select_chip;
2075         nand->cmd_ctrl = sunxi_nfc_cmd_ctrl;
2076         nand->read_buf = sunxi_nfc_read_buf;
2077         nand->write_buf = sunxi_nfc_write_buf;
2078         nand->read_byte = sunxi_nfc_read_byte;
2079         nand->setup_data_interface = sunxi_nfc_setup_data_interface;
2080
2081         mtd = nand_to_mtd(nand);
2082         mtd->dev.parent = dev;
2083
2084         ret = nand_scan_ident(mtd, nsels, NULL);
2085         if (ret)
2086                 return ret;
2087
2088         if (nand->bbt_options & NAND_BBT_USE_FLASH)
2089                 nand->bbt_options |= NAND_BBT_NO_OOB;
2090
2091         if (nand->options & NAND_NEED_SCRAMBLING)
2092                 nand->options |= NAND_NO_SUBPAGE_WRITE;
2093
2094         nand->options |= NAND_SUBPAGE_READ;
2095
2096         ret = sunxi_nand_ecc_init(mtd, &nand->ecc, np);
2097         if (ret) {
2098                 dev_err(dev, "ECC init failed: %d\n", ret);
2099                 return ret;
2100         }
2101
2102         ret = nand_scan_tail(mtd);
2103         if (ret) {
2104                 dev_err(dev, "nand_scan_tail failed: %d\n", ret);
2105                 return ret;
2106         }
2107
2108         ret = mtd_device_register(mtd, NULL, 0);
2109         if (ret) {
2110                 dev_err(dev, "failed to register mtd device: %d\n", ret);
2111                 nand_cleanup(nand);
2112                 return ret;
2113         }
2114
2115         list_add_tail(&chip->node, &nfc->chips);
2116
2117         return 0;
2118 }
2119
2120 static int sunxi_nand_chips_init(struct device *dev, struct sunxi_nfc *nfc)
2121 {
2122         struct device_node *np = dev->of_node;
2123         struct device_node *nand_np;
2124         int nchips = of_get_child_count(np);
2125         int ret;
2126
2127         if (nchips > 8) {
2128                 dev_err(dev, "too many NAND chips: %d (max = 8)\n", nchips);
2129                 return -EINVAL;
2130         }
2131
2132         for_each_child_of_node(np, nand_np) {
2133                 ret = sunxi_nand_chip_init(dev, nfc, nand_np);
2134                 if (ret) {
2135                         of_node_put(nand_np);
2136                         return ret;
2137                 }
2138         }
2139
2140         return 0;
2141 }
2142
2143 static void sunxi_nand_chips_cleanup(struct sunxi_nfc *nfc)
2144 {
2145         struct sunxi_nand_chip *chip;
2146
2147         while (!list_empty(&nfc->chips)) {
2148                 chip = list_first_entry(&nfc->chips, struct sunxi_nand_chip,
2149                                         node);
2150                 nand_release(&chip->nand);
2151                 sunxi_nand_ecc_cleanup(&chip->nand.ecc);
2152                 list_del(&chip->node);
2153         }
2154 }
2155
2156 static int sunxi_nfc_probe(struct platform_device *pdev)
2157 {
2158         struct device *dev = &pdev->dev;
2159         struct resource *r;
2160         struct sunxi_nfc *nfc;
2161         int irq;
2162         int ret;
2163
2164         nfc = devm_kzalloc(dev, sizeof(*nfc), GFP_KERNEL);
2165         if (!nfc)
2166                 return -ENOMEM;
2167
2168         nfc->dev = dev;
2169         nand_hw_control_init(&nfc->controller);
2170         INIT_LIST_HEAD(&nfc->chips);
2171
2172         r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2173         nfc->regs = devm_ioremap_resource(dev, r);
2174         if (IS_ERR(nfc->regs))
2175                 return PTR_ERR(nfc->regs);
2176
2177         irq = platform_get_irq(pdev, 0);
2178         if (irq < 0) {
2179                 dev_err(dev, "failed to retrieve irq\n");
2180                 return irq;
2181         }
2182
2183         nfc->ahb_clk = devm_clk_get(dev, "ahb");
2184         if (IS_ERR(nfc->ahb_clk)) {
2185                 dev_err(dev, "failed to retrieve ahb clk\n");
2186                 return PTR_ERR(nfc->ahb_clk);
2187         }
2188
2189         ret = clk_prepare_enable(nfc->ahb_clk);
2190         if (ret)
2191                 return ret;
2192
2193         nfc->mod_clk = devm_clk_get(dev, "mod");
2194         if (IS_ERR(nfc->mod_clk)) {
2195                 dev_err(dev, "failed to retrieve mod clk\n");
2196                 ret = PTR_ERR(nfc->mod_clk);
2197                 goto out_ahb_clk_unprepare;
2198         }
2199
2200         ret = clk_prepare_enable(nfc->mod_clk);
2201         if (ret)
2202                 goto out_ahb_clk_unprepare;
2203
2204         nfc->reset = devm_reset_control_get_optional(dev, "ahb");
2205         if (!IS_ERR(nfc->reset)) {
2206                 ret = reset_control_deassert(nfc->reset);
2207                 if (ret) {
2208                         dev_err(dev, "reset err %d\n", ret);
2209                         goto out_mod_clk_unprepare;
2210                 }
2211         } else if (PTR_ERR(nfc->reset) != -ENOENT) {
2212                 ret = PTR_ERR(nfc->reset);
2213                 goto out_mod_clk_unprepare;
2214         }
2215
2216         ret = sunxi_nfc_rst(nfc);
2217         if (ret)
2218                 goto out_ahb_reset_reassert;
2219
2220         writel(0, nfc->regs + NFC_REG_INT);
2221         ret = devm_request_irq(dev, irq, sunxi_nfc_interrupt,
2222                                0, "sunxi-nand", nfc);
2223         if (ret)
2224                 goto out_ahb_reset_reassert;
2225
2226         nfc->dmac = dma_request_slave_channel(dev, "rxtx");
2227         if (nfc->dmac) {
2228                 struct dma_slave_config dmac_cfg = { };
2229
2230                 dmac_cfg.src_addr = r->start + NFC_REG_IO_DATA;
2231                 dmac_cfg.dst_addr = dmac_cfg.src_addr;
2232                 dmac_cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
2233                 dmac_cfg.dst_addr_width = dmac_cfg.src_addr_width;
2234                 dmac_cfg.src_maxburst = 4;
2235                 dmac_cfg.dst_maxburst = 4;
2236                 dmaengine_slave_config(nfc->dmac, &dmac_cfg);
2237         } else {
2238                 dev_warn(dev, "failed to request rxtx DMA channel\n");
2239         }
2240
2241         platform_set_drvdata(pdev, nfc);
2242
2243         ret = sunxi_nand_chips_init(dev, nfc);
2244         if (ret) {
2245                 dev_err(dev, "failed to init nand chips\n");
2246                 goto out_release_dmac;
2247         }
2248
2249         return 0;
2250
2251 out_release_dmac:
2252         if (nfc->dmac)
2253                 dma_release_channel(nfc->dmac);
2254 out_ahb_reset_reassert:
2255         if (!IS_ERR(nfc->reset))
2256                 reset_control_assert(nfc->reset);
2257 out_mod_clk_unprepare:
2258         clk_disable_unprepare(nfc->mod_clk);
2259 out_ahb_clk_unprepare:
2260         clk_disable_unprepare(nfc->ahb_clk);
2261
2262         return ret;
2263 }
2264
2265 static int sunxi_nfc_remove(struct platform_device *pdev)
2266 {
2267         struct sunxi_nfc *nfc = platform_get_drvdata(pdev);
2268
2269         sunxi_nand_chips_cleanup(nfc);
2270
2271         if (!IS_ERR(nfc->reset))
2272                 reset_control_assert(nfc->reset);
2273
2274         if (nfc->dmac)
2275                 dma_release_channel(nfc->dmac);
2276         clk_disable_unprepare(nfc->mod_clk);
2277         clk_disable_unprepare(nfc->ahb_clk);
2278
2279         return 0;
2280 }
2281
2282 static const struct of_device_id sunxi_nfc_ids[] = {
2283         { .compatible = "allwinner,sun4i-a10-nand" },
2284         { /* sentinel */ }
2285 };
2286 MODULE_DEVICE_TABLE(of, sunxi_nfc_ids);
2287
2288 static struct platform_driver sunxi_nfc_driver = {
2289         .driver = {
2290                 .name = "sunxi_nand",
2291                 .of_match_table = sunxi_nfc_ids,
2292         },
2293         .probe = sunxi_nfc_probe,
2294         .remove = sunxi_nfc_remove,
2295 };
2296 module_platform_driver(sunxi_nfc_driver);
2297
2298 MODULE_LICENSE("GPL v2");
2299 MODULE_AUTHOR("Boris BREZILLON");
2300 MODULE_DESCRIPTION("Allwinner NAND Flash Controller driver");
2301 MODULE_ALIAS("platform:sunxi_nand");