drivers/mtd/nand/raw/marvell_nand.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Marvell NAND flash controller driver
   4  *
   5  * Copyright (C) 2017 Marvell
   6  * Author: Miquel RAYNAL <miquel.raynal@free-electrons.com>
   7  *
   8  */
   9
  10 #include <linux/module.h>
  11 #include <linux/clk.h>
  12 #include <linux/mtd/rawnand.h>
  13 #include <linux/of_platform.h>
  14 #include <linux/iopoll.h>
  15 #include <linux/interrupt.h>
  16 #include <linux/slab.h>
  17 #include <linux/mfd/syscon.h>
  18 #include <linux/regmap.h>
  19 #include <asm/unaligned.h>
  20
  21 #include <linux/dmaengine.h>
  22 #include <linux/dma-mapping.h>
  23 #include <linux/dma/pxa-dma.h>
  24 #include <linux/platform_data/mtd-nand-pxa3xx.h>
  25
  26 /* Data FIFO granularity, FIFO reads/writes must be a multiple of this length */
  27 #define FIFO_DEPTH              8
  28 #define FIFO_REP(x)             (x / sizeof(u32))
  29 #define BCH_SEQ_READS           (32 / FIFO_DEPTH)
  30 /* NFC does not support transfers of larger chunks at a time */
  31 #define MAX_CHUNK_SIZE          2112
  32 /* NFCv1 cannot read more that 7 bytes of ID */
  33 #define NFCV1_READID_LEN        7
  34 /* Polling is done at a pace of POLL_PERIOD us until POLL_TIMEOUT is reached */
  35 #define POLL_PERIOD             0
  36 #define POLL_TIMEOUT            100000
  37 /* Interrupt maximum wait period in ms */
  38 #define IRQ_TIMEOUT             1000
  39 /* Latency in clock cycles between SoC pins and NFC logic */
  40 #define MIN_RD_DEL_CNT          3
  41 /* Maximum number of contiguous address cycles */
  42 #define MAX_ADDRESS_CYC_NFCV1   5
  43 #define MAX_ADDRESS_CYC_NFCV2   7
  44 /* System control registers/bits to enable the NAND controller on some SoCs */
  45 #define GENCONF_SOC_DEVICE_MUX  0x208
  46 #define GENCONF_SOC_DEVICE_MUX_NFC_EN BIT(0)
  47 #define GENCONF_SOC_DEVICE_MUX_ECC_CLK_RST BIT(20)
  48 #define GENCONF_SOC_DEVICE_MUX_ECC_CORE_RST BIT(21)
  49 #define GENCONF_SOC_DEVICE_MUX_NFC_INT_EN BIT(25)
  50 #define GENCONF_CLK_GATING_CTRL 0x220
  51 #define GENCONF_CLK_GATING_CTRL_ND_GATE BIT(2)
  52 #define GENCONF_ND_CLK_CTRL     0x700
  53 #define GENCONF_ND_CLK_CTRL_EN  BIT(0)
  54
  55 /* NAND controller data flash control register */
  56 #define NDCR                    0x00
  57 #define NDCR_ALL_INT            GENMASK(11, 0)
  58 #define NDCR_CS1_CMDDM          BIT(7)
  59 #define NDCR_CS0_CMDDM          BIT(8)
  60 #define NDCR_RDYM               BIT(11)
  61 #define NDCR_ND_ARB_EN          BIT(12)
  62 #define NDCR_RA_START           BIT(15)
  63 #define NDCR_RD_ID_CNT(x)       (min_t(unsigned int, x, 0x7) << 16)
  64 #define NDCR_PAGE_SZ(x)         (x >= 2048 ? BIT(24) : 0)
  65 #define NDCR_DWIDTH_M           BIT(26)
  66 #define NDCR_DWIDTH_C           BIT(27)
  67 #define NDCR_ND_RUN             BIT(28)
  68 #define NDCR_DMA_EN             BIT(29)
  69 #define NDCR_ECC_EN             BIT(30)
  70 #define NDCR_SPARE_EN           BIT(31)
  71 #define NDCR_GENERIC_FIELDS_MASK (~(NDCR_RA_START | NDCR_PAGE_SZ(2048) | \
  72                                     NDCR_DWIDTH_M | NDCR_DWIDTH_C))
  73
  74 /* NAND interface timing parameter 0 register */
  75 #define NDTR0                   0x04
  76 #define NDTR0_TRP(x)            ((min_t(unsigned int, x, 0xF) & 0x7) << 0)
  77 #define NDTR0_TRH(x)            (min_t(unsigned int, x, 0x7) << 3)
  78 #define NDTR0_ETRP(x)           ((min_t(unsigned int, x, 0xF) & 0x8) << 3)
  79 #define NDTR0_SEL_NRE_EDGE      BIT(7)
  80 #define NDTR0_TWP(x)            (min_t(unsigned int, x, 0x7) << 8)
  81 #define NDTR0_TWH(x)            (min_t(unsigned int, x, 0x7) << 11)
  82 #define NDTR0_TCS(x)            (min_t(unsigned int, x, 0x7) << 16)
  83 #define NDTR0_TCH(x)            (min_t(unsigned int, x, 0x7) << 19)
  84 #define NDTR0_RD_CNT_DEL(x)     (min_t(unsigned int, x, 0xF) << 22)
  85 #define NDTR0_SELCNTR           BIT(26)
  86 #define NDTR0_TADL(x)           (min_t(unsigned int, x, 0x1F) << 27)
  87
  88 /* NAND interface timing parameter 1 register */
  89 #define NDTR1                   0x0C
  90 #define NDTR1_TAR(x)            (min_t(unsigned int, x, 0xF) << 0)
  91 #define NDTR1_TWHR(x)           (min_t(unsigned int, x, 0xF) << 4)
  92 #define NDTR1_TRHW(x)           (min_t(unsigned int, x / 16, 0x3) << 8)
  93 #define NDTR1_PRESCALE          BIT(14)
  94 #define NDTR1_WAIT_MODE         BIT(15)
  95 #define NDTR1_TR(x)             (min_t(unsigned int, x, 0xFFFF) << 16)
  96
  97 /* NAND controller status register */
  98 #define NDSR                    0x14
  99 #define NDSR_WRCMDREQ           BIT(0)
 100 #define NDSR_RDDREQ             BIT(1)
 101 #define NDSR_WRDREQ             BIT(2)
 102 #define NDSR_CORERR             BIT(3)
 103 #define NDSR_UNCERR             BIT(4)
 104 #define NDSR_CMDD(cs)           BIT(8 - cs)
 105 #define NDSR_RDY(rb)            BIT(11 + rb)
 106 #define NDSR_ERRCNT(x)          ((x >> 16) & 0x1F)
 107
 108 /* NAND ECC control register */
 109 #define NDECCCTRL               0x28
 110 #define NDECCCTRL_BCH_EN        BIT(0)
 111
 112 /* NAND controller data buffer register */
 113 #define NDDB                    0x40
 114
 115 /* NAND controller command buffer 0 register */
 116 #define NDCB0                   0x48
 117 #define NDCB0_CMD1(x)           ((x & 0xFF) << 0)
 118 #define NDCB0_CMD2(x)           ((x & 0xFF) << 8)
 119 #define NDCB0_ADDR_CYC(x)       ((x & 0x7) << 16)
 120 #define NDCB0_ADDR_GET_NUM_CYC(x) (((x) >> 16) & 0x7)
 121 #define NDCB0_DBC               BIT(19)
 122 #define NDCB0_CMD_TYPE(x)       ((x & 0x7) << 21)
 123 #define NDCB0_CSEL              BIT(24)
 124 #define NDCB0_RDY_BYP           BIT(27)
 125 #define NDCB0_LEN_OVRD          BIT(28)
 126 #define NDCB0_CMD_XTYPE(x)      ((x & 0x7) << 29)
 127
 128 /* NAND controller command buffer 1 register */
 129 #define NDCB1                   0x4C
 130 #define NDCB1_COLS(x)           ((x & 0xFFFF) << 0)
 131 #define NDCB1_ADDRS_PAGE(x)     (x << 16)
 132
 133 /* NAND controller command buffer 2 register */
 134 #define NDCB2                   0x50
 135 #define NDCB2_ADDR5_PAGE(x)     (((x >> 16) & 0xFF) << 0)
 136 #define NDCB2_ADDR5_CYC(x)      ((x & 0xFF) << 0)
 137
 138 /* NAND controller command buffer 3 register */
 139 #define NDCB3                   0x54
 140 #define NDCB3_ADDR6_CYC(x)      ((x & 0xFF) << 16)
 141 #define NDCB3_ADDR7_CYC(x)      ((x & 0xFF) << 24)
 142
 143 /* NAND controller command buffer 0 register 'type' and 'xtype' fields */
 144 #define TYPE_READ               0
 145 #define TYPE_WRITE              1
 146 #define TYPE_ERASE              2
 147 #define TYPE_READ_ID            3
 148 #define TYPE_STATUS             4
 149 #define TYPE_RESET              5
 150 #define TYPE_NAKED_CMD          6
 151 #define TYPE_NAKED_ADDR         7
 152 #define TYPE_MASK               7
 153 #define XTYPE_MONOLITHIC_RW     0
 154 #define XTYPE_LAST_NAKED_RW     1
 155 #define XTYPE_FINAL_COMMAND     3
 156 #define XTYPE_READ              4
 157 #define XTYPE_WRITE_DISPATCH    4
 158 #define XTYPE_NAKED_RW          5
 159 #define XTYPE_COMMAND_DISPATCH  6
 160 #define XTYPE_MASK              7
 161
 162 /**
 163  * Marvell ECC engine works differently than the others, in order to limit the
 164  * size of the IP, hardware engineers chose to set a fixed strength at 16 bits
 165  * per subpage, and depending on a the desired strength needed by the NAND chip,
 166  * a particular layout mixing data/spare/ecc is defined, with a possible last
 167  * chunk smaller that the others.
 168  *
 169  * @writesize:          Full page size on which the layout applies
 170  * @chunk:              Desired ECC chunk size on which the layout applies
 171  * @strength:           Desired ECC strength (per chunk size bytes) on which the
 172  *                      layout applies
 173  * @nchunks:            Total number of chunks
 174  * @full_chunk_cnt:     Number of full-sized chunks, which is the number of
 175  *                      repetitions of the pattern:
 176  *                      (data_bytes + spare_bytes + ecc_bytes).
 177  * @data_bytes:         Number of data bytes per chunk
 178  * @spare_bytes:        Number of spare bytes per chunk
 179  * @ecc_bytes:          Number of ecc bytes per chunk
 180  * @last_data_bytes:    Number of data bytes in the last chunk
 181  * @last_spare_bytes:   Number of spare bytes in the last chunk
 182  * @last_ecc_bytes:     Number of ecc bytes in the last chunk
 183  */
 184 struct marvell_hw_ecc_layout {
 185         /* Constraints */
 186         int writesize;
 187         int chunk;
 188         int strength;
 189         /* Corresponding layout */
 190         int nchunks;
 191         int full_chunk_cnt;
 192         int data_bytes;
 193         int spare_bytes;
 194         int ecc_bytes;
 195         int last_data_bytes;
 196         int last_spare_bytes;
 197         int last_ecc_bytes;
 198 };
 199
 200 #define MARVELL_LAYOUT(ws, dc, ds, nc, fcc, db, sb, eb, ldb, lsb, leb)  \
 201         {                                                               \
 202                 .writesize = ws,                                        \
 203                 .chunk = dc,                                            \
 204                 .strength = ds,                                         \
 205                 .nchunks = nc,                                          \
 206                 .full_chunk_cnt = fcc,                                  \
 207                 .data_bytes = db,                                       \
 208                 .spare_bytes = sb,                                      \
 209                 .ecc_bytes = eb,                                        \
 210                 .last_data_bytes = ldb,                                 \
 211                 .last_spare_bytes = lsb,                                \
 212                 .last_ecc_bytes = leb,                                  \
 213         }
 214
 215 /* Layouts explained in AN-379_Marvell_SoC_NFC_ECC */
 216 static const struct marvell_hw_ecc_layout marvell_nfc_layouts[] = {
 217         MARVELL_LAYOUT(  512,   512,  1,  1,  1,  512,  8,  8,  0,  0,  0),
 218         MARVELL_LAYOUT( 2048,   512,  1,  1,  1, 2048, 40, 24,  0,  0,  0),
 219         MARVELL_LAYOUT( 2048,   512,  4,  1,  1, 2048, 32, 30,  0,  0,  0),
 220         MARVELL_LAYOUT( 4096,   512,  4,  2,  2, 2048, 32, 30,  0,  0,  0),
 221         MARVELL_LAYOUT( 4096,   512,  8,  5,  4, 1024,  0, 30,  0, 64, 30),
 222 };
 223
 224 /**
 225  * The Nand Flash Controller has up to 4 CE and 2 RB pins. The CE selection
 226  * is made by a field in NDCB0 register, and in another field in NDCB2 register.
 227  * The datasheet describes the logic with an error: ADDR5 field is once
 228  * declared at the beginning of NDCB2, and another time at its end. Because the
 229  * ADDR5 field of NDCB2 may be used by other bytes, it would be more logical
 230  * to use the last bit of this field instead of the first ones.
 231  *
 232  * @cs:                 Wanted CE lane.
 233  * @ndcb0_csel:         Value of the NDCB0 register with or without the flag
 234  *                      selecting the wanted CE lane. This is set once when
 235  *                      the Device Tree is probed.
 236  * @rb:                 Ready/Busy pin for the flash chip
 237  */
 238 struct marvell_nand_chip_sel {
 239         unsigned int cs;
 240         u32 ndcb0_csel;
 241         unsigned int rb;
 242 };
 243
 244 /**
 245  * NAND chip structure: stores NAND chip device related information
 246  *
 247  * @chip:               Base NAND chip structure
 248  * @node:               Used to store NAND chips into a list
 249  * @layout              NAND layout when using hardware ECC
 250  * @ndcr:               Controller register value for this NAND chip
 251  * @ndtr0:              Timing registers 0 value for this NAND chip
 252  * @ndtr1:              Timing registers 1 value for this NAND chip
 253  * @selected_die:       Current active CS
 254  * @nsels:              Number of CS lines required by the NAND chip
 255  * @sels:               Array of CS lines descriptions
 256  */
 257 struct marvell_nand_chip {
 258         struct nand_chip chip;
 259         struct list_head node;
 260         const struct marvell_hw_ecc_layout *layout;
 261         u32 ndcr;
 262         u32 ndtr0;
 263         u32 ndtr1;
 264         int addr_cyc;
 265         int selected_die;
 266         unsigned int nsels;
 267         struct marvell_nand_chip_sel sels[0];
 268 };
 269
 270 static inline struct marvell_nand_chip *to_marvell_nand(struct nand_chip *chip)
 271 {
 272         return container_of(chip, struct marvell_nand_chip, chip);
 273 }
 274
 275 static inline struct marvell_nand_chip_sel *to_nand_sel(struct marvell_nand_chip
 276                                                         *nand)
 277 {
 278         return &nand->sels[nand->selected_die];
 279 }
 280
 281 /**
 282  * NAND controller capabilities for distinction between compatible strings
 283  *
 284  * @max_cs_nb:          Number of Chip Select lines available
 285  * @max_rb_nb:          Number of Ready/Busy lines available
 286  * @need_system_controller: Indicates if the SoC needs to have access to the
 287  *                      system controller (ie. to enable the NAND controller)
 288  * @legacy_of_bindings: Indicates if DT parsing must be done using the old
 289  *                      fashion way
 290  * @is_nfcv2:           NFCv2 has numerous enhancements compared to NFCv1, ie.
 291  *                      BCH error detection and correction algorithm,
 292  *                      NDCB3 register has been added
 293  * @use_dma:            Use dma for data transfers
 294  */
 295 struct marvell_nfc_caps {
 296         unsigned int max_cs_nb;
 297         unsigned int max_rb_nb;
 298         bool need_system_controller;
 299         bool legacy_of_bindings;
 300         bool is_nfcv2;
 301         bool use_dma;
 302 };
 303
 304 /**
 305  * NAND controller structure: stores Marvell NAND controller information
 306  *
 307  * @controller:         Base controller structure
 308  * @dev:                Parent device (used to print error messages)
 309  * @regs:               NAND controller registers
 310  * @core_clk:           Core clock
 311  * @reg_clk:            Regiters clock
 312  * @complete:           Completion object to wait for NAND controller events
 313  * @assigned_cs:        Bitmask describing already assigned CS lines
 314  * @chips:              List containing all the NAND chips attached to
 315  *                      this NAND controller
 316  * @caps:               NAND controller capabilities for each compatible string
 317  * @dma_chan:           DMA channel (NFCv1 only)
 318  * @dma_buf:            32-bit aligned buffer for DMA transfers (NFCv1 only)
 319  */
 320 struct marvell_nfc {
 321         struct nand_controller controller;
 322         struct device *dev;
 323         void __iomem *regs;
 324         struct clk *core_clk;
 325         struct clk *reg_clk;
 326         struct completion complete;
 327         unsigned long assigned_cs;
 328         struct list_head chips;
 329         struct nand_chip *selected_chip;
 330         const struct marvell_nfc_caps *caps;
 331
 332         /* DMA (NFCv1 only) */
 333         bool use_dma;
 334         struct dma_chan *dma_chan;
 335         u8 *dma_buf;
 336 };
 337
 338 static inline struct marvell_nfc *to_marvell_nfc(struct nand_controller *ctrl)
 339 {
 340         return container_of(ctrl, struct marvell_nfc, controller);
 341 }
 342
 343 /**
 344  * NAND controller timings expressed in NAND Controller clock cycles
 345  *
 346  * @tRP:                ND_nRE pulse width
 347  * @tRH:                ND_nRE high duration
 348  * @tWP:                ND_nWE pulse time
 349  * @tWH:                ND_nWE high duration
 350  * @tCS:                Enable signal setup time
 351  * @tCH:                Enable signal hold time
 352  * @tADL:               Address to write data delay
 353  * @tAR:                ND_ALE low to ND_nRE low delay
 354  * @tWHR:               ND_nWE high to ND_nRE low for status read
 355  * @tRHW:               ND_nRE high duration, read to write delay
 356  * @tR:                 ND_nWE high to ND_nRE low for read
 357  */
 358 struct marvell_nfc_timings {
 359         /* NDTR0 fields */
 360         unsigned int tRP;
 361         unsigned int tRH;
 362         unsigned int tWP;
 363         unsigned int tWH;
 364         unsigned int tCS;
 365         unsigned int tCH;
 366         unsigned int tADL;
 367         /* NDTR1 fields */
 368         unsigned int tAR;
 369         unsigned int tWHR;
 370         unsigned int tRHW;
 371         unsigned int tR;
 372 };
 373
 374 /**
 375  * Derives a duration in numbers of clock cycles.
 376  *
 377  * @ps: Duration in pico-seconds
 378  * @period_ns:  Clock period in nano-seconds
 379  *
 380  * Convert the duration in nano-seconds, then divide by the period and
 381  * return the number of clock periods.
 382  */
 383 #define TO_CYCLES(ps, period_ns) (DIV_ROUND_UP(ps / 1000, period_ns))
 384 #define TO_CYCLES64(ps, period_ns) (DIV_ROUND_UP_ULL(div_u64(ps, 1000), \
 385                                                      period_ns))
 386
 387 /**
 388  * NAND driver structure filled during the parsing of the ->exec_op() subop
 389  * subset of instructions.
 390  *
 391  * @ndcb:               Array of values written to NDCBx registers
 392  * @cle_ale_delay_ns:   Optional delay after the last CMD or ADDR cycle
 393  * @rdy_timeout_ms:     Timeout for waits on Ready/Busy pin
 394  * @rdy_delay_ns:       Optional delay after waiting for the RB pin
 395  * @data_delay_ns:      Optional delay after the data xfer
 396  * @data_instr_idx:     Index of the data instruction in the subop
 397  * @data_instr:         Pointer to the data instruction in the subop
 398  */
 399 struct marvell_nfc_op {
 400         u32 ndcb[4];
 401         unsigned int cle_ale_delay_ns;
 402         unsigned int rdy_timeout_ms;
 403         unsigned int rdy_delay_ns;
 404         unsigned int data_delay_ns;
 405         unsigned int data_instr_idx;
 406         const struct nand_op_instr *data_instr;
 407 };
 408
 409 /*
 410  * Internal helper to conditionnally apply a delay (from the above structure,
 411  * most of the time).
 412  */
 413 static void cond_delay(unsigned int ns)
 414 {
 415         if (!ns)
 416                 return;
 417
 418         if (ns < 10000)
 419                 ndelay(ns);
 420         else
 421                 udelay(DIV_ROUND_UP(ns, 1000));
 422 }
 423
 424 /*
 425  * The controller has many flags that could generate interrupts, most of them
 426  * are disabled and polling is used. For the very slow signals, using interrupts
 427  * may relax the CPU charge.
 428  */
 429 static void marvell_nfc_disable_int(struct marvell_nfc *nfc, u32 int_mask)
 430 {
 431         u32 reg;
 432
 433         /* Writing 1 disables the interrupt */
 434         reg = readl_relaxed(nfc->regs + NDCR);
 435         writel_relaxed(reg | int_mask, nfc->regs + NDCR);
 436 }
 437
 438 static void marvell_nfc_enable_int(struct marvell_nfc *nfc, u32 int_mask)
 439 {
 440         u32 reg;
 441
 442         /* Writing 0 enables the interrupt */
 443         reg = readl_relaxed(nfc->regs + NDCR);
 444         writel_relaxed(reg & ~int_mask, nfc->regs + NDCR);
 445 }
 446
 447 static u32 marvell_nfc_clear_int(struct marvell_nfc *nfc, u32 int_mask)
 448 {
 449         u32 reg;
 450
 451         reg = readl_relaxed(nfc->regs + NDSR);
 452         writel_relaxed(int_mask, nfc->regs + NDSR);
 453
 454         return reg & int_mask;
 455 }
 456
 457 static void marvell_nfc_force_byte_access(struct nand_chip *chip,
 458                                           bool force_8bit)
 459 {
 460         struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
 461         u32 ndcr;
 462
 463         /*
 464          * Callers of this function do not verify if the NAND is using a 16-bit
 465          * an 8-bit bus for normal operations, so we need to take care of that
 466          * here by leaving the configuration unchanged if the NAND does not have
 467          * the NAND_BUSWIDTH_16 flag set.
 468          */
 469         if (!(chip->options & NAND_BUSWIDTH_16))
 470                 return;
 471
 472         ndcr = readl_relaxed(nfc->regs + NDCR);
 473
 474         if (force_8bit)
 475                 ndcr &= ~(NDCR_DWIDTH_M | NDCR_DWIDTH_C);
 476         else
 477                 ndcr |= NDCR_DWIDTH_M | NDCR_DWIDTH_C;
 478
 479         writel_relaxed(ndcr, nfc->regs + NDCR);
 480 }
 481
 482 static int marvell_nfc_wait_ndrun(struct nand_chip *chip)
 483 {
 484         struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
 485         u32 val;
 486         int ret;
 487
 488         /*
 489          * The command is being processed, wait for the ND_RUN bit to be
 490          * cleared by the NFC. If not, we must clear it by hand.
 491          */
 492         ret = readl_relaxed_poll_timeout(nfc->regs + NDCR, val,
 493                                          (val & NDCR_ND_RUN) == 0,
 494                                          POLL_PERIOD, POLL_TIMEOUT);
 495         if (ret) {
 496                 dev_err(nfc->dev, "Timeout on NAND controller run mode\n");
 497                 writel_relaxed(readl(nfc->regs + NDCR) & ~NDCR_ND_RUN,
 498                                nfc->regs + NDCR);
 499                 return ret;
 500         }
 501
 502         return 0;
 503 }
 504
 505 /*
 506  * Any time a command has to be sent to the controller, the following sequence
 507  * has to be followed:
 508  * - call marvell_nfc_prepare_cmd()
 509  *      -> activate the ND_RUN bit that will kind of 'start a job'
 510  *      -> wait the signal indicating the NFC is waiting for a command
 511  * - send the command (cmd and address cycles)
 512  * - enventually send or receive the data
 513  * - call marvell_nfc_end_cmd() with the corresponding flag
 514  *      -> wait the flag to be triggered or cancel the job with a timeout
 515  *
 516  * The following helpers are here to factorize the code a bit so that
 517  * specialized functions responsible for executing the actual NAND
 518  * operations do not have to replicate the same code blocks.
 519  */
 520 static int marvell_nfc_prepare_cmd(struct nand_chip *chip)
 521 {
 522         struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
 523         u32 ndcr, val;
 524         int ret;
 525
 526         /* Poll ND_RUN and clear NDSR before issuing any command */
 527         ret = marvell_nfc_wait_ndrun(chip);
 528         if (ret) {
 529                 dev_err(nfc->dev, "Last operation did not succeed\n");
 530                 return ret;
 531         }
 532
 533         ndcr = readl_relaxed(nfc->regs + NDCR);
 534         writel_relaxed(readl(nfc->regs + NDSR), nfc->regs + NDSR);
 535
 536         /* Assert ND_RUN bit and wait the NFC to be ready */
 537         writel_relaxed(ndcr | NDCR_ND_RUN, nfc->regs + NDCR);
 538         ret = readl_relaxed_poll_timeout(nfc->regs + NDSR, val,
 539                                          val & NDSR_WRCMDREQ,
 540                                          POLL_PERIOD, POLL_TIMEOUT);
 541         if (ret) {
 542                 dev_err(nfc->dev, "Timeout on WRCMDRE\n");
 543                 return -ETIMEDOUT;
 544         }
 545
 546         /* Command may be written, clear WRCMDREQ status bit */
 547         writel_relaxed(NDSR_WRCMDREQ, nfc->regs + NDSR);
 548
 549         return 0;
 550 }
 551
 552 static void marvell_nfc_send_cmd(struct nand_chip *chip,
 553                                  struct marvell_nfc_op *nfc_op)
 554 {
 555         struct marvell_nand_chip *marvell_nand = to_marvell_nand(chip);
 556         struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
 557
 558         dev_dbg(nfc->dev, "\nNDCR:  0x%08x\n"
 559                 "NDCB0: 0x%08x\nNDCB1: 0x%08x\nNDCB2: 0x%08x\nNDCB3: 0x%08x\n",
 560                 (u32)readl_relaxed(nfc->regs + NDCR), nfc_op->ndcb[0],
 561                 nfc_op->ndcb[1], nfc_op->ndcb[2], nfc_op->ndcb[3]);
 562
 563         writel_relaxed(to_nand_sel(marvell_nand)->ndcb0_csel | nfc_op->ndcb[0],
 564                        nfc->regs + NDCB0);
 565         writel_relaxed(nfc_op->ndcb[1], nfc->regs + NDCB0);
 566         writel(nfc_op->ndcb[2], nfc->regs + NDCB0);
 567
 568         /*
 569          * Write NDCB0 four times only if LEN_OVRD is set or if ADDR6 or ADDR7
 570          * fields are used (only available on NFCv2).
 571          */
 572         if (nfc_op->ndcb[0] & NDCB0_LEN_OVRD ||
 573             NDCB0_ADDR_GET_NUM_CYC(nfc_op->ndcb[0]) >= 6) {
 574                 if (!WARN_ON_ONCE(!nfc->caps->is_nfcv2))
 575                         writel(nfc_op->ndcb[3], nfc->regs + NDCB0);
 576         }
 577 }
 578
 579 static int marvell_nfc_end_cmd(struct nand_chip *chip, int flag,
 580                                const char *label)
 581 {
 582         struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
 583         u32 val;
 584         int ret;
 585
 586         ret = readl_relaxed_poll_timeout(nfc->regs + NDSR, val,
 587                                          val & flag,
 588                                          POLL_PERIOD, POLL_TIMEOUT);
 589
 590         if (ret) {
 591                 dev_err(nfc->dev, "Timeout on %s (NDSR: 0x%08x)\n",
 592                         label, val);
 593                 if (nfc->dma_chan)
 594                         dmaengine_terminate_all(nfc->dma_chan);
 595                 return ret;
 596         }
 597
 598         /*
 599          * DMA function uses this helper to poll on CMDD bits without wanting
 600          * them to be cleared.
 601          */
 602         if (nfc->use_dma && (readl_relaxed(nfc->regs + NDCR) & NDCR_DMA_EN))
 603                 return 0;
 604
 605         writel_relaxed(flag, nfc->regs + NDSR);
 606
 607         return 0;
 608 }
 609
 610 static int marvell_nfc_wait_cmdd(struct nand_chip *chip)
 611 {
 612         struct marvell_nand_chip *marvell_nand = to_marvell_nand(chip);
 613         int cs_flag = NDSR_CMDD(to_nand_sel(marvell_nand)->ndcb0_csel);
 614
 615         return marvell_nfc_end_cmd(chip, cs_flag, "CMDD");
 616 }
 617
 618 static int marvell_nfc_wait_op(struct nand_chip *chip, unsigned int timeout_ms)
 619 {
 620         struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
 621         u32 pending;
 622         int ret;
 623
 624         /* Timeout is expressed in ms */
 625         if (!timeout_ms)
 626                 timeout_ms = IRQ_TIMEOUT;
 627
 628         init_completion(&nfc->complete);
 629
 630         marvell_nfc_enable_int(nfc, NDCR_RDYM);
 631         ret = wait_for_completion_timeout(&nfc->complete,
 632                                           msecs_to_jiffies(timeout_ms));
 633         marvell_nfc_disable_int(nfc, NDCR_RDYM);
 634         pending = marvell_nfc_clear_int(nfc, NDSR_RDY(0) | NDSR_RDY(1));
 635
 636         /*
 637          * In case the interrupt was not served in the required time frame,
 638          * check if the ISR was not served or if something went actually wrong.
 639          */
 640         if (!ret && !pending) {
 641                 dev_err(nfc->dev, "Timeout waiting for RB signal\n");
 642                 return -ETIMEDOUT;
 643         }
 644
 645         return 0;
 646 }
 647
 648 static void marvell_nfc_select_chip(struct mtd_info *mtd, int die_nr)
 649 {
 650         struct nand_chip *chip = mtd_to_nand(mtd);
 651         struct marvell_nand_chip *marvell_nand = to_marvell_nand(chip);
 652         struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
 653         u32 ndcr_generic;
 654
 655         if (chip == nfc->selected_chip && die_nr == marvell_nand->selected_die)
 656                 return;
 657
 658         if (die_nr < 0 || die_nr >= marvell_nand->nsels) {
 659                 nfc->selected_chip = NULL;
 660                 marvell_nand->selected_die = -1;
 661                 return;
 662         }
 663
 664         writel_relaxed(marvell_nand->ndtr0, nfc->regs + NDTR0);
 665         writel_relaxed(marvell_nand->ndtr1, nfc->regs + NDTR1);
 666
 667         /*
 668          * Reset the NDCR register to a clean state for this particular chip,
 669          * also clear ND_RUN bit.
 670          */
 671         ndcr_generic = readl_relaxed(nfc->regs + NDCR) &
 672                        NDCR_GENERIC_FIELDS_MASK & ~NDCR_ND_RUN;
 673         writel_relaxed(ndcr_generic | marvell_nand->ndcr, nfc->regs + NDCR);
 674
 675         /* Also reset the interrupt status register */
 676         marvell_nfc_clear_int(nfc, NDCR_ALL_INT);
 677
 678         nfc->selected_chip = chip;
 679         marvell_nand->selected_die = die_nr;
 680 }
 681
 682 static irqreturn_t marvell_nfc_isr(int irq, void *dev_id)
 683 {
 684         struct marvell_nfc *nfc = dev_id;
 685         u32 st = readl_relaxed(nfc->regs + NDSR);
 686         u32 ien = (~readl_relaxed(nfc->regs + NDCR)) & NDCR_ALL_INT;
 687
 688         /*
 689          * RDY interrupt mask is one bit in NDCR while there are two status
 690          * bit in NDSR (RDY[cs0/cs2] and RDY[cs1/cs3]).
 691          */
 692         if (st & NDSR_RDY(1))
 693                 st |= NDSR_RDY(0);
 694
 695         if (!(st & ien))
 696                 return IRQ_NONE;
 697
 698         marvell_nfc_disable_int(nfc, st & NDCR_ALL_INT);
 699
 700         if (st & (NDSR_RDY(0) | NDSR_RDY(1)))
 701                 complete(&nfc->complete);
 702
 703         return IRQ_HANDLED;
 704 }
 705
 706 /* HW ECC related functions */
 707 static void marvell_nfc_enable_hw_ecc(struct nand_chip *chip)
 708 {
 709         struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
 710         u32 ndcr = readl_relaxed(nfc->regs + NDCR);
 711
 712         if (!(ndcr & NDCR_ECC_EN)) {
 713                 writel_relaxed(ndcr | NDCR_ECC_EN, nfc->regs + NDCR);
 714
 715                 /*
 716                  * When enabling BCH, set threshold to 0 to always know the
 717                  * number of corrected bitflips.
 718                  */
 719                 if (chip->ecc.algo == NAND_ECC_BCH)
 720                         writel_relaxed(NDECCCTRL_BCH_EN, nfc->regs + NDECCCTRL);
 721         }
 722 }
 723
 724 static void marvell_nfc_disable_hw_ecc(struct nand_chip *chip)
 725 {
 726         struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
 727         u32 ndcr = readl_relaxed(nfc->regs + NDCR);
 728
 729         if (ndcr & NDCR_ECC_EN) {
 730                 writel_relaxed(ndcr & ~NDCR_ECC_EN, nfc->regs + NDCR);
 731                 if (chip->ecc.algo == NAND_ECC_BCH)
 732                         writel_relaxed(0, nfc->regs + NDECCCTRL);
 733         }
 734 }
 735
 736 /* DMA related helpers */
 737 static void marvell_nfc_enable_dma(struct marvell_nfc *nfc)
 738 {
 739         u32 reg;
 740
 741         reg = readl_relaxed(nfc->regs + NDCR);
 742         writel_relaxed(reg | NDCR_DMA_EN, nfc->regs + NDCR);
 743 }
 744
 745 static void marvell_nfc_disable_dma(struct marvell_nfc *nfc)
 746 {
 747         u32 reg;
 748
 749         reg = readl_relaxed(nfc->regs + NDCR);
 750         writel_relaxed(reg & ~NDCR_DMA_EN, nfc->regs + NDCR);
 751 }
 752
 753 /* Read/write PIO/DMA accessors */
 754 static int marvell_nfc_xfer_data_dma(struct marvell_nfc *nfc,
 755                                      enum dma_data_direction direction,
 756                                      unsigned int len)
 757 {
 758         unsigned int dma_len = min_t(int, ALIGN(len, 32), MAX_CHUNK_SIZE);
 759         struct dma_async_tx_descriptor *tx;
 760         struct scatterlist sg;
 761         dma_cookie_t cookie;
 762         int ret;
 763
 764         marvell_nfc_enable_dma(nfc);
 765         /* Prepare the DMA transfer */
 766         sg_init_one(&sg, nfc->dma_buf, dma_len);
 767         dma_map_sg(nfc->dma_chan->device->dev, &sg, 1, direction);
 768         tx = dmaengine_prep_slave_sg(nfc->dma_chan, &sg, 1,
 769                                      direction == DMA_FROM_DEVICE ?
 770                                      DMA_DEV_TO_MEM : DMA_MEM_TO_DEV,
 771                                      DMA_PREP_INTERRUPT);
 772         if (!tx) {
 773                 dev_err(nfc->dev, "Could not prepare DMA S/G list\n");
 774                 return -ENXIO;
 775         }
 776
 777         /* Do the task and wait for it to finish */
 778         cookie = dmaengine_submit(tx);
 779         ret = dma_submit_error(cookie);
 780         if (ret)
 781                 return -EIO;
 782
 783         dma_async_issue_pending(nfc->dma_chan);
 784         ret = marvell_nfc_wait_cmdd(nfc->selected_chip);
 785         dma_unmap_sg(nfc->dma_chan->device->dev, &sg, 1, direction);
 786         marvell_nfc_disable_dma(nfc);
 787         if (ret) {
 788                 dev_err(nfc->dev, "Timeout waiting for DMA (status: %d)\n",
 789                         dmaengine_tx_status(nfc->dma_chan, cookie, NULL));
 790                 dmaengine_terminate_all(nfc->dma_chan);
 791                 return -ETIMEDOUT;
 792         }
 793
 794         return 0;
 795 }
 796
 797 static int marvell_nfc_xfer_data_in_pio(struct marvell_nfc *nfc, u8 *in,
 798                                         unsigned int len)
 799 {
 800         unsigned int last_len = len % FIFO_DEPTH;
 801         unsigned int last_full_offset = round_down(len, FIFO_DEPTH);
 802         int i;
 803
 804         for (i = 0; i < last_full_offset; i += FIFO_DEPTH)
 805                 ioread32_rep(nfc->regs + NDDB, in + i, FIFO_REP(FIFO_DEPTH));
 806
 807         if (last_len) {
 808                 u8 tmp_buf[FIFO_DEPTH];
 809
 810                 ioread32_rep(nfc->regs + NDDB, tmp_buf, FIFO_REP(FIFO_DEPTH));
 811                 memcpy(in + last_full_offset, tmp_buf, last_len);
 812         }
 813
 814         return 0;
 815 }
 816
 817 static int marvell_nfc_xfer_data_out_pio(struct marvell_nfc *nfc, const u8 *out,
 818                                          unsigned int len)
 819 {
 820         unsigned int last_len = len % FIFO_DEPTH;
 821         unsigned int last_full_offset = round_down(len, FIFO_DEPTH);
 822         int i;
 823
 824         for (i = 0; i < last_full_offset; i += FIFO_DEPTH)
 825                 iowrite32_rep(nfc->regs + NDDB, out + i, FIFO_REP(FIFO_DEPTH));
 826
 827         if (last_len) {
 828                 u8 tmp_buf[FIFO_DEPTH];
 829
 830                 memcpy(tmp_buf, out + last_full_offset, last_len);
 831                 iowrite32_rep(nfc->regs + NDDB, tmp_buf, FIFO_REP(FIFO_DEPTH));
 832         }
 833
 834         return 0;
 835 }
 836
 837 static void marvell_nfc_check_empty_chunk(struct nand_chip *chip,
 838                                           u8 *data, int data_len,
 839                                           u8 *spare, int spare_len,
 840                                           u8 *ecc, int ecc_len,
 841                                           unsigned int *max_bitflips)
 842 {
 843         struct mtd_info *mtd = nand_to_mtd(chip);
 844         int bf;
 845
 846         /*
 847          * Blank pages (all 0xFF) that have not been written may be recognized
 848          * as bad if bitflips occur, so whenever an uncorrectable error occurs,
 849          * check if the entire page (with ECC bytes) is actually blank or not.
 850          */
 851         if (!data)
 852                 data_len = 0;
 853         if (!spare)
 854                 spare_len = 0;
 855         if (!ecc)
 856                 ecc_len = 0;
 857
 858         bf = nand_check_erased_ecc_chunk(data, data_len, ecc, ecc_len,
 859                                          spare, spare_len, chip->ecc.strength);
 860         if (bf < 0) {
 861                 mtd->ecc_stats.failed++;
 862                 return;
 863         }
 864
 865         /* Update the stats and max_bitflips */
 866         mtd->ecc_stats.corrected += bf;
 867         *max_bitflips = max_t(unsigned int, *max_bitflips, bf);
 868 }
 869
 870 /*
 871  * Check a chunk is correct or not according to hardware ECC engine.
 872  * mtd->ecc_stats.corrected is updated, as well as max_bitflips, however
 873  * mtd->ecc_stats.failure is not, the function will instead return a non-zero
 874  * value indicating that a check on the emptyness of the subpage must be
 875  * performed before declaring the subpage corrupted.
 876  */
 877 static int marvell_nfc_hw_ecc_correct(struct nand_chip *chip,
 878                                       unsigned int *max_bitflips)
 879 {
 880         struct mtd_info *mtd = nand_to_mtd(chip);
 881         struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
 882         int bf = 0;
 883         u32 ndsr;
 884
 885         ndsr = readl_relaxed(nfc->regs + NDSR);
 886
 887         /* Check uncorrectable error flag */
 888         if (ndsr & NDSR_UNCERR) {
 889                 writel_relaxed(ndsr, nfc->regs + NDSR);
 890
 891                 /*
 892                  * Do not increment ->ecc_stats.failed now, instead, return a
 893                  * non-zero value to indicate that this chunk was apparently
 894                  * bad, and it should be check to see if it empty or not. If
 895                  * the chunk (with ECC bytes) is not declared empty, the calling
 896                  * function must increment the failure count.
 897                  */
 898                 return -EBADMSG;
 899         }
 900
 901         /* Check correctable error flag */
 902         if (ndsr & NDSR_CORERR) {
 903                 writel_relaxed(ndsr, nfc->regs + NDSR);
 904
 905                 if (chip->ecc.algo == NAND_ECC_BCH)
 906                         bf = NDSR_ERRCNT(ndsr);
 907                 else
 908                         bf = 1;
 909         }
 910
 911         /* Update the stats and max_bitflips */
 912         mtd->ecc_stats.corrected += bf;
 913         *max_bitflips = max_t(unsigned int, *max_bitflips, bf);
 914
 915         return 0;
 916 }
 917
 918 /* Hamming read helpers */
 919 static int marvell_nfc_hw_ecc_hmg_do_read_page(struct nand_chip *chip,
 920                                                u8 *data_buf, u8 *oob_buf,
 921                                                bool raw, int page)
 922 {
 923         struct marvell_nand_chip *marvell_nand = to_marvell_nand(chip);
 924         struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
 925         const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
 926         struct marvell_nfc_op nfc_op = {
 927                 .ndcb[0] = NDCB0_CMD_TYPE(TYPE_READ) |
 928                            NDCB0_ADDR_CYC(marvell_nand->addr_cyc) |
 929                            NDCB0_DBC |
 930                            NDCB0_CMD1(NAND_CMD_READ0) |
 931                            NDCB0_CMD2(NAND_CMD_READSTART),
 932                 .ndcb[1] = NDCB1_ADDRS_PAGE(page),
 933                 .ndcb[2] = NDCB2_ADDR5_PAGE(page),
 934         };
 935         unsigned int oob_bytes = lt->spare_bytes + (raw ? lt->ecc_bytes : 0);
 936         int ret;
 937
 938         /* NFCv2 needs more information about the operation being executed */
 939         if (nfc->caps->is_nfcv2)
 940                 nfc_op.ndcb[0] |= NDCB0_CMD_XTYPE(XTYPE_MONOLITHIC_RW);
 941
 942         ret = marvell_nfc_prepare_cmd(chip);
 943         if (ret)
 944                 return ret;
 945
 946         marvell_nfc_send_cmd(chip, &nfc_op);
 947         ret = marvell_nfc_end_cmd(chip, NDSR_RDDREQ,
 948                                   "RDDREQ while draining FIFO (data/oob)");
 949         if (ret)
 950                 return ret;
 951
 952         /*
 953          * Read the page then the OOB area. Unlike what is shown in current
 954          * documentation, spare bytes are protected by the ECC engine, and must
 955          * be at the beginning of the OOB area or running this driver on legacy
 956          * systems will prevent the discovery of the BBM/BBT.
 957          */
 958         if (nfc->use_dma) {
 959                 marvell_nfc_xfer_data_dma(nfc, DMA_FROM_DEVICE,
 960                                           lt->data_bytes + oob_bytes);
 961                 memcpy(data_buf, nfc->dma_buf, lt->data_bytes);
 962                 memcpy(oob_buf, nfc->dma_buf + lt->data_bytes, oob_bytes);
 963         } else {
 964                 marvell_nfc_xfer_data_in_pio(nfc, data_buf, lt->data_bytes);
 965                 marvell_nfc_xfer_data_in_pio(nfc, oob_buf, oob_bytes);
 966         }
 967
 968         ret = marvell_nfc_wait_cmdd(chip);
 969
 970         return ret;
 971 }
 972
 973 static int marvell_nfc_hw_ecc_hmg_read_page_raw(struct mtd_info *mtd,
 974                                                 struct nand_chip *chip, u8 *buf,
 975                                                 int oob_required, int page)
 976 {
 977         return marvell_nfc_hw_ecc_hmg_do_read_page(chip, buf, chip->oob_poi,
 978                                                    true, page);
 979 }
 980
 981 static int marvell_nfc_hw_ecc_hmg_read_page(struct mtd_info *mtd,
 982                                             struct nand_chip *chip,
 983                                             u8 *buf, int oob_required,
 984                                             int page)
 985 {
 986         const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
 987         unsigned int full_sz = lt->data_bytes + lt->spare_bytes + lt->ecc_bytes;
 988         int max_bitflips = 0, ret;
 989         u8 *raw_buf;
 990
 991         marvell_nfc_enable_hw_ecc(chip);
 992         marvell_nfc_hw_ecc_hmg_do_read_page(chip, buf, chip->oob_poi, false,
 993                                             page);
 994         ret = marvell_nfc_hw_ecc_correct(chip, &max_bitflips);
 995         marvell_nfc_disable_hw_ecc(chip);
 996
 997         if (!ret)
 998                 return max_bitflips;
 999
1000         /*
1001          * When ECC failures are detected, check if the full page has been
1002          * written or not. Ignore the failure if it is actually empty.
1003          */
1004         raw_buf = kmalloc(full_sz, GFP_KERNEL);
1005         if (!raw_buf)
1006                 return -ENOMEM;
1007
1008         marvell_nfc_hw_ecc_hmg_do_read_page(chip, raw_buf, raw_buf +
1009                                             lt->data_bytes, true, page);
1010         marvell_nfc_check_empty_chunk(chip, raw_buf, full_sz, NULL, 0, NULL, 0,
1011                                       &max_bitflips);
1012         kfree(raw_buf);
1013
1014         return max_bitflips;
1015 }
1016
1017 /*
1018  * Spare area in Hamming layouts is not protected by the ECC engine (even if
1019  * it appears before the ECC bytes when reading), the ->read_oob_raw() function
1020  * also stands for ->read_oob().
1021  */
1022 static int marvell_nfc_hw_ecc_hmg_read_oob_raw(struct mtd_info *mtd,
1023                                                struct nand_chip *chip, int page)
1024 {
1025         /* Invalidate page cache */
1026         chip->pagebuf = -1;
1027
1028         return marvell_nfc_hw_ecc_hmg_do_read_page(chip, chip->data_buf,
1029                                                    chip->oob_poi, true, page);
1030 }
1031
1032 /* Hamming write helpers */
1033 static int marvell_nfc_hw_ecc_hmg_do_write_page(struct nand_chip *chip,
1034                                                 const u8 *data_buf,
1035                                                 const u8 *oob_buf, bool raw,
1036                                                 int page)
1037 {
1038         struct marvell_nand_chip *marvell_nand = to_marvell_nand(chip);
1039         struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
1040         const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
1041         struct marvell_nfc_op nfc_op = {
1042                 .ndcb[0] = NDCB0_CMD_TYPE(TYPE_WRITE) |
1043                            NDCB0_ADDR_CYC(marvell_nand->addr_cyc) |
1044                            NDCB0_CMD1(NAND_CMD_SEQIN) |
1045                            NDCB0_CMD2(NAND_CMD_PAGEPROG) |
1046                            NDCB0_DBC,
1047                 .ndcb[1] = NDCB1_ADDRS_PAGE(page),
1048                 .ndcb[2] = NDCB2_ADDR5_PAGE(page),
1049         };
1050         unsigned int oob_bytes = lt->spare_bytes + (raw ? lt->ecc_bytes : 0);
1051         int ret;
1052
1053         /* NFCv2 needs more information about the operation being executed */
1054         if (nfc->caps->is_nfcv2)
1055                 nfc_op.ndcb[0] |= NDCB0_CMD_XTYPE(XTYPE_MONOLITHIC_RW);
1056
1057         ret = marvell_nfc_prepare_cmd(chip);
1058         if (ret)
1059                 return ret;
1060
1061         marvell_nfc_send_cmd(chip, &nfc_op);
1062         ret = marvell_nfc_end_cmd(chip, NDSR_WRDREQ,
1063                                   "WRDREQ while loading FIFO (data)");
1064         if (ret)
1065                 return ret;
1066
1067         /* Write the page then the OOB area */
1068         if (nfc->use_dma) {
1069                 memcpy(nfc->dma_buf, data_buf, lt->data_bytes);
1070                 memcpy(nfc->dma_buf + lt->data_bytes, oob_buf, oob_bytes);
1071                 marvell_nfc_xfer_data_dma(nfc, DMA_TO_DEVICE, lt->data_bytes +
1072                                           lt->ecc_bytes + lt->spare_bytes);
1073         } else {
1074                 marvell_nfc_xfer_data_out_pio(nfc, data_buf, lt->data_bytes);
1075                 marvell_nfc_xfer_data_out_pio(nfc, oob_buf, oob_bytes);
1076         }
1077
1078         ret = marvell_nfc_wait_cmdd(chip);
1079         if (ret)
1080                 return ret;
1081
1082         ret = marvell_nfc_wait_op(chip,
1083                                   PSEC_TO_MSEC(chip->data_interface.timings.sdr.tPROG_max));
1084         return ret;
1085 }
1086
1087 static int marvell_nfc_hw_ecc_hmg_write_page_raw(struct mtd_info *mtd,
1088                                                  struct nand_chip *chip,
1089                                                  const u8 *buf,
1090                                                  int oob_required, int page)
1091 {
1092         return marvell_nfc_hw_ecc_hmg_do_write_page(chip, buf, chip->oob_poi,
1093                                                     true, page);
1094 }
1095
1096 static int marvell_nfc_hw_ecc_hmg_write_page(struct mtd_info *mtd,
1097                                              struct nand_chip *chip,
1098                                              const u8 *buf,
1099                                              int oob_required, int page)
1100 {
1101         int ret;
1102
1103         marvell_nfc_enable_hw_ecc(chip);
1104         ret = marvell_nfc_hw_ecc_hmg_do_write_page(chip, buf, chip->oob_poi,
1105                                                    false, page);
1106         marvell_nfc_disable_hw_ecc(chip);
1107
1108         return ret;
1109 }
1110
1111 /*
1112  * Spare area in Hamming layouts is not protected by the ECC engine (even if
1113  * it appears before the ECC bytes when reading), the ->write_oob_raw() function
1114  * also stands for ->write_oob().
1115  */
1116 static int marvell_nfc_hw_ecc_hmg_write_oob_raw(struct mtd_info *mtd,
1117                                                 struct nand_chip *chip,
1118                                                 int page)
1119 {
1120         /* Invalidate page cache */
1121         chip->pagebuf = -1;
1122
1123         memset(chip->data_buf, 0xFF, mtd->writesize);
1124
1125         return marvell_nfc_hw_ecc_hmg_do_write_page(chip, chip->data_buf,
1126                                                     chip->oob_poi, true, page);
1127 }
1128
1129 /* BCH read helpers */
1130 static int marvell_nfc_hw_ecc_bch_read_page_raw(struct mtd_info *mtd,
1131                                                 struct nand_chip *chip, u8 *buf,
1132                                                 int oob_required, int page)
1133 {
1134         const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
1135         u8 *oob = chip->oob_poi;
1136         int chunk_size = lt->data_bytes + lt->spare_bytes + lt->ecc_bytes;
1137         int ecc_offset = (lt->full_chunk_cnt * lt->spare_bytes) +
1138                 lt->last_spare_bytes;
1139         int data_len = lt->data_bytes;
1140         int spare_len = lt->spare_bytes;
1141         int ecc_len = lt->ecc_bytes;
1142         int chunk;
1143
1144         if (oob_required)
1145                 memset(chip->oob_poi, 0xFF, mtd->oobsize);
1146
1147         nand_read_page_op(chip, page, 0, NULL, 0);
1148
1149         for (chunk = 0; chunk < lt->nchunks; chunk++) {
1150                 /* Update last chunk length */
1151                 if (chunk >= lt->full_chunk_cnt) {
1152                         data_len = lt->last_data_bytes;
1153                         spare_len = lt->last_spare_bytes;
1154                         ecc_len = lt->last_ecc_bytes;
1155                 }
1156
1157                 /* Read data bytes*/
1158                 nand_change_read_column_op(chip, chunk * chunk_size,
1159                                            buf + (lt->data_bytes * chunk),
1160                                            data_len, false);
1161
1162                 /* Read spare bytes */
1163                 nand_read_data_op(chip, oob + (lt->spare_bytes * chunk),
1164                                   spare_len, false);
1165
1166                 /* Read ECC bytes */
1167                 nand_read_data_op(chip, oob + ecc_offset +
1168                                   (ALIGN(lt->ecc_bytes, 32) * chunk),
1169                                   ecc_len, false);
1170         }
1171
1172         return 0;
1173 }
1174
1175 static void marvell_nfc_hw_ecc_bch_read_chunk(struct nand_chip *chip, int chunk,
1176                                               u8 *data, unsigned int data_len,
1177                                               u8 *spare, unsigned int spare_len,
1178                                               int page)
1179 {
1180         struct marvell_nand_chip *marvell_nand = to_marvell_nand(chip);
1181         struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
1182         const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
1183         int i, ret;
1184         struct marvell_nfc_op nfc_op = {
1185                 .ndcb[0] = NDCB0_CMD_TYPE(TYPE_READ) |
1186                            NDCB0_ADDR_CYC(marvell_nand->addr_cyc) |
1187                            NDCB0_LEN_OVRD,
1188                 .ndcb[1] = NDCB1_ADDRS_PAGE(page),
1189                 .ndcb[2] = NDCB2_ADDR5_PAGE(page),
1190                 .ndcb[3] = data_len + spare_len,
1191         };
1192
1193         ret = marvell_nfc_prepare_cmd(chip);
1194         if (ret)
1195                 return;
1196
1197         if (chunk == 0)
1198                 nfc_op.ndcb[0] |= NDCB0_DBC |
1199                                   NDCB0_CMD1(NAND_CMD_READ0) |
1200                                   NDCB0_CMD2(NAND_CMD_READSTART);
1201
1202         /*
1203          * Trigger the monolithic read on the first chunk, then naked read on
1204          * intermediate chunks and finally a last naked read on the last chunk.
1205          */
1206         if (chunk == 0)
1207                 nfc_op.ndcb[0] |= NDCB0_CMD_XTYPE(XTYPE_MONOLITHIC_RW);
1208         else if (chunk < lt->nchunks - 1)
1209                 nfc_op.ndcb[0] |= NDCB0_CMD_XTYPE(XTYPE_NAKED_RW);
1210         else
1211                 nfc_op.ndcb[0] |= NDCB0_CMD_XTYPE(XTYPE_LAST_NAKED_RW);
1212
1213         marvell_nfc_send_cmd(chip, &nfc_op);
1214
1215         /*
1216          * According to the datasheet, when reading from NDDB
1217          * with BCH enabled, after each 32 bytes reads, we
1218          * have to make sure that the NDSR.RDDREQ bit is set.
1219          *
1220          * Drain the FIFO, 8 32-bit reads at a time, and skip
1221          * the polling on the last read.
1222          *
1223          * Length is a multiple of 32 bytes, hence it is a multiple of 8 too.
1224          */
1225         for (i = 0; i < data_len; i += FIFO_DEPTH * BCH_SEQ_READS) {
1226                 marvell_nfc_end_cmd(chip, NDSR_RDDREQ,
1227                                     "RDDREQ while draining FIFO (data)");
1228                 marvell_nfc_xfer_data_in_pio(nfc, data,
1229                                              FIFO_DEPTH * BCH_SEQ_READS);
1230                 data += FIFO_DEPTH * BCH_SEQ_READS;
1231         }
1232
1233         for (i = 0; i < spare_len; i += FIFO_DEPTH * BCH_SEQ_READS) {
1234                 marvell_nfc_end_cmd(chip, NDSR_RDDREQ,
1235                                     "RDDREQ while draining FIFO (OOB)");
1236                 marvell_nfc_xfer_data_in_pio(nfc, spare,
1237                                              FIFO_DEPTH * BCH_SEQ_READS);
1238                 spare += FIFO_DEPTH * BCH_SEQ_READS;
1239         }
1240 }
1241
1242 static int marvell_nfc_hw_ecc_bch_read_page(struct mtd_info *mtd,
1243                                             struct nand_chip *chip,
1244                                             u8 *buf, int oob_required,
1245                                             int page)
1246 {
1247         const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
1248         int data_len = lt->data_bytes, spare_len = lt->spare_bytes, ecc_len;
1249         u8 *data = buf, *spare = chip->oob_poi, *ecc;
1250         int max_bitflips = 0;
1251         u32 failure_mask = 0;
1252         int chunk, ecc_offset_in_page, ret;
1253
1254         /*
1255          * With BCH, OOB is not fully used (and thus not read entirely), not
1256          * expected bytes could show up at the end of the OOB buffer if not
1257          * explicitly erased.
1258          */
1259         if (oob_required)
1260                 memset(chip->oob_poi, 0xFF, mtd->oobsize);
1261
1262         marvell_nfc_enable_hw_ecc(chip);
1263
1264         for (chunk = 0; chunk < lt->nchunks; chunk++) {
1265                 /* Update length for the last chunk */
1266                 if (chunk >= lt->full_chunk_cnt) {
1267                         data_len = lt->last_data_bytes;
1268                         spare_len = lt->last_spare_bytes;
1269                 }
1270
1271                 /* Read the chunk and detect number of bitflips */
1272                 marvell_nfc_hw_ecc_bch_read_chunk(chip, chunk, data, data_len,
1273                                                   spare, spare_len, page);
1274                 ret = marvell_nfc_hw_ecc_correct(chip, &max_bitflips);
1275                 if (ret)
1276                         failure_mask |= BIT(chunk);
1277
1278                 data += data_len;
1279                 spare += spare_len;
1280         }
1281
1282         marvell_nfc_disable_hw_ecc(chip);
1283
1284         if (!failure_mask)
1285                 return max_bitflips;
1286
1287         /*
1288          * Please note that dumping the ECC bytes during a normal read with OOB
1289          * area would add a significant overhead as ECC bytes are "consumed" by
1290          * the controller in normal mode and must be re-read in raw mode. To
1291          * avoid dropping the performances, we prefer not to include them. The
1292          * user should re-read the page in raw mode if ECC bytes are required.
1293          *
1294          * However, for any subpage read error reported by ->correct(), the ECC
1295          * bytes must be read in raw mode and the full subpage must be checked
1296          * to see if it is entirely empty of if there was an actual error.
1297          */
1298         for (chunk = 0; chunk < lt->nchunks; chunk++) {
1299                 /* No failure reported for this chunk, move to the next one */
1300                 if (!(failure_mask & BIT(chunk)))
1301                         continue;
1302
1303                 /* Derive ECC bytes positions (in page/buffer) and length */
1304                 ecc = chip->oob_poi +
1305                         (lt->full_chunk_cnt * lt->spare_bytes) +
1306                         lt->last_spare_bytes +
1307                         (chunk * ALIGN(lt->ecc_bytes, 32));
1308                 ecc_offset_in_page =
1309                         (chunk * (lt->data_bytes + lt->spare_bytes +
1310                                   lt->ecc_bytes)) +
1311                         (chunk < lt->full_chunk_cnt ?
1312                          lt->data_bytes + lt->spare_bytes :
1313                          lt->last_data_bytes + lt->last_spare_bytes);
1314                 ecc_len = chunk < lt->full_chunk_cnt ?
1315                         lt->ecc_bytes : lt->last_ecc_bytes;
1316
1317                 /* Do the actual raw read of the ECC bytes */
1318                 nand_change_read_column_op(chip, ecc_offset_in_page,
1319                                            ecc, ecc_len, false);
1320
1321                 /* Derive data/spare bytes positions (in buffer) and length */
1322                 data = buf + (chunk * lt->data_bytes);
1323                 data_len = chunk < lt->full_chunk_cnt ?
1324                         lt->data_bytes : lt->last_data_bytes;
1325                 spare = chip->oob_poi + (chunk * (lt->spare_bytes +
1326                                                   lt->ecc_bytes));
1327                 spare_len = chunk < lt->full_chunk_cnt ?
1328                         lt->spare_bytes : lt->last_spare_bytes;
1329
1330                 /* Check the entire chunk (data + spare + ecc) for emptyness */
1331                 marvell_nfc_check_empty_chunk(chip, data, data_len, spare,
1332                                               spare_len, ecc, ecc_len,
1333                                               &max_bitflips);
1334         }
1335
1336         return max_bitflips;
1337 }
1338
1339 static int marvell_nfc_hw_ecc_bch_read_oob_raw(struct mtd_info *mtd,
1340                                                struct nand_chip *chip, int page)
1341 {
1342         /* Invalidate page cache */
1343         chip->pagebuf = -1;
1344
1345         return chip->ecc.read_page_raw(mtd, chip, chip->data_buf, true, page);
1346 }
1347
1348 static int marvell_nfc_hw_ecc_bch_read_oob(struct mtd_info *mtd,
1349                                            struct nand_chip *chip, int page)
1350 {
1351         /* Invalidate page cache */
1352         chip->pagebuf = -1;
1353
1354         return chip->ecc.read_page(mtd, chip, chip->data_buf, true, page);
1355 }
1356
1357 /* BCH write helpers */
1358 static int marvell_nfc_hw_ecc_bch_write_page_raw(struct mtd_info *mtd,
1359                                                  struct nand_chip *chip,
1360                                                  const u8 *buf,
1361                                                  int oob_required, int page)
1362 {
1363         const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
1364         int full_chunk_size = lt->data_bytes + lt->spare_bytes + lt->ecc_bytes;
1365         int data_len = lt->data_bytes;
1366         int spare_len = lt->spare_bytes;
1367         int ecc_len = lt->ecc_bytes;
1368         int spare_offset = 0;
1369         int ecc_offset = (lt->full_chunk_cnt * lt->spare_bytes) +
1370                 lt->last_spare_bytes;
1371         int chunk;
1372
1373         nand_prog_page_begin_op(chip, page, 0, NULL, 0);
1374
1375         for (chunk = 0; chunk < lt->nchunks; chunk++) {
1376                 if (chunk >= lt->full_chunk_cnt) {
1377                         data_len = lt->last_data_bytes;
1378                         spare_len = lt->last_spare_bytes;
1379                         ecc_len = lt->last_ecc_bytes;
1380                 }
1381
1382                 /* Point to the column of the next chunk */
1383                 nand_change_write_column_op(chip, chunk * full_chunk_size,
1384                                             NULL, 0, false);
1385
1386                 /* Write the data */
1387                 nand_write_data_op(chip, buf + (chunk * lt->data_bytes),
1388                                    data_len, false);
1389
1390                 if (!oob_required)
1391                         continue;
1392
1393                 /* Write the spare bytes */
1394                 if (spare_len)
1395                         nand_write_data_op(chip, chip->oob_poi + spare_offset,
1396                                            spare_len, false);
1397
1398                 /* Write the ECC bytes */
1399                 if (ecc_len)
1400                         nand_write_data_op(chip, chip->oob_poi + ecc_offset,
1401                                            ecc_len, false);
1402
1403                 spare_offset += spare_len;
1404                 ecc_offset += ALIGN(ecc_len, 32);
1405         }
1406
1407         return nand_prog_page_end_op(chip);
1408 }
1409
1410 static int
1411 marvell_nfc_hw_ecc_bch_write_chunk(struct nand_chip *chip, int chunk,
1412                                    const u8 *data, unsigned int data_len,
1413                                    const u8 *spare, unsigned int spare_len,
1414                                    int page)
1415 {
1416         struct marvell_nand_chip *marvell_nand = to_marvell_nand(chip);
1417         struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
1418         const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
1419         u32 xtype;
1420         int ret;
1421         struct marvell_nfc_op nfc_op = {
1422                 .ndcb[0] = NDCB0_CMD_TYPE(TYPE_WRITE) | NDCB0_LEN_OVRD,
1423                 .ndcb[3] = data_len + spare_len,
1424         };
1425
1426         /*
1427          * First operation dispatches the CMD_SEQIN command, issue the address
1428          * cycles and asks for the first chunk of data.
1429          * All operations in the middle (if any) will issue a naked write and
1430          * also ask for data.
1431          * Last operation (if any) asks for the last chunk of data through a
1432          * last naked write.
1433          */
1434         if (chunk == 0) {
1435                 if (lt->nchunks == 1)
1436                         xtype = XTYPE_MONOLITHIC_RW;
1437                 else
1438                         xtype = XTYPE_WRITE_DISPATCH;
1439
1440                 nfc_op.ndcb[0] |= NDCB0_CMD_XTYPE(xtype) |
1441                                   NDCB0_ADDR_CYC(marvell_nand->addr_cyc) |
1442                                   NDCB0_CMD1(NAND_CMD_SEQIN);
1443                 nfc_op.ndcb[1] |= NDCB1_ADDRS_PAGE(page);
1444                 nfc_op.ndcb[2] |= NDCB2_ADDR5_PAGE(page);
1445         } else if (chunk < lt->nchunks - 1) {
1446                 nfc_op.ndcb[0] |= NDCB0_CMD_XTYPE(XTYPE_NAKED_RW);
1447         } else {
1448                 nfc_op.ndcb[0] |= NDCB0_CMD_XTYPE(XTYPE_LAST_NAKED_RW);
1449         }
1450
1451         /* Always dispatch the PAGEPROG command on the last chunk */
1452         if (chunk == lt->nchunks - 1)
1453                 nfc_op.ndcb[0] |= NDCB0_CMD2(NAND_CMD_PAGEPROG) | NDCB0_DBC;
1454
1455         ret = marvell_nfc_prepare_cmd(chip);
1456         if (ret)
1457                 return ret;
1458
1459         marvell_nfc_send_cmd(chip, &nfc_op);
1460         ret = marvell_nfc_end_cmd(chip, NDSR_WRDREQ,
1461                                   "WRDREQ while loading FIFO (data)");
1462         if (ret)
1463                 return ret;
1464
1465         /* Transfer the contents */
1466         iowrite32_rep(nfc->regs + NDDB, data, FIFO_REP(data_len));
1467         iowrite32_rep(nfc->regs + NDDB, spare, FIFO_REP(spare_len));
1468
1469         return 0;
1470 }
1471
1472 static int marvell_nfc_hw_ecc_bch_write_page(struct mtd_info *mtd,
1473                                              struct nand_chip *chip,
1474                                              const u8 *buf,
1475                                              int oob_required, int page)
1476 {
1477         const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
1478         const u8 *data = buf;
1479         const u8 *spare = chip->oob_poi;
1480         int data_len = lt->data_bytes;
1481         int spare_len = lt->spare_bytes;
1482         int chunk, ret;
1483
1484         /* Spare data will be written anyway, so clear it to avoid garbage */
1485         if (!oob_required)
1486                 memset(chip->oob_poi, 0xFF, mtd->oobsize);
1487
1488         marvell_nfc_enable_hw_ecc(chip);
1489
1490         for (chunk = 0; chunk < lt->nchunks; chunk++) {
1491                 if (chunk >= lt->full_chunk_cnt) {
1492                         data_len = lt->last_data_bytes;
1493                         spare_len = lt->last_spare_bytes;
1494                 }
1495
1496                 marvell_nfc_hw_ecc_bch_write_chunk(chip, chunk, data, data_len,
1497                                                    spare, spare_len, page);
1498                 data += data_len;
1499                 spare += spare_len;
1500
1501                 /*
1502                  * Waiting only for CMDD or PAGED is not enough, ECC are
1503                  * partially written. No flag is set once the operation is
1504                  * really finished but the ND_RUN bit is cleared, so wait for it
1505                  * before stepping into the next command.
1506                  */
1507                 marvell_nfc_wait_ndrun(chip);
1508         }
1509
1510         ret = marvell_nfc_wait_op(chip,
1511                                   PSEC_TO_MSEC(chip->data_interface.timings.sdr.tPROG_max));
1512
1513         marvell_nfc_disable_hw_ecc(chip);
1514
1515         if (ret)
1516                 return ret;
1517
1518         return 0;
1519 }
1520
1521 static int marvell_nfc_hw_ecc_bch_write_oob_raw(struct mtd_info *mtd,
1522                                                 struct nand_chip *chip,
1523                                                 int page)
1524 {
1525         /* Invalidate page cache */
1526         chip->pagebuf = -1;
1527
1528         memset(chip->data_buf, 0xFF, mtd->writesize);
1529
1530         return chip->ecc.write_page_raw(mtd, chip, chip->data_buf, true, page);
1531 }
1532
1533 static int marvell_nfc_hw_ecc_bch_write_oob(struct mtd_info *mtd,
1534                                             struct nand_chip *chip, int page)
1535 {
1536         /* Invalidate page cache */
1537         chip->pagebuf = -1;
1538
1539         memset(chip->data_buf, 0xFF, mtd->writesize);
1540
1541         return chip->ecc.write_page(mtd, chip, chip->data_buf, true, page);
1542 }
1543
1544 /* NAND framework ->exec_op() hooks and related helpers */
1545 static void marvell_nfc_parse_instructions(struct nand_chip *chip,
1546                                            const struct nand_subop *subop,
1547                                            struct marvell_nfc_op *nfc_op)
1548 {
1549         const struct nand_op_instr *instr = NULL;
1550         struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
1551         bool first_cmd = true;
1552         unsigned int op_id;
1553         int i;
1554
1555         /* Reset the input structure as most of its fields will be OR'ed */
1556         memset(nfc_op, 0, sizeof(struct marvell_nfc_op));
1557
1558         for (op_id = 0; op_id < subop->ninstrs; op_id++) {
1559                 unsigned int offset, naddrs;
1560                 const u8 *addrs;
1561                 int len;
1562
1563                 instr = &subop->instrs[op_id];
1564
1565                 switch (instr->type) {
1566                 case NAND_OP_CMD_INSTR:
1567                         if (first_cmd)
1568                                 nfc_op->ndcb[0] |=
1569                                         NDCB0_CMD1(instr->ctx.cmd.opcode);
1570                         else
1571                                 nfc_op->ndcb[0] |=
1572                                         NDCB0_CMD2(instr->ctx.cmd.opcode) |
1573                                         NDCB0_DBC;
1574
1575                         nfc_op->cle_ale_delay_ns = instr->delay_ns;
1576                         first_cmd = false;
1577                         break;
1578
1579                 case NAND_OP_ADDR_INSTR:
1580                         offset = nand_subop_get_addr_start_off(subop, op_id);
1581                         naddrs = nand_subop_get_num_addr_cyc(subop, op_id);
1582                         addrs = &instr->ctx.addr.addrs[offset];
1583
1584                         nfc_op->ndcb[0] |= NDCB0_ADDR_CYC(naddrs);
1585
1586                         for (i = 0; i < min_t(unsigned int, 4, naddrs); i++)
1587                                 nfc_op->ndcb[1] |= addrs[i] << (8 * i);
1588
1589                         if (naddrs >= 5)
1590                                 nfc_op->ndcb[2] |= NDCB2_ADDR5_CYC(addrs[4]);
1591                         if (naddrs >= 6)
1592                                 nfc_op->ndcb[3] |= NDCB3_ADDR6_CYC(addrs[5]);
1593                         if (naddrs == 7)
1594                                 nfc_op->ndcb[3] |= NDCB3_ADDR7_CYC(addrs[6]);
1595
1596                         nfc_op->cle_ale_delay_ns = instr->delay_ns;
1597                         break;
1598
1599                 case NAND_OP_DATA_IN_INSTR:
1600                         nfc_op->data_instr = instr;
1601                         nfc_op->data_instr_idx = op_id;
1602                         nfc_op->ndcb[0] |= NDCB0_CMD_TYPE(TYPE_READ);
1603                         if (nfc->caps->is_nfcv2) {
1604                                 nfc_op->ndcb[0] |=
1605                                         NDCB0_CMD_XTYPE(XTYPE_MONOLITHIC_RW) |
1606                                         NDCB0_LEN_OVRD;
1607                                 len = nand_subop_get_data_len(subop, op_id);
1608                                 nfc_op->ndcb[3] |= round_up(len, FIFO_DEPTH);
1609                         }
1610                         nfc_op->data_delay_ns = instr->delay_ns;
1611                         break;
1612
1613                 case NAND_OP_DATA_OUT_INSTR:
1614                         nfc_op->data_instr = instr;
1615                         nfc_op->data_instr_idx = op_id;
1616                         nfc_op->ndcb[0] |= NDCB0_CMD_TYPE(TYPE_WRITE);
1617                         if (nfc->caps->is_nfcv2) {
1618                                 nfc_op->ndcb[0] |=
1619                                         NDCB0_CMD_XTYPE(XTYPE_MONOLITHIC_RW) |
1620                                         NDCB0_LEN_OVRD;
1621                                 len = nand_subop_get_data_len(subop, op_id);
1622                                 nfc_op->ndcb[3] |= round_up(len, FIFO_DEPTH);
1623                         }
1624                         nfc_op->data_delay_ns = instr->delay_ns;
1625                         break;
1626
1627                 case NAND_OP_WAITRDY_INSTR:
1628                         nfc_op->rdy_timeout_ms = instr->ctx.waitrdy.timeout_ms;
1629                         nfc_op->rdy_delay_ns = instr->delay_ns;
1630                         break;
1631                 }
1632         }
1633 }
1634
1635 static int marvell_nfc_xfer_data_pio(struct nand_chip *chip,
1636                                      const struct nand_subop *subop,
1637                                      struct marvell_nfc_op *nfc_op)
1638 {
1639         struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
1640         const struct nand_op_instr *instr = nfc_op->data_instr;
1641         unsigned int op_id = nfc_op->data_instr_idx;
1642         unsigned int len = nand_subop_get_data_len(subop, op_id);
1643         unsigned int offset = nand_subop_get_data_start_off(subop, op_id);
1644         bool reading = (instr->type == NAND_OP_DATA_IN_INSTR);
1645         int ret;
1646
1647         if (instr->ctx.data.force_8bit)
1648                 marvell_nfc_force_byte_access(chip, true);
1649
1650         if (reading) {
1651                 u8 *in = instr->ctx.data.buf.in + offset;
1652
1653                 ret = marvell_nfc_xfer_data_in_pio(nfc, in, len);
1654         } else {
1655                 const u8 *out = instr->ctx.data.buf.out + offset;
1656
1657                 ret = marvell_nfc_xfer_data_out_pio(nfc, out, len);
1658         }
1659
1660         if (instr->ctx.data.force_8bit)
1661                 marvell_nfc_force_byte_access(chip, false);
1662
1663         return ret;
1664 }
1665
1666 static int marvell_nfc_monolithic_access_exec(struct nand_chip *chip,
1667                                               const struct nand_subop *subop)
1668 {
1669         struct marvell_nfc_op nfc_op;
1670         bool reading;
1671         int ret;
1672
1673         marvell_nfc_parse_instructions(chip, subop, &nfc_op);
1674         reading = (nfc_op.data_instr->type == NAND_OP_DATA_IN_INSTR);
1675
1676         ret = marvell_nfc_prepare_cmd(chip);
1677         if (ret)
1678                 return ret;
1679
1680         marvell_nfc_send_cmd(chip, &nfc_op);
1681         ret = marvell_nfc_end_cmd(chip, NDSR_RDDREQ | NDSR_WRDREQ,
1682                                   "RDDREQ/WRDREQ while draining raw data");
1683         if (ret)
1684                 return ret;
1685
1686         cond_delay(nfc_op.cle_ale_delay_ns);
1687
1688         if (reading) {
1689                 if (nfc_op.rdy_timeout_ms) {
1690                         ret = marvell_nfc_wait_op(chip, nfc_op.rdy_timeout_ms);
1691                         if (ret)
1692                                 return ret;
1693                 }
1694
1695                 cond_delay(nfc_op.rdy_delay_ns);
1696         }
1697
1698         marvell_nfc_xfer_data_pio(chip, subop, &nfc_op);
1699         ret = marvell_nfc_wait_cmdd(chip);
1700         if (ret)
1701                 return ret;
1702
1703         cond_delay(nfc_op.data_delay_ns);
1704
1705         if (!reading) {
1706                 if (nfc_op.rdy_timeout_ms) {
1707                         ret = marvell_nfc_wait_op(chip, nfc_op.rdy_timeout_ms);
1708                         if (ret)
1709                                 return ret;
1710                 }
1711
1712                 cond_delay(nfc_op.rdy_delay_ns);
1713         }
1714
1715         /*
1716          * NDCR ND_RUN bit should be cleared automatically at the end of each
1717          * operation but experience shows that the behavior is buggy when it
1718          * comes to writes (with LEN_OVRD). Clear it by hand in this case.
1719          */
1720         if (!reading) {
1721                 struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
1722
1723                 writel_relaxed(readl(nfc->regs + NDCR) & ~NDCR_ND_RUN,
1724                                nfc->regs + NDCR);
1725         }
1726
1727         return 0;
1728 }
1729
1730 static int marvell_nfc_naked_access_exec(struct nand_chip *chip,
1731                                          const struct nand_subop *subop)
1732 {
1733         struct marvell_nfc_op nfc_op;
1734         int ret;
1735
1736         marvell_nfc_parse_instructions(chip, subop, &nfc_op);
1737
1738         /*
1739          * Naked access are different in that they need to be flagged as naked
1740          * by the controller. Reset the controller registers fields that inform
1741          * on the type and refill them according to the ongoing operation.
1742          */
1743         nfc_op.ndcb[0] &= ~(NDCB0_CMD_TYPE(TYPE_MASK) |
1744                             NDCB0_CMD_XTYPE(XTYPE_MASK));
1745         switch (subop->instrs[0].type) {
1746         case NAND_OP_CMD_INSTR:
1747                 nfc_op.ndcb[0] |= NDCB0_CMD_TYPE(TYPE_NAKED_CMD);
1748                 break;
1749         case NAND_OP_ADDR_INSTR:
1750                 nfc_op.ndcb[0] |= NDCB0_CMD_TYPE(TYPE_NAKED_ADDR);
1751                 break;
1752         case NAND_OP_DATA_IN_INSTR:
1753                 nfc_op.ndcb[0] |= NDCB0_CMD_TYPE(TYPE_READ) |
1754                                   NDCB0_CMD_XTYPE(XTYPE_LAST_NAKED_RW);
1755                 break;
1756         case NAND_OP_DATA_OUT_INSTR:
1757                 nfc_op.ndcb[0] |= NDCB0_CMD_TYPE(TYPE_WRITE) |
1758                                   NDCB0_CMD_XTYPE(XTYPE_LAST_NAKED_RW);
1759                 break;
1760         default:
1761                 /* This should never happen */
1762                 break;
1763         }
1764
1765         ret = marvell_nfc_prepare_cmd(chip);
1766         if (ret)
1767                 return ret;
1768
1769         marvell_nfc_send_cmd(chip, &nfc_op);
1770
1771         if (!nfc_op.data_instr) {
1772                 ret = marvell_nfc_wait_cmdd(chip);
1773                 cond_delay(nfc_op.cle_ale_delay_ns);
1774                 return ret;
1775         }
1776
1777         ret = marvell_nfc_end_cmd(chip, NDSR_RDDREQ | NDSR_WRDREQ,
1778                                   "RDDREQ/WRDREQ while draining raw data");
1779         if (ret)
1780                 return ret;
1781
1782         marvell_nfc_xfer_data_pio(chip, subop, &nfc_op);
1783         ret = marvell_nfc_wait_cmdd(chip);
1784         if (ret)
1785                 return ret;
1786
1787         /*
1788          * NDCR ND_RUN bit should be cleared automatically at the end of each
1789          * operation but experience shows that the behavior is buggy when it
1790          * comes to writes (with LEN_OVRD). Clear it by hand in this case.
1791          */
1792         if (subop->instrs[0].type == NAND_OP_DATA_OUT_INSTR) {
1793                 struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
1794
1795                 writel_relaxed(readl(nfc->regs + NDCR) & ~NDCR_ND_RUN,
1796                                nfc->regs + NDCR);
1797         }
1798
1799         return 0;
1800 }
1801
1802 static int marvell_nfc_naked_waitrdy_exec(struct nand_chip *chip,
1803                                           const struct nand_subop *subop)
1804 {
1805         struct marvell_nfc_op nfc_op;
1806         int ret;
1807
1808         marvell_nfc_parse_instructions(chip, subop, &nfc_op);
1809
1810         ret = marvell_nfc_wait_op(chip, nfc_op.rdy_timeout_ms);
1811         cond_delay(nfc_op.rdy_delay_ns);
1812
1813         return ret;
1814 }
1815
1816 static int marvell_nfc_read_id_type_exec(struct nand_chip *chip,
1817                                          const struct nand_subop *subop)
1818 {
1819         struct marvell_nfc_op nfc_op;
1820         int ret;
1821
1822         marvell_nfc_parse_instructions(chip, subop, &nfc_op);
1823         nfc_op.ndcb[0] &= ~NDCB0_CMD_TYPE(TYPE_READ);
1824         nfc_op.ndcb[0] |= NDCB0_CMD_TYPE(TYPE_READ_ID);
1825
1826         ret = marvell_nfc_prepare_cmd(chip);
1827         if (ret)
1828                 return ret;
1829
1830         marvell_nfc_send_cmd(chip, &nfc_op);
1831         ret = marvell_nfc_end_cmd(chip, NDSR_RDDREQ,
1832                                   "RDDREQ while reading ID");
1833         if (ret)
1834                 return ret;
1835
1836         cond_delay(nfc_op.cle_ale_delay_ns);
1837
1838         if (nfc_op.rdy_timeout_ms) {
1839                 ret = marvell_nfc_wait_op(chip, nfc_op.rdy_timeout_ms);
1840                 if (ret)
1841                         return ret;
1842         }
1843
1844         cond_delay(nfc_op.rdy_delay_ns);
1845
1846         marvell_nfc_xfer_data_pio(chip, subop, &nfc_op);
1847         ret = marvell_nfc_wait_cmdd(chip);
1848         if (ret)
1849                 return ret;
1850
1851         cond_delay(nfc_op.data_delay_ns);
1852
1853         return 0;
1854 }
1855
1856 static int marvell_nfc_read_status_exec(struct nand_chip *chip,
1857                                         const struct nand_subop *subop)
1858 {
1859         struct marvell_nfc_op nfc_op;
1860         int ret;
1861
1862         marvell_nfc_parse_instructions(chip, subop, &nfc_op);
1863         nfc_op.ndcb[0] &= ~NDCB0_CMD_TYPE(TYPE_READ);
1864         nfc_op.ndcb[0] |= NDCB0_CMD_TYPE(TYPE_STATUS);
1865
1866         ret = marvell_nfc_prepare_cmd(chip);
1867         if (ret)
1868                 return ret;
1869
1870         marvell_nfc_send_cmd(chip, &nfc_op);
1871         ret = marvell_nfc_end_cmd(chip, NDSR_RDDREQ,
1872                                   "RDDREQ while reading status");
1873         if (ret)
1874                 return ret;
1875
1876         cond_delay(nfc_op.cle_ale_delay_ns);
1877
1878         if (nfc_op.rdy_timeout_ms) {
1879                 ret = marvell_nfc_wait_op(chip, nfc_op.rdy_timeout_ms);
1880                 if (ret)
1881                         return ret;
1882         }
1883
1884         cond_delay(nfc_op.rdy_delay_ns);
1885
1886         marvell_nfc_xfer_data_pio(chip, subop, &nfc_op);
1887         ret = marvell_nfc_wait_cmdd(chip);
1888         if (ret)
1889                 return ret;
1890
1891         cond_delay(nfc_op.data_delay_ns);
1892
1893         return 0;
1894 }
1895
1896 static int marvell_nfc_reset_cmd_type_exec(struct nand_chip *chip,
1897                                            const struct nand_subop *subop)
1898 {
1899         struct marvell_nfc_op nfc_op;
1900         int ret;
1901
1902         marvell_nfc_parse_instructions(chip, subop, &nfc_op);
1903         nfc_op.ndcb[0] |= NDCB0_CMD_TYPE(TYPE_RESET);
1904
1905         ret = marvell_nfc_prepare_cmd(chip);
1906         if (ret)
1907                 return ret;
1908
1909         marvell_nfc_send_cmd(chip, &nfc_op);
1910         ret = marvell_nfc_wait_cmdd(chip);
1911         if (ret)
1912                 return ret;
1913
1914         cond_delay(nfc_op.cle_ale_delay_ns);
1915
1916         ret = marvell_nfc_wait_op(chip, nfc_op.rdy_timeout_ms);
1917         if (ret)
1918                 return ret;
1919
1920         cond_delay(nfc_op.rdy_delay_ns);
1921
1922         return 0;
1923 }
1924
1925 static int marvell_nfc_erase_cmd_type_exec(struct nand_chip *chip,
1926                                            const struct nand_subop *subop)
1927 {
1928         struct marvell_nfc_op nfc_op;
1929         int ret;
1930
1931         marvell_nfc_parse_instructions(chip, subop, &nfc_op);
1932         nfc_op.ndcb[0] |= NDCB0_CMD_TYPE(TYPE_ERASE);
1933
1934         ret = marvell_nfc_prepare_cmd(chip);
1935         if (ret)
1936                 return ret;
1937
1938         marvell_nfc_send_cmd(chip, &nfc_op);
1939         ret = marvell_nfc_wait_cmdd(chip);
1940         if (ret)
1941                 return ret;
1942
1943         cond_delay(nfc_op.cle_ale_delay_ns);
1944
1945         ret = marvell_nfc_wait_op(chip, nfc_op.rdy_timeout_ms);
1946         if (ret)
1947                 return ret;
1948
1949         cond_delay(nfc_op.rdy_delay_ns);
1950
1951         return 0;
1952 }
1953
1954 static const struct nand_op_parser marvell_nfcv2_op_parser = NAND_OP_PARSER(
1955         /* Monolithic reads/writes */
1956         NAND_OP_PARSER_PATTERN(
1957                 marvell_nfc_monolithic_access_exec,
1958                 NAND_OP_PARSER_PAT_CMD_ELEM(false),
1959                 NAND_OP_PARSER_PAT_ADDR_ELEM(true, MAX_ADDRESS_CYC_NFCV2),
1960                 NAND_OP_PARSER_PAT_CMD_ELEM(true),
1961                 NAND_OP_PARSER_PAT_WAITRDY_ELEM(true),
1962                 NAND_OP_PARSER_PAT_DATA_IN_ELEM(false, MAX_CHUNK_SIZE)),
1963         NAND_OP_PARSER_PATTERN(
1964                 marvell_nfc_monolithic_access_exec,
1965                 NAND_OP_PARSER_PAT_CMD_ELEM(false),
1966                 NAND_OP_PARSER_PAT_ADDR_ELEM(false, MAX_ADDRESS_CYC_NFCV2),
1967                 NAND_OP_PARSER_PAT_DATA_OUT_ELEM(false, MAX_CHUNK_SIZE),
1968                 NAND_OP_PARSER_PAT_CMD_ELEM(true),
1969                 NAND_OP_PARSER_PAT_WAITRDY_ELEM(true)),
1970         /* Naked commands */
1971         NAND_OP_PARSER_PATTERN(
1972                 marvell_nfc_naked_access_exec,
1973                 NAND_OP_PARSER_PAT_CMD_ELEM(false)),
1974         NAND_OP_PARSER_PATTERN(
1975                 marvell_nfc_naked_access_exec,
1976                 NAND_OP_PARSER_PAT_ADDR_ELEM(false, MAX_ADDRESS_CYC_NFCV2)),
1977         NAND_OP_PARSER_PATTERN(
1978                 marvell_nfc_naked_access_exec,
1979                 NAND_OP_PARSER_PAT_DATA_IN_ELEM(false, MAX_CHUNK_SIZE)),
1980         NAND_OP_PARSER_PATTERN(
1981                 marvell_nfc_naked_access_exec,
1982                 NAND_OP_PARSER_PAT_DATA_OUT_ELEM(false, MAX_CHUNK_SIZE)),
1983         NAND_OP_PARSER_PATTERN(
1984                 marvell_nfc_naked_waitrdy_exec,
1985                 NAND_OP_PARSER_PAT_WAITRDY_ELEM(false)),
1986         );
1987
1988 static const struct nand_op_parser marvell_nfcv1_op_parser = NAND_OP_PARSER(
1989         /* Naked commands not supported, use a function for each pattern */
1990         NAND_OP_PARSER_PATTERN(
1991                 marvell_nfc_read_id_type_exec,
1992                 NAND_OP_PARSER_PAT_CMD_ELEM(false),
1993                 NAND_OP_PARSER_PAT_ADDR_ELEM(false, MAX_ADDRESS_CYC_NFCV1),
1994                 NAND_OP_PARSER_PAT_DATA_IN_ELEM(false, 8)),
1995         NAND_OP_PARSER_PATTERN(
1996                 marvell_nfc_erase_cmd_type_exec,
1997                 NAND_OP_PARSER_PAT_CMD_ELEM(false),
1998                 NAND_OP_PARSER_PAT_ADDR_ELEM(false, MAX_ADDRESS_CYC_NFCV1),
1999                 NAND_OP_PARSER_PAT_CMD_ELEM(false),
2000                 NAND_OP_PARSER_PAT_WAITRDY_ELEM(false)),
2001         NAND_OP_PARSER_PATTERN(
2002                 marvell_nfc_read_status_exec,
2003                 NAND_OP_PARSER_PAT_CMD_ELEM(false),
2004                 NAND_OP_PARSER_PAT_DATA_IN_ELEM(false, 1)),
2005         NAND_OP_PARSER_PATTERN(
2006                 marvell_nfc_reset_cmd_type_exec,
2007                 NAND_OP_PARSER_PAT_CMD_ELEM(false),
2008                 NAND_OP_PARSER_PAT_WAITRDY_ELEM(false)),
2009         NAND_OP_PARSER_PATTERN(
2010                 marvell_nfc_naked_waitrdy_exec,
2011                 NAND_OP_PARSER_PAT_WAITRDY_ELEM(false)),
2012         );
2013
2014 static int marvell_nfc_exec_op(struct nand_chip *chip,
2015                                const struct nand_operation *op,
2016                                bool check_only)
2017 {
2018         struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
2019
2020         if (nfc->caps->is_nfcv2)
2021                 return nand_op_parser_exec_op(chip, &marvell_nfcv2_op_parser,
2022                                               op, check_only);
2023         else
2024                 return nand_op_parser_exec_op(chip, &marvell_nfcv1_op_parser,
2025                                               op, check_only);
2026 }
2027
2028 /*
2029  * Layouts were broken in old pxa3xx_nand driver, these are supposed to be
2030  * usable.
2031  */
2032 static int marvell_nand_ooblayout_ecc(struct mtd_info *mtd, int section,
2033                                       struct mtd_oob_region *oobregion)
2034 {
2035         struct nand_chip *chip = mtd_to_nand(mtd);
2036         const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
2037
2038         if (section)
2039                 return -ERANGE;
2040
2041         oobregion->length = (lt->full_chunk_cnt * lt->ecc_bytes) +
2042                             lt->last_ecc_bytes;
2043         oobregion->offset = mtd->oobsize - oobregion->length;
2044
2045         return 0;
2046 }
2047
2048 static int marvell_nand_ooblayout_free(struct mtd_info *mtd, int section,
2049                                        struct mtd_oob_region *oobregion)
2050 {
2051         struct nand_chip *chip = mtd_to_nand(mtd);
2052         const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
2053
2054         if (section)
2055                 return -ERANGE;
2056
2057         /*
2058          * Bootrom looks in bytes 0 & 5 for bad blocks for the
2059          * 4KB page / 4bit BCH combination.
2060          */
2061         if (mtd->writesize == SZ_4K && lt->data_bytes == SZ_2K)
2062                 oobregion->offset = 6;
2063         else
2064                 oobregion->offset = 2;
2065
2066         oobregion->length = (lt->full_chunk_cnt * lt->spare_bytes) +
2067                             lt->last_spare_bytes - oobregion->offset;
2068
2069         return 0;
2070 }
2071
2072 static const struct mtd_ooblayout_ops marvell_nand_ooblayout_ops = {
2073         .ecc = marvell_nand_ooblayout_ecc,
2074         .free = marvell_nand_ooblayout_free,
2075 };
2076
2077 static int marvell_nand_hw_ecc_ctrl_init(struct mtd_info *mtd,
2078                                          struct nand_ecc_ctrl *ecc)
2079 {
2080         struct nand_chip *chip = mtd_to_nand(mtd);
2081         struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
2082         const struct marvell_hw_ecc_layout *l;
2083         int i;
2084
2085         if (!nfc->caps->is_nfcv2 &&
2086             (mtd->writesize + mtd->oobsize > MAX_CHUNK_SIZE)) {
2087                 dev_err(nfc->dev,
2088                         "NFCv1: writesize (%d) cannot be bigger than a chunk (%d)\n",
2089                         mtd->writesize, MAX_CHUNK_SIZE - mtd->oobsize);
2090                 return -ENOTSUPP;
2091         }
2092
2093         to_marvell_nand(chip)->layout = NULL;
2094         for (i = 0; i < ARRAY_SIZE(marvell_nfc_layouts); i++) {
2095                 l = &marvell_nfc_layouts[i];
2096                 if (mtd->writesize == l->writesize &&
2097                     ecc->size == l->chunk && ecc->strength == l->strength) {
2098                         to_marvell_nand(chip)->layout = l;
2099                         break;
2100                 }
2101         }
2102
2103         if (!to_marvell_nand(chip)->layout ||
2104             (!nfc->caps->is_nfcv2 && ecc->strength > 1)) {
2105                 dev_err(nfc->dev,
2106                         "ECC strength %d at page size %d is not supported\n",
2107                         ecc->strength, mtd->writesize);
2108                 return -ENOTSUPP;
2109         }
2110
2111         mtd_set_ooblayout(mtd, &marvell_nand_ooblayout_ops);
2112         ecc->steps = l->nchunks;
2113         ecc->size = l->data_bytes;
2114
2115         if (ecc->strength == 1) {
2116                 chip->ecc.algo = NAND_ECC_HAMMING;
2117                 ecc->read_page_raw = marvell_nfc_hw_ecc_hmg_read_page_raw;
2118                 ecc->read_page = marvell_nfc_hw_ecc_hmg_read_page;
2119                 ecc->read_oob_raw = marvell_nfc_hw_ecc_hmg_read_oob_raw;
2120                 ecc->read_oob = ecc->read_oob_raw;
2121                 ecc->write_page_raw = marvell_nfc_hw_ecc_hmg_write_page_raw;
2122                 ecc->write_page = marvell_nfc_hw_ecc_hmg_write_page;
2123                 ecc->write_oob_raw = marvell_nfc_hw_ecc_hmg_write_oob_raw;
2124                 ecc->write_oob = ecc->write_oob_raw;
2125         } else {
2126                 chip->ecc.algo = NAND_ECC_BCH;
2127                 ecc->strength = 16;
2128                 ecc->read_page_raw = marvell_nfc_hw_ecc_bch_read_page_raw;
2129                 ecc->read_page = marvell_nfc_hw_ecc_bch_read_page;
2130                 ecc->read_oob_raw = marvell_nfc_hw_ecc_bch_read_oob_raw;
2131                 ecc->read_oob = marvell_nfc_hw_ecc_bch_read_oob;
2132                 ecc->write_page_raw = marvell_nfc_hw_ecc_bch_write_page_raw;
2133                 ecc->write_page = marvell_nfc_hw_ecc_bch_write_page;
2134                 ecc->write_oob_raw = marvell_nfc_hw_ecc_bch_write_oob_raw;
2135                 ecc->write_oob = marvell_nfc_hw_ecc_bch_write_oob;
2136         }
2137
2138         return 0;
2139 }
2140
2141 static int marvell_nand_ecc_init(struct mtd_info *mtd,
2142                                  struct nand_ecc_ctrl *ecc)
2143 {
2144         struct nand_chip *chip = mtd_to_nand(mtd);
2145         struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
2146         int ret;
2147
2148         if (ecc->mode != NAND_ECC_NONE && (!ecc->size || !ecc->strength)) {
2149                 if (chip->ecc_step_ds && chip->ecc_strength_ds) {
2150                         ecc->size = chip->ecc_step_ds;
2151                         ecc->strength = chip->ecc_strength_ds;
2152                 } else {
2153                         dev_info(nfc->dev,
2154                                  "No minimum ECC strength, using 1b/512B\n");
2155                         ecc->size = 512;
2156                         ecc->strength = 1;
2157                 }
2158         }
2159
2160         switch (ecc->mode) {
2161         case NAND_ECC_HW:
2162                 ret = marvell_nand_hw_ecc_ctrl_init(mtd, ecc);
2163                 if (ret)
2164                         return ret;
2165                 break;
2166         case NAND_ECC_NONE:
2167         case NAND_ECC_SOFT:
2168         case NAND_ECC_ON_DIE:
2169                 if (!nfc->caps->is_nfcv2 && mtd->writesize != SZ_512 &&
2170                     mtd->writesize != SZ_2K) {
2171                         dev_err(nfc->dev, "NFCv1 cannot write %d bytes pages\n",
2172                                 mtd->writesize);
2173                         return -EINVAL;
2174                 }
2175                 break;
2176         default:
2177                 return -EINVAL;
2178         }
2179
2180         return 0;
2181 }
2182
2183 static u8 bbt_pattern[] = {'M', 'V', 'B', 'b', 't', '0' };
2184 static u8 bbt_mirror_pattern[] = {'1', 't', 'b', 'B', 'V', 'M' };
2185
2186 static struct nand_bbt_descr bbt_main_descr = {
2187         .options = NAND_BBT_LASTBLOCK | NAND_BBT_CREATE | NAND_BBT_WRITE |
2188                    NAND_BBT_2BIT | NAND_BBT_VERSION,
2189         .offs = 8,
2190         .len = 6,
2191         .veroffs = 14,
2192         .maxblocks = 8, /* Last 8 blocks in each chip */
2193         .pattern = bbt_pattern
2194 };
2195
2196 static struct nand_bbt_descr bbt_mirror_descr = {
2197         .options = NAND_BBT_LASTBLOCK | NAND_BBT_CREATE | NAND_BBT_WRITE |
2198                    NAND_BBT_2BIT | NAND_BBT_VERSION,
2199         .offs = 8,
2200         .len = 6,
2201         .veroffs = 14,
2202         .maxblocks = 8, /* Last 8 blocks in each chip */
2203         .pattern = bbt_mirror_pattern
2204 };
2205
2206 static int marvell_nfc_setup_data_interface(struct mtd_info *mtd, int chipnr,
2207                                             const struct nand_data_interface
2208                                             *conf)
2209 {
2210         struct nand_chip *chip = mtd_to_nand(mtd);
2211         struct marvell_nand_chip *marvell_nand = to_marvell_nand(chip);
2212         struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
2213         unsigned int period_ns = 1000000000 / clk_get_rate(nfc->core_clk) * 2;
2214         const struct nand_sdr_timings *sdr;
2215         struct marvell_nfc_timings nfc_tmg;
2216         int read_delay;
2217
2218         sdr = nand_get_sdr_timings(conf);
2219         if (IS_ERR(sdr))
2220                 return PTR_ERR(sdr);
2221
2222         /*
2223          * SDR timings are given in pico-seconds while NFC timings must be
2224          * expressed in NAND controller clock cycles, which is half of the
2225          * frequency of the accessible ECC clock retrieved by clk_get_rate().
2226          * This is not written anywhere in the datasheet but was observed
2227          * with an oscilloscope.
2228          *
2229          * NFC datasheet gives equations from which thoses calculations
2230          * are derived, they tend to be slightly more restrictives than the
2231          * given core timings and may improve the overall speed.
2232          */
2233         nfc_tmg.tRP = TO_CYCLES(DIV_ROUND_UP(sdr->tRC_min, 2), period_ns) - 1;
2234         nfc_tmg.tRH = nfc_tmg.tRP;
2235         nfc_tmg.tWP = TO_CYCLES(DIV_ROUND_UP(sdr->tWC_min, 2), period_ns) - 1;
2236         nfc_tmg.tWH = nfc_tmg.tWP;
2237         nfc_tmg.tCS = TO_CYCLES(sdr->tCS_min, period_ns);
2238         nfc_tmg.tCH = TO_CYCLES(sdr->tCH_min, period_ns) - 1;
2239         nfc_tmg.tADL = TO_CYCLES(sdr->tADL_min, period_ns);
2240         /*
2241          * Read delay is the time of propagation from SoC pins to NFC internal
2242          * logic. With non-EDO timings, this is MIN_RD_DEL_CNT clock cycles. In
2243          * EDO mode, an additional delay of tRH must be taken into account so
2244          * the data is sampled on the falling edge instead of the rising edge.
2245          */
2246         read_delay = sdr->tRC_min >= 30000 ?
2247                 MIN_RD_DEL_CNT : MIN_RD_DEL_CNT + nfc_tmg.tRH;
2248
2249         nfc_tmg.tAR = TO_CYCLES(sdr->tAR_min, period_ns);
2250         /*
2251          * tWHR and tRHW are supposed to be read to write delays (and vice
2252          * versa) but in some cases, ie. when doing a change column, they must
2253          * be greater than that to be sure tCCS delay is respected.
2254          */
2255         nfc_tmg.tWHR = TO_CYCLES(max_t(int, sdr->tWHR_min, sdr->tCCS_min),
2256                                  period_ns) - 2,
2257         nfc_tmg.tRHW = TO_CYCLES(max_t(int, sdr->tRHW_min, sdr->tCCS_min),
2258                                  period_ns);
2259
2260         /*
2261          * NFCv2: Use WAIT_MODE (wait for RB line), do not rely only on delays.
2262          * NFCv1: No WAIT_MODE, tR must be maximal.
2263          */
2264         if (nfc->caps->is_nfcv2) {
2265                 nfc_tmg.tR = TO_CYCLES(sdr->tWB_max, period_ns);
2266         } else {
2267                 nfc_tmg.tR = TO_CYCLES64(sdr->tWB_max + sdr->tR_max,
2268                                          period_ns);
2269                 if (nfc_tmg.tR + 3 > nfc_tmg.tCH)
2270                         nfc_tmg.tR = nfc_tmg.tCH - 3;
2271                 else
2272                         nfc_tmg.tR = 0;
2273         }
2274
2275         if (chipnr < 0)
2276                 return 0;
2277
2278         marvell_nand->ndtr0 =
2279                 NDTR0_TRP(nfc_tmg.tRP) |
2280                 NDTR0_TRH(nfc_tmg.tRH) |
2281                 NDTR0_ETRP(nfc_tmg.tRP) |
2282                 NDTR0_TWP(nfc_tmg.tWP) |
2283                 NDTR0_TWH(nfc_tmg.tWH) |
2284                 NDTR0_TCS(nfc_tmg.tCS) |
2285                 NDTR0_TCH(nfc_tmg.tCH);
2286
2287         marvell_nand->ndtr1 =
2288                 NDTR1_TAR(nfc_tmg.tAR) |
2289                 NDTR1_TWHR(nfc_tmg.tWHR) |
2290                 NDTR1_TR(nfc_tmg.tR);
2291
2292         if (nfc->caps->is_nfcv2) {
2293                 marvell_nand->ndtr0 |=
2294                         NDTR0_RD_CNT_DEL(read_delay) |
2295                         NDTR0_SELCNTR |
2296                         NDTR0_TADL(nfc_tmg.tADL);
2297
2298                 marvell_nand->ndtr1 |=
2299                         NDTR1_TRHW(nfc_tmg.tRHW) |
2300                         NDTR1_WAIT_MODE;
2301         }
2302
2303         return 0;
2304 }
2305
2306 static int marvell_nand_attach_chip(struct nand_chip *chip)
2307 {
2308         struct mtd_info *mtd = nand_to_mtd(chip);
2309         struct marvell_nand_chip *marvell_nand = to_marvell_nand(chip);
2310         struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
2311         struct pxa3xx_nand_platform_data *pdata = dev_get_platdata(nfc->dev);
2312         int ret;
2313
2314         if (pdata && pdata->flash_bbt)
2315                 chip->bbt_options |= NAND_BBT_USE_FLASH;
2316
2317         if (chip->bbt_options & NAND_BBT_USE_FLASH) {
2318                 /*
2319                  * We'll use a bad block table stored in-flash and don't
2320                  * allow writing the bad block marker to the flash.
2321                  */
2322                 chip->bbt_options |= NAND_BBT_NO_OOB_BBM;
2323                 chip->bbt_td = &bbt_main_descr;
2324                 chip->bbt_md = &bbt_mirror_descr;
2325         }
2326
2327         /* Save the chip-specific fields of NDCR */
2328         marvell_nand->ndcr = NDCR_PAGE_SZ(mtd->writesize);
2329         if (chip->options & NAND_BUSWIDTH_16)
2330                 marvell_nand->ndcr |= NDCR_DWIDTH_M | NDCR_DWIDTH_C;
2331
2332         /*
2333          * On small page NANDs, only one cycle is needed to pass the
2334          * column address.
2335          */
2336         if (mtd->writesize <= 512) {
2337                 marvell_nand->addr_cyc = 1;
2338         } else {
2339                 marvell_nand->addr_cyc = 2;
2340                 marvell_nand->ndcr |= NDCR_RA_START;
2341         }
2342
2343         /*
2344          * Now add the number of cycles needed to pass the row
2345          * address.
2346          *
2347          * Addressing a chip using CS 2 or 3 should also need the third row
2348          * cycle but due to inconsistance in the documentation and lack of
2349          * hardware to test this situation, this case is not supported.
2350          */
2351         if (chip->options & NAND_ROW_ADDR_3)
2352                 marvell_nand->addr_cyc += 3;
2353         else
2354                 marvell_nand->addr_cyc += 2;
2355
2356         if (pdata) {
2357                 chip->ecc.size = pdata->ecc_step_size;
2358                 chip->ecc.strength = pdata->ecc_strength;
2359         }
2360
2361         ret = marvell_nand_ecc_init(mtd, &chip->ecc);
2362         if (ret) {
2363                 dev_err(nfc->dev, "ECC init failed: %d\n", ret);
2364                 return ret;
2365         }
2366
2367         if (chip->ecc.mode == NAND_ECC_HW) {
2368                 /*
2369                  * Subpage write not available with hardware ECC, prohibit also
2370                  * subpage read as in userspace subpage access would still be
2371                  * allowed and subpage write, if used, would lead to numerous
2372                  * uncorrectable ECC errors.
2373                  */
2374                 chip->options |= NAND_NO_SUBPAGE_WRITE;
2375         }
2376
2377         if (pdata || nfc->caps->legacy_of_bindings) {
2378                 /*
2379                  * We keep the MTD name unchanged to avoid breaking platforms
2380                  * where the MTD cmdline parser is used and the bootloader
2381                  * has not been updated to use the new naming scheme.
2382                  */
2383                 mtd->name = "pxa3xx_nand-0";
2384         } else if (!mtd->name) {
2385                 /*
2386                  * If the new bindings are used and the bootloader has not been
2387                  * updated to pass a new mtdparts parameter on the cmdline, you
2388                  * should define the following property in your NAND node, ie:
2389                  *
2390                  *      label = "main-storage";
2391                  *
2392                  * This way, mtd->name will be set by the core when
2393                  * nand_set_flash_node() is called.
2394                  */
2395                 mtd->name = devm_kasprintf(nfc->dev, GFP_KERNEL,
2396                                            "%s:nand.%d", dev_name(nfc->dev),
2397                                            marvell_nand->sels[0].cs);
2398                 if (!mtd->name) {
2399                         dev_err(nfc->dev, "Failed to allocate mtd->name\n");
2400                         return -ENOMEM;
2401                 }
2402         }
2403
2404         return 0;
2405 }
2406
2407 static const struct nand_controller_ops marvell_nand_controller_ops = {
2408         .attach_chip = marvell_nand_attach_chip,
2409 };
2410
2411 static int marvell_nand_chip_init(struct device *dev, struct marvell_nfc *nfc,
2412                                   struct device_node *np)
2413 {
2414         struct pxa3xx_nand_platform_data *pdata = dev_get_platdata(dev);
2415         struct marvell_nand_chip *marvell_nand;
2416         struct mtd_info *mtd;
2417         struct nand_chip *chip;
2418         int nsels, ret, i;
2419         u32 cs, rb;
2420
2421         /*
2422          * The legacy "num-cs" property indicates the number of CS on the only
2423          * chip connected to the controller (legacy bindings does not support
2424          * more than one chip). The CS and RB pins are always the #0.
2425          *
2426          * When not using legacy bindings, a couple of "reg" and "nand-rb"
2427          * properties must be filled. For each chip, expressed as a subnode,
2428          * "reg" points to the CS lines and "nand-rb" to the RB line.
2429          */
2430         if (pdata || nfc->caps->legacy_of_bindings) {
2431                 nsels = 1;
2432         } else {
2433                 nsels = of_property_count_elems_of_size(np, "reg", sizeof(u32));
2434                 if (nsels <= 0) {
2435                         dev_err(dev, "missing/invalid reg property\n");
2436                         return -EINVAL;
2437                 }
2438         }
2439
2440         /* Alloc the nand chip structure */
2441         marvell_nand = devm_kzalloc(dev, sizeof(*marvell_nand) +
2442                                     (nsels *
2443                                      sizeof(struct marvell_nand_chip_sel)),
2444                                     GFP_KERNEL);
2445         if (!marvell_nand) {
2446                 dev_err(dev, "could not allocate chip structure\n");
2447                 return -ENOMEM;
2448         }
2449
2450         marvell_nand->nsels = nsels;
2451         marvell_nand->selected_die = -1;
2452
2453         for (i = 0; i < nsels; i++) {
2454                 if (pdata || nfc->caps->legacy_of_bindings) {
2455                         /*
2456                          * Legacy bindings use the CS lines in natural
2457                          * order (0, 1, ...)
2458                          */
2459                         cs = i;
2460                 } else {
2461                         /* Retrieve CS id */
2462                         ret = of_property_read_u32_index(np, "reg", i, &cs);
2463                         if (ret) {
2464                                 dev_err(dev, "could not retrieve reg property: %d\n",
2465                                         ret);
2466                                 return ret;
2467                         }
2468                 }
2469
2470                 if (cs >= nfc->caps->max_cs_nb) {
2471                         dev_err(dev, "invalid reg value: %u (max CS = %d)\n",
2472                                 cs, nfc->caps->max_cs_nb);
2473                         return -EINVAL;
2474                 }
2475
2476                 if (test_and_set_bit(cs, &nfc->assigned_cs)) {
2477                         dev_err(dev, "CS %d already assigned\n", cs);
2478                         return -EINVAL;
2479                 }
2480
2481                 /*
2482                  * The cs variable represents the chip select id, which must be
2483                  * converted in bit fields for NDCB0 and NDCB2 to select the
2484                  * right chip. Unfortunately, due to a lack of information on
2485                  * the subject and incoherent documentation, the user should not
2486                  * use CS1 and CS3 at all as asserting them is not supported in
2487                  * a reliable way (due to multiplexing inside ADDR5 field).
2488                  */
2489                 marvell_nand->sels[i].cs = cs;
2490                 switch (cs) {
2491                 case 0:
2492                 case 2:
2493                         marvell_nand->sels[i].ndcb0_csel = 0;
2494                         break;
2495                 case 1:
2496                 case 3:
2497                         marvell_nand->sels[i].ndcb0_csel = NDCB0_CSEL;
2498                         break;
2499                 default:
2500                         return -EINVAL;
2501                 }
2502
2503                 /* Retrieve RB id */
2504                 if (pdata || nfc->caps->legacy_of_bindings) {
2505                         /* Legacy bindings always use RB #0 */
2506                         rb = 0;
2507                 } else {
2508                         ret = of_property_read_u32_index(np, "nand-rb", i,
2509                                                          &rb);
2510                         if (ret) {
2511                                 dev_err(dev,
2512                                         "could not retrieve RB property: %d\n",
2513                                         ret);
2514                                 return ret;
2515                         }
2516                 }
2517
2518                 if (rb >= nfc->caps->max_rb_nb) {
2519                         dev_err(dev, "invalid reg value: %u (max RB = %d)\n",
2520                                 rb, nfc->caps->max_rb_nb);
2521                         return -EINVAL;
2522                 }
2523
2524                 marvell_nand->sels[i].rb = rb;
2525         }
2526
2527         chip = &marvell_nand->chip;
2528         chip->controller = &nfc->controller;
2529         nand_set_flash_node(chip, np);
2530
2531         chip->exec_op = marvell_nfc_exec_op;
2532         chip->select_chip = marvell_nfc_select_chip;
2533         if (!of_property_read_bool(np, "marvell,nand-keep-config"))
2534                 chip->setup_data_interface = marvell_nfc_setup_data_interface;
2535
2536         mtd = nand_to_mtd(chip);
2537         mtd->dev.parent = dev;
2538
2539         /*
2540          * Default to HW ECC engine mode. If the nand-ecc-mode property is given
2541          * in the DT node, this entry will be overwritten in nand_scan_ident().
2542          */
2543         chip->ecc.mode = NAND_ECC_HW;
2544
2545         /*
2546          * Save a reference value for timing registers before
2547          * ->setup_data_interface() is called.
2548          */
2549         marvell_nand->ndtr0 = readl_relaxed(nfc->regs + NDTR0);
2550         marvell_nand->ndtr1 = readl_relaxed(nfc->regs + NDTR1);
2551
2552         chip->options |= NAND_BUSWIDTH_AUTO;
2553
2554         ret = nand_scan(chip, marvell_nand->nsels);
2555         if (ret) {
2556                 dev_err(dev, "could not scan the nand chip\n");
2557                 return ret;
2558         }
2559
2560         if (pdata)
2561                 /* Legacy bindings support only one chip */
2562                 ret = mtd_device_register(mtd, pdata->parts, pdata->nr_parts);
2563         else
2564                 ret = mtd_device_register(mtd, NULL, 0);
2565         if (ret) {
2566                 dev_err(dev, "failed to register mtd device: %d\n", ret);
2567                 nand_cleanup(chip);
2568                 return ret;
2569         }
2570
2571         list_add_tail(&marvell_nand->node, &nfc->chips);
2572
2573         return 0;
2574 }
2575
2576 static void marvell_nand_chips_cleanup(struct marvell_nfc *nfc)
2577 {
2578         struct marvell_nand_chip *entry, *temp;
2579
2580         list_for_each_entry_safe(entry, temp, &nfc->chips, node) {
2581                 nand_release(&entry->chip);
2582                 list_del(&entry->node);
2583         }
2584 }
2585
2586 static int marvell_nand_chips_init(struct device *dev, struct marvell_nfc *nfc)
2587 {
2588         struct device_node *np = dev->of_node;
2589         struct device_node *nand_np;
2590         int max_cs = nfc->caps->max_cs_nb;
2591         int nchips;
2592         int ret;
2593
2594         if (!np)
2595                 nchips = 1;
2596         else
2597                 nchips = of_get_child_count(np);
2598
2599         if (nchips > max_cs) {
2600                 dev_err(dev, "too many NAND chips: %d (max = %d CS)\n", nchips,
2601                         max_cs);
2602                 return -EINVAL;
2603         }
2604
2605         /*
2606          * Legacy bindings do not use child nodes to exhibit NAND chip
2607          * properties and layout. Instead, NAND properties are mixed with the
2608          * controller ones, and partitions are defined as direct subnodes of the
2609          * NAND controller node.
2610          */
2611         if (nfc->caps->legacy_of_bindings) {
2612                 ret = marvell_nand_chip_init(dev, nfc, np);
2613                 return ret;
2614         }
2615
2616         for_each_child_of_node(np, nand_np) {
2617                 ret = marvell_nand_chip_init(dev, nfc, nand_np);
2618                 if (ret) {
2619                         of_node_put(nand_np);
2620                         goto cleanup_chips;
2621                 }
2622         }
2623
2624         return 0;
2625
2626 cleanup_chips:
2627         marvell_nand_chips_cleanup(nfc);
2628
2629         return ret;
2630 }
2631
2632 static int marvell_nfc_init_dma(struct marvell_nfc *nfc)
2633 {
2634         struct platform_device *pdev = container_of(nfc->dev,
2635                                                     struct platform_device,
2636                                                     dev);
2637         struct dma_slave_config config = {};
2638         struct resource *r;
2639         int ret;
2640
2641         if (!IS_ENABLED(CONFIG_PXA_DMA)) {
2642                 dev_warn(nfc->dev,
2643                          "DMA not enabled in configuration\n");
2644                 return -ENOTSUPP;
2645         }
2646
2647         ret = dma_set_mask_and_coherent(nfc->dev, DMA_BIT_MASK(32));
2648         if (ret)
2649                 return ret;
2650
2651         nfc->dma_chan = dma_request_slave_channel(nfc->dev, "data");
2652         if (!nfc->dma_chan) {
2653                 dev_err(nfc->dev,
2654                         "Unable to request data DMA channel\n");
2655                 return -ENODEV;
2656         }
2657
2658         r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2659         if (!r)
2660                 return -ENXIO;
2661
2662         config.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
2663         config.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
2664         config.src_addr = r->start + NDDB;
2665         config.dst_addr = r->start + NDDB;
2666         config.src_maxburst = 32;
2667         config.dst_maxburst = 32;
2668         ret = dmaengine_slave_config(nfc->dma_chan, &config);
2669         if (ret < 0) {
2670                 dev_err(nfc->dev, "Failed to configure DMA channel\n");
2671                 return ret;
2672         }
2673
2674         /*
2675          * DMA must act on length multiple of 32 and this length may be
2676          * bigger than the destination buffer. Use this buffer instead
2677          * for DMA transfers and then copy the desired amount of data to
2678          * the provided buffer.
2679          */
2680         nfc->dma_buf = kmalloc(MAX_CHUNK_SIZE, GFP_KERNEL | GFP_DMA);
2681         if (!nfc->dma_buf)
2682                 return -ENOMEM;
2683
2684         nfc->use_dma = true;
2685
2686         return 0;
2687 }
2688
2689 static void marvell_nfc_reset(struct marvell_nfc *nfc)
2690 {
2691         /*
2692          * ECC operations and interruptions are only enabled when specifically
2693          * needed. ECC shall not be activated in the early stages (fails probe).
2694          * Arbiter flag, even if marked as "reserved", must be set (empirical).
2695          * SPARE_EN bit must always be set or ECC bytes will not be at the same
2696          * offset in the read page and this will fail the protection.
2697          */
2698         writel_relaxed(NDCR_ALL_INT | NDCR_ND_ARB_EN | NDCR_SPARE_EN |
2699                        NDCR_RD_ID_CNT(NFCV1_READID_LEN), nfc->regs + NDCR);
2700         writel_relaxed(0xFFFFFFFF, nfc->regs + NDSR);
2701         writel_relaxed(0, nfc->regs + NDECCCTRL);
2702 }
2703
2704 static int marvell_nfc_init(struct marvell_nfc *nfc)
2705 {
2706         struct device_node *np = nfc->dev->of_node;
2707
2708         /*
2709          * Some SoCs like A7k/A8k need to enable manually the NAND
2710          * controller, gated clocks and reset bits to avoid being bootloader
2711          * dependent. This is done through the use of the System Functions
2712          * registers.
2713          */
2714         if (nfc->caps->need_system_controller) {
2715                 struct regmap *sysctrl_base =
2716                         syscon_regmap_lookup_by_phandle(np,
2717                                                         "marvell,system-controller");
2718
2719                 if (IS_ERR(sysctrl_base))
2720                         return PTR_ERR(sysctrl_base);
2721
2722                 regmap_write(sysctrl_base, GENCONF_SOC_DEVICE_MUX,
2723                              GENCONF_SOC_DEVICE_MUX_NFC_EN |
2724                              GENCONF_SOC_DEVICE_MUX_ECC_CLK_RST |
2725                              GENCONF_SOC_DEVICE_MUX_ECC_CORE_RST |
2726                              GENCONF_SOC_DEVICE_MUX_NFC_INT_EN);
2727
2728                 regmap_update_bits(sysctrl_base, GENCONF_CLK_GATING_CTRL,
2729                                    GENCONF_CLK_GATING_CTRL_ND_GATE,
2730                                    GENCONF_CLK_GATING_CTRL_ND_GATE);
2731
2732                 regmap_update_bits(sysctrl_base, GENCONF_ND_CLK_CTRL,
2733                                    GENCONF_ND_CLK_CTRL_EN,
2734                                    GENCONF_ND_CLK_CTRL_EN);
2735         }
2736
2737         /* Configure the DMA if appropriate */
2738         if (!nfc->caps->is_nfcv2)
2739                 marvell_nfc_init_dma(nfc);
2740
2741         marvell_nfc_reset(nfc);
2742
2743         return 0;
2744 }
2745
2746 static int marvell_nfc_probe(struct platform_device *pdev)
2747 {
2748         struct device *dev = &pdev->dev;
2749         struct resource *r;
2750         struct marvell_nfc *nfc;
2751         int ret;
2752         int irq;
2753
2754         nfc = devm_kzalloc(&pdev->dev, sizeof(struct marvell_nfc),
2755                            GFP_KERNEL);
2756         if (!nfc)
2757                 return -ENOMEM;
2758
2759         nfc->dev = dev;
2760         nand_controller_init(&nfc->controller);
2761         nfc->controller.ops = &marvell_nand_controller_ops;
2762         INIT_LIST_HEAD(&nfc->chips);
2763
2764         r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2765         nfc->regs = devm_ioremap_resource(dev, r);
2766         if (IS_ERR(nfc->regs))
2767                 return PTR_ERR(nfc->regs);
2768
2769         irq = platform_get_irq(pdev, 0);
2770         if (irq < 0) {
2771                 dev_err(dev, "failed to retrieve irq\n");
2772                 return irq;
2773         }
2774
2775         nfc->core_clk = devm_clk_get(&pdev->dev, "core");
2776
2777         /* Managed the legacy case (when the first clock was not named) */
2778         if (nfc->core_clk == ERR_PTR(-ENOENT))
2779                 nfc->core_clk = devm_clk_get(&pdev->dev, NULL);
2780
2781         if (IS_ERR(nfc->core_clk))
2782                 return PTR_ERR(nfc->core_clk);
2783
2784         ret = clk_prepare_enable(nfc->core_clk);
2785         if (ret)
2786                 return ret;
2787
2788         nfc->reg_clk = devm_clk_get(&pdev->dev, "reg");
2789         if (IS_ERR(nfc->reg_clk)) {
2790                 if (PTR_ERR(nfc->reg_clk) != -ENOENT) {
2791                         ret = PTR_ERR(nfc->reg_clk);
2792                         goto unprepare_core_clk;
2793                 }
2794
2795                 nfc->reg_clk = NULL;
2796         }
2797
2798         ret = clk_prepare_enable(nfc->reg_clk);
2799         if (ret)
2800                 goto unprepare_core_clk;
2801
2802         marvell_nfc_disable_int(nfc, NDCR_ALL_INT);
2803         marvell_nfc_clear_int(nfc, NDCR_ALL_INT);
2804         ret = devm_request_irq(dev, irq, marvell_nfc_isr,
2805                                0, "marvell-nfc", nfc);
2806         if (ret)
2807                 goto unprepare_reg_clk;
2808
2809         /* Get NAND controller capabilities */
2810         if (pdev->id_entry)
2811                 nfc->caps = (void *)pdev->id_entry->driver_data;
2812         else
2813                 nfc->caps = of_device_get_match_data(&pdev->dev);
2814
2815         if (!nfc->caps) {
2816                 dev_err(dev, "Could not retrieve NFC caps\n");
2817                 ret = -EINVAL;
2818                 goto unprepare_reg_clk;
2819         }
2820
2821         /* Init the controller and then probe the chips */
2822         ret = marvell_nfc_init(nfc);
2823         if (ret)
2824                 goto unprepare_reg_clk;
2825
2826         platform_set_drvdata(pdev, nfc);
2827
2828         ret = marvell_nand_chips_init(dev, nfc);
2829         if (ret)
2830                 goto unprepare_reg_clk;
2831
2832         return 0;
2833
2834 unprepare_reg_clk:
2835         clk_disable_unprepare(nfc->reg_clk);
2836 unprepare_core_clk:
2837         clk_disable_unprepare(nfc->core_clk);
2838
2839         return ret;
2840 }
2841
2842 static int marvell_nfc_remove(struct platform_device *pdev)
2843 {
2844         struct marvell_nfc *nfc = platform_get_drvdata(pdev);
2845
2846         marvell_nand_chips_cleanup(nfc);
2847
2848         if (nfc->use_dma) {
2849                 dmaengine_terminate_all(nfc->dma_chan);
2850                 dma_release_channel(nfc->dma_chan);
2851         }
2852
2853         clk_disable_unprepare(nfc->reg_clk);
2854         clk_disable_unprepare(nfc->core_clk);
2855
2856         return 0;
2857 }
2858
2859 static int __maybe_unused marvell_nfc_suspend(struct device *dev)
2860 {
2861         struct marvell_nfc *nfc = dev_get_drvdata(dev);
2862         struct marvell_nand_chip *chip;
2863
2864         list_for_each_entry(chip, &nfc->chips, node)
2865                 marvell_nfc_wait_ndrun(&chip->chip);
2866
2867         clk_disable_unprepare(nfc->reg_clk);
2868         clk_disable_unprepare(nfc->core_clk);
2869
2870         return 0;
2871 }
2872
2873 static int __maybe_unused marvell_nfc_resume(struct device *dev)
2874 {
2875         struct marvell_nfc *nfc = dev_get_drvdata(dev);
2876         int ret;
2877
2878         ret = clk_prepare_enable(nfc->core_clk);
2879         if (ret < 0)
2880                 return ret;
2881
2882         ret = clk_prepare_enable(nfc->reg_clk);
2883         if (ret < 0) {
2884                 clk_disable_unprepare(nfc->core_clk);
2885                 return ret;
2886         }
2887
2888         /*
2889          * Reset nfc->selected_chip so the next command will cause the timing
2890          * registers to be restored in marvell_nfc_select_chip().
2891          */
2892         nfc->selected_chip = NULL;
2893
2894         /* Reset registers that have lost their contents */
2895         marvell_nfc_reset(nfc);
2896
2897         return 0;
2898 }
2899
2900 static const struct dev_pm_ops marvell_nfc_pm_ops = {
2901         SET_SYSTEM_SLEEP_PM_OPS(marvell_nfc_suspend, marvell_nfc_resume)
2902 };
2903
2904 static const struct marvell_nfc_caps marvell_armada_8k_nfc_caps = {
2905         .max_cs_nb = 4,
2906         .max_rb_nb = 2,
2907         .need_system_controller = true,
2908         .is_nfcv2 = true,
2909 };
2910
2911 static const struct marvell_nfc_caps marvell_armada370_nfc_caps = {
2912         .max_cs_nb = 4,
2913         .max_rb_nb = 2,
2914         .is_nfcv2 = true,
2915 };
2916
2917 static const struct marvell_nfc_caps marvell_pxa3xx_nfc_caps = {
2918         .max_cs_nb = 2,
2919         .max_rb_nb = 1,
2920         .use_dma = true,
2921 };
2922
2923 static const struct marvell_nfc_caps marvell_armada_8k_nfc_legacy_caps = {
2924         .max_cs_nb = 4,
2925         .max_rb_nb = 2,
2926         .need_system_controller = true,
2927         .legacy_of_bindings = true,
2928         .is_nfcv2 = true,
2929 };
2930
2931 static const struct marvell_nfc_caps marvell_armada370_nfc_legacy_caps = {
2932         .max_cs_nb = 4,
2933         .max_rb_nb = 2,
2934         .legacy_of_bindings = true,
2935         .is_nfcv2 = true,
2936 };
2937
2938 static const struct marvell_nfc_caps marvell_pxa3xx_nfc_legacy_caps = {
2939         .max_cs_nb = 2,
2940         .max_rb_nb = 1,
2941         .legacy_of_bindings = true,
2942         .use_dma = true,
2943 };
2944
2945 static const struct platform_device_id marvell_nfc_platform_ids[] = {
2946         {
2947                 .name = "pxa3xx-nand",
2948                 .driver_data = (kernel_ulong_t)&marvell_pxa3xx_nfc_legacy_caps,
2949         },
2950         { /* sentinel */ },
2951 };
2952 MODULE_DEVICE_TABLE(platform, marvell_nfc_platform_ids);
2953
2954 static const struct of_device_id marvell_nfc_of_ids[] = {
2955         {
2956                 .compatible = "marvell,armada-8k-nand-controller",
2957                 .data = &marvell_armada_8k_nfc_caps,
2958         },
2959         {
2960                 .compatible = "marvell,armada370-nand-controller",
2961                 .data = &marvell_armada370_nfc_caps,
2962         },
2963         {
2964                 .compatible = "marvell,pxa3xx-nand-controller",
2965                 .data = &marvell_pxa3xx_nfc_caps,
2966         },
2967         /* Support for old/deprecated bindings: */
2968         {
2969                 .compatible = "marvell,armada-8k-nand",
2970                 .data = &marvell_armada_8k_nfc_legacy_caps,
2971         },
2972         {
2973                 .compatible = "marvell,armada370-nand",
2974                 .data = &marvell_armada370_nfc_legacy_caps,
2975         },
2976         {
2977                 .compatible = "marvell,pxa3xx-nand",
2978                 .data = &marvell_pxa3xx_nfc_legacy_caps,
2979         },
2980         { /* sentinel */ },
2981 };
2982 MODULE_DEVICE_TABLE(of, marvell_nfc_of_ids);
2983
2984 static struct platform_driver marvell_nfc_driver = {
2985         .driver = {
2986                 .name           = "marvell-nfc",
2987                 .of_match_table = marvell_nfc_of_ids,
2988                 .pm             = &marvell_nfc_pm_ops,
2989         },
2990         .id_table = marvell_nfc_platform_ids,
2991         .probe = marvell_nfc_probe,
2992         .remove = marvell_nfc_remove,
2993 };
2994 module_platform_driver(marvell_nfc_driver);
2995
2996 MODULE_LICENSE("GPL");
2997 MODULE_DESCRIPTION("Marvell NAND controller driver");